Cleaning, Preprocessing and Forming of the dataset.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Importing relevant libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import sklearn as sk\n",
    "import matplotlib as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dataset containing the features of songs (acousticness, danceability......)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "spotifydf = pd.read_csv('data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>acousticness</th>\n",
       "      <th>artist_name</th>\n",
       "      <th>danceability</th>\n",
       "      <th>duration_ms</th>\n",
       "      <th>energy</th>\n",
       "      <th>explicit</th>\n",
       "      <th>id</th>\n",
       "      <th>instrumentalness</th>\n",
       "      <th>key</th>\n",
       "      <th>liveness</th>\n",
       "      <th>loudness</th>\n",
       "      <th>mode</th>\n",
       "      <th>title</th>\n",
       "      <th>popularity</th>\n",
       "      <th>release_date</th>\n",
       "      <th>speechiness</th>\n",
       "      <th>tempo</th>\n",
       "      <th>valence</th>\n",
       "      <th>year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.995</td>\n",
       "      <td>Carl Woitschach</td>\n",
       "      <td>0.708</td>\n",
       "      <td>158648</td>\n",
       "      <td>0.1950</td>\n",
       "      <td>0</td>\n",
       "      <td>6KbQ3uYMLKb5jDxLF7wYDD</td>\n",
       "      <td>0.563</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1510</td>\n",
       "      <td>-12.428</td>\n",
       "      <td>1</td>\n",
       "      <td>Singende Bataillone 1. Teil</td>\n",
       "      <td>0</td>\n",
       "      <td>1928</td>\n",
       "      <td>0.0506</td>\n",
       "      <td>118.469</td>\n",
       "      <td>0.7790</td>\n",
       "      <td>1928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.994</td>\n",
       "      <td>Robert Schumann &amp; Vladimir Horowitz</td>\n",
       "      <td>0.379</td>\n",
       "      <td>282133</td>\n",
       "      <td>0.0135</td>\n",
       "      <td>0</td>\n",
       "      <td>6KuQTIu1KoTTkLXKrwlLPV</td>\n",
       "      <td>0.901</td>\n",
       "      <td>8</td>\n",
       "      <td>0.0763</td>\n",
       "      <td>-28.454</td>\n",
       "      <td>1</td>\n",
       "      <td>Fantasiestücke, Op. 111: Più tosto lento</td>\n",
       "      <td>0</td>\n",
       "      <td>1928</td>\n",
       "      <td>0.0462</td>\n",
       "      <td>83.972</td>\n",
       "      <td>0.0767</td>\n",
       "      <td>1928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.604</td>\n",
       "      <td>Seweryn Goszczyński</td>\n",
       "      <td>0.749</td>\n",
       "      <td>104300</td>\n",
       "      <td>0.2200</td>\n",
       "      <td>0</td>\n",
       "      <td>6L63VW0PibdM1HDSBoqnoM</td>\n",
       "      <td>0.000</td>\n",
       "      <td>5</td>\n",
       "      <td>0.1190</td>\n",
       "      <td>-19.924</td>\n",
       "      <td>0</td>\n",
       "      <td>Chapter 1.18 - Zamek kaniowski</td>\n",
       "      <td>0</td>\n",
       "      <td>1928</td>\n",
       "      <td>0.9290</td>\n",
       "      <td>107.177</td>\n",
       "      <td>0.8800</td>\n",
       "      <td>1928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.995</td>\n",
       "      <td>Francisco Canaro</td>\n",
       "      <td>0.781</td>\n",
       "      <td>180760</td>\n",
       "      <td>0.1300</td>\n",
       "      <td>0</td>\n",
       "      <td>6M94FkXd15sOAOQYRnWPN8</td>\n",
       "      <td>0.887</td>\n",
       "      <td>1</td>\n",
       "      <td>0.1110</td>\n",
       "      <td>-14.734</td>\n",
       "      <td>0</td>\n",
       "      <td>Bebamos Juntos - Instrumental (Remasterizado)</td>\n",
       "      <td>0</td>\n",
       "      <td>1928-09-25</td>\n",
       "      <td>0.0926</td>\n",
       "      <td>108.003</td>\n",
       "      <td>0.7200</td>\n",
       "      <td>1928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.990</td>\n",
       "      <td>Frédéric Chopin &amp; Vladimir Horowitz</td>\n",
       "      <td>0.210</td>\n",
       "      <td>687733</td>\n",
       "      <td>0.2040</td>\n",
       "      <td>0</td>\n",
       "      <td>6N6tiFZ9vLTSOIxkj8qKrd</td>\n",
       "      <td>0.908</td>\n",
       "      <td>11</td>\n",
       "      <td>0.0980</td>\n",
       "      <td>-16.829</td>\n",
       "      <td>1</td>\n",
       "      <td>Polonaise-Fantaisie in A-Flat Major, Op. 61</td>\n",
       "      <td>1</td>\n",
       "      <td>1928</td>\n",
       "      <td>0.0424</td>\n",
       "      <td>62.149</td>\n",
       "      <td>0.0693</td>\n",
       "      <td>1928</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   acousticness                          artist_name  danceability  \\\n",
       "0         0.995                      Carl Woitschach         0.708   \n",
       "1         0.994  Robert Schumann & Vladimir Horowitz         0.379   \n",
       "2         0.604                  Seweryn Goszczyński         0.749   \n",
       "3         0.995                     Francisco Canaro         0.781   \n",
       "4         0.990  Frédéric Chopin & Vladimir Horowitz         0.210   \n",
       "\n",
       "   duration_ms  energy  explicit                      id  instrumentalness  \\\n",
       "0       158648  0.1950         0  6KbQ3uYMLKb5jDxLF7wYDD             0.563   \n",
       "1       282133  0.0135         0  6KuQTIu1KoTTkLXKrwlLPV             0.901   \n",
       "2       104300  0.2200         0  6L63VW0PibdM1HDSBoqnoM             0.000   \n",
       "3       180760  0.1300         0  6M94FkXd15sOAOQYRnWPN8             0.887   \n",
       "4       687733  0.2040         0  6N6tiFZ9vLTSOIxkj8qKrd             0.908   \n",
       "\n",
       "   key  liveness  loudness  mode  \\\n",
       "0   10    0.1510   -12.428     1   \n",
       "1    8    0.0763   -28.454     1   \n",
       "2    5    0.1190   -19.924     0   \n",
       "3    1    0.1110   -14.734     0   \n",
       "4   11    0.0980   -16.829     1   \n",
       "\n",
       "                                           title  popularity release_date  \\\n",
       "0                    Singende Bataillone 1. Teil           0         1928   \n",
       "1       Fantasiestücke, Op. 111: Più tosto lento           0         1928   \n",
       "2                 Chapter 1.18 - Zamek kaniowski           0         1928   \n",
       "3  Bebamos Juntos - Instrumental (Remasterizado)           0   1928-09-25   \n",
       "4    Polonaise-Fantaisie in A-Flat Major, Op. 61           1         1928   \n",
       "\n",
       "   speechiness    tempo  valence  year  \n",
       "0       0.0506  118.469   0.7790  1928  \n",
       "1       0.0462   83.972   0.0767  1928  \n",
       "2       0.9290  107.177   0.8800  1928  \n",
       "3       0.0926  108.003   0.7200  1928  \n",
       "4       0.0424   62.149   0.0693  1928  "
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spotifydf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "spotifydf = spotifydf.drop(['duration_ms','explicit','mode','release_date'],axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cleaning the spotify dataset by dropping irrelevant features above, and normalising the un-normalised features below"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import normalize\n",
    "x_array = np.array([spotifydf['tempo'],spotifydf['loudness']])\n",
    "normalized_x =normalize(x_array)\n",
    "spotifydf = spotifydf.drop(['tempo','loudness'],axis=1)\n",
    "spotifydf.insert(3,'tempo',normalized_x[0])\n",
    "spotifydf.insert(4,'loudness',normalized_x[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>acousticness</th>\n",
       "      <th>artist_name</th>\n",
       "      <th>danceability</th>\n",
       "      <th>tempo</th>\n",
       "      <th>loudness</th>\n",
       "      <th>energy</th>\n",
       "      <th>id</th>\n",
       "      <th>instrumentalness</th>\n",
       "      <th>key</th>\n",
       "      <th>liveness</th>\n",
       "      <th>title</th>\n",
       "      <th>popularity</th>\n",
       "      <th>speechiness</th>\n",
       "      <th>valence</th>\n",
       "      <th>year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.995</td>\n",
       "      <td>Carl Woitschach</td>\n",
       "      <td>0.708</td>\n",
       "      <td>0.002377</td>\n",
       "      <td>-0.002373</td>\n",
       "      <td>0.1950</td>\n",
       "      <td>6KbQ3uYMLKb5jDxLF7wYDD</td>\n",
       "      <td>0.563</td>\n",
       "      <td>10</td>\n",
       "      <td>0.1510</td>\n",
       "      <td>Singende Bataillone 1. Teil</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0506</td>\n",
       "      <td>0.7790</td>\n",
       "      <td>1928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.994</td>\n",
       "      <td>Robert Schumann &amp; Vladimir Horowitz</td>\n",
       "      <td>0.379</td>\n",
       "      <td>0.001685</td>\n",
       "      <td>-0.005434</td>\n",
       "      <td>0.0135</td>\n",
       "      <td>6KuQTIu1KoTTkLXKrwlLPV</td>\n",
       "      <td>0.901</td>\n",
       "      <td>8</td>\n",
       "      <td>0.0763</td>\n",
       "      <td>Fantasiestücke, Op. 111: Più tosto lento</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0462</td>\n",
       "      <td>0.0767</td>\n",
       "      <td>1928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.604</td>\n",
       "      <td>Seweryn Goszczyński</td>\n",
       "      <td>0.749</td>\n",
       "      <td>0.002150</td>\n",
       "      <td>-0.003805</td>\n",
       "      <td>0.2200</td>\n",
       "      <td>6L63VW0PibdM1HDSBoqnoM</td>\n",
       "      <td>0.000</td>\n",
       "      <td>5</td>\n",
       "      <td>0.1190</td>\n",
       "      <td>Chapter 1.18 - Zamek kaniowski</td>\n",
       "      <td>0</td>\n",
       "      <td>0.9290</td>\n",
       "      <td>0.8800</td>\n",
       "      <td>1928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.995</td>\n",
       "      <td>Francisco Canaro</td>\n",
       "      <td>0.781</td>\n",
       "      <td>0.002167</td>\n",
       "      <td>-0.002814</td>\n",
       "      <td>0.1300</td>\n",
       "      <td>6M94FkXd15sOAOQYRnWPN8</td>\n",
       "      <td>0.887</td>\n",
       "      <td>1</td>\n",
       "      <td>0.1110</td>\n",
       "      <td>Bebamos Juntos - Instrumental (Remasterizado)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0926</td>\n",
       "      <td>0.7200</td>\n",
       "      <td>1928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.990</td>\n",
       "      <td>Frédéric Chopin &amp; Vladimir Horowitz</td>\n",
       "      <td>0.210</td>\n",
       "      <td>0.001247</td>\n",
       "      <td>-0.003214</td>\n",
       "      <td>0.2040</td>\n",
       "      <td>6N6tiFZ9vLTSOIxkj8qKrd</td>\n",
       "      <td>0.908</td>\n",
       "      <td>11</td>\n",
       "      <td>0.0980</td>\n",
       "      <td>Polonaise-Fantaisie in A-Flat Major, Op. 61</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0424</td>\n",
       "      <td>0.0693</td>\n",
       "      <td>1928</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   acousticness                          artist_name  danceability     tempo  \\\n",
       "0         0.995                      Carl Woitschach         0.708  0.002377   \n",
       "1         0.994  Robert Schumann & Vladimir Horowitz         0.379  0.001685   \n",
       "2         0.604                  Seweryn Goszczyński         0.749  0.002150   \n",
       "3         0.995                     Francisco Canaro         0.781  0.002167   \n",
       "4         0.990  Frédéric Chopin & Vladimir Horowitz         0.210  0.001247   \n",
       "\n",
       "   loudness  energy                      id  instrumentalness  key  liveness  \\\n",
       "0 -0.002373  0.1950  6KbQ3uYMLKb5jDxLF7wYDD             0.563   10    0.1510   \n",
       "1 -0.005434  0.0135  6KuQTIu1KoTTkLXKrwlLPV             0.901    8    0.0763   \n",
       "2 -0.003805  0.2200  6L63VW0PibdM1HDSBoqnoM             0.000    5    0.1190   \n",
       "3 -0.002814  0.1300  6M94FkXd15sOAOQYRnWPN8             0.887    1    0.1110   \n",
       "4 -0.003214  0.2040  6N6tiFZ9vLTSOIxkj8qKrd             0.908   11    0.0980   \n",
       "\n",
       "                                           title  popularity  speechiness  \\\n",
       "0                    Singende Bataillone 1. Teil           0       0.0506   \n",
       "1       Fantasiestücke, Op. 111: Più tosto lento           0       0.0462   \n",
       "2                 Chapter 1.18 - Zamek kaniowski           0       0.9290   \n",
       "3  Bebamos Juntos - Instrumental (Remasterizado)           0       0.0926   \n",
       "4    Polonaise-Fantaisie in A-Flat Major, Op. 61           1       0.0424   \n",
       "\n",
       "   valence  year  \n",
       "0   0.7790  1928  \n",
       "1   0.0767  1928  \n",
       "2   0.8800  1928  \n",
       "3   0.7200  1928  \n",
       "4   0.0693  1928  "
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spotifydf.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### songdatamsd contains the song_id & song name, songdataplaycount contains song_id, user_id and play_count  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "msddf=pd.read_csv('songdatamsd.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "msddfplay=pd.read_csv('songdataplaycount.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "msddf=msddf.merge(msddfplay)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "163206"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "msddf['song_id'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>song_id</th>\n",
       "      <th>title</th>\n",
       "      <th>release</th>\n",
       "      <th>artist_name</th>\n",
       "      <th>year</th>\n",
       "      <th>user_id</th>\n",
       "      <th>play_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>SOIMMJJ12AF72AD643</td>\n",
       "      <td>Rock-N-Rule</td>\n",
       "      <td>I'm Only A Man (Bonus Track Version)</td>\n",
       "      <td>Emery</td>\n",
       "      <td>2007</td>\n",
       "      <td>9d1a3409803399b141edf3fc4215c6e3d1006f3a</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>SOIMMJJ12AF72AD643</td>\n",
       "      <td>Rock-N-Rule</td>\n",
       "      <td>I'm Only A Man (Bonus Track Version)</td>\n",
       "      <td>Emery</td>\n",
       "      <td>2007</td>\n",
       "      <td>81a064843a1f8d5ace2315d908b5fef570274f5d</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>SOIMMJJ12AF72AD643</td>\n",
       "      <td>Rock-N-Rule</td>\n",
       "      <td>I'm Only A Man (Bonus Track Version)</td>\n",
       "      <td>Emery</td>\n",
       "      <td>2007</td>\n",
       "      <td>0cf16c7e8732c041fea86777b6a13c7f82fce492</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>SOIMMJJ12AF72AD643</td>\n",
       "      <td>Rock-N-Rule</td>\n",
       "      <td>I'm Only A Man (Bonus Track Version)</td>\n",
       "      <td>Emery</td>\n",
       "      <td>2007</td>\n",
       "      <td>f34abd0f222d240f9cd7cf21f0707cb5222e01d6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>SOGNNYL12A6D4F910B</td>\n",
       "      <td>Raspberry Beret (LP Version)</td>\n",
       "      <td>The Hits/The B-Sides</td>\n",
       "      <td>Prince &amp; The Revolution</td>\n",
       "      <td>0</td>\n",
       "      <td>04383d89810d0113066c28b631097711d2550b04</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              song_id                         title  \\\n",
       "0  SOIMMJJ12AF72AD643                   Rock-N-Rule   \n",
       "1  SOIMMJJ12AF72AD643                   Rock-N-Rule   \n",
       "2  SOIMMJJ12AF72AD643                   Rock-N-Rule   \n",
       "3  SOIMMJJ12AF72AD643                   Rock-N-Rule   \n",
       "4  SOGNNYL12A6D4F910B  Raspberry Beret (LP Version)   \n",
       "\n",
       "                                release              artist_name  year  \\\n",
       "0  I'm Only A Man (Bonus Track Version)                    Emery  2007   \n",
       "1  I'm Only A Man (Bonus Track Version)                    Emery  2007   \n",
       "2  I'm Only A Man (Bonus Track Version)                    Emery  2007   \n",
       "3  I'm Only A Man (Bonus Track Version)                    Emery  2007   \n",
       "4                  The Hits/The B-Sides  Prince & The Revolution     0   \n",
       "\n",
       "                                    user_id  play_count  \n",
       "0  9d1a3409803399b141edf3fc4215c6e3d1006f3a           1  \n",
       "1  81a064843a1f8d5ace2315d908b5fef570274f5d           2  \n",
       "2  0cf16c7e8732c041fea86777b6a13c7f82fce492           2  \n",
       "3  f34abd0f222d240f9cd7cf21f0707cb5222e01d6           1  \n",
       "4  04383d89810d0113066c28b631097711d2550b04          16  "
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "msddf.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Encoding the user_id because they are irrelevant and they take too much space, and again, deleting irrelevant columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "le = LabelEncoder()\n",
    "x=np.array(msddf['user_id'])\n",
    "x= le.fit_transform(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "110000"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "msddf['user_id'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "msddf = msddf.drop(['release','year','song_id'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>artist_name</th>\n",
       "      <th>user_id</th>\n",
       "      <th>play_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>Rock-N-Rule</td>\n",
       "      <td>Emery</td>\n",
       "      <td>9d1a3409803399b141edf3fc4215c6e3d1006f3a</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>Rock-N-Rule</td>\n",
       "      <td>Emery</td>\n",
       "      <td>81a064843a1f8d5ace2315d908b5fef570274f5d</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>Rock-N-Rule</td>\n",
       "      <td>Emery</td>\n",
       "      <td>0cf16c7e8732c041fea86777b6a13c7f82fce492</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>Rock-N-Rule</td>\n",
       "      <td>Emery</td>\n",
       "      <td>f34abd0f222d240f9cd7cf21f0707cb5222e01d6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>Raspberry Beret (LP Version)</td>\n",
       "      <td>Prince &amp; The Revolution</td>\n",
       "      <td>04383d89810d0113066c28b631097711d2550b04</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          title              artist_name  \\\n",
       "0                   Rock-N-Rule                    Emery   \n",
       "1                   Rock-N-Rule                    Emery   \n",
       "2                   Rock-N-Rule                    Emery   \n",
       "3                   Rock-N-Rule                    Emery   \n",
       "4  Raspberry Beret (LP Version)  Prince & The Revolution   \n",
       "\n",
       "                                    user_id  play_count  \n",
       "0  9d1a3409803399b141edf3fc4215c6e3d1006f3a           1  \n",
       "1  81a064843a1f8d5ace2315d908b5fef570274f5d           2  \n",
       "2  0cf16c7e8732c041fea86777b6a13c7f82fce492           2  \n",
       "3  f34abd0f222d240f9cd7cf21f0707cb5222e01d6           1  \n",
       "4  04383d89810d0113066c28b631097711d2550b04          16  "
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "msddf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "msddf = msddf.drop(['user_id'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>artist_name</th>\n",
       "      <th>play_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>Rock-N-Rule</td>\n",
       "      <td>Emery</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>Rock-N-Rule</td>\n",
       "      <td>Emery</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>Rock-N-Rule</td>\n",
       "      <td>Emery</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>Rock-N-Rule</td>\n",
       "      <td>Emery</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>Raspberry Beret (LP Version)</td>\n",
       "      <td>Prince &amp; The Revolution</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          title              artist_name  play_count\n",
       "0                   Rock-N-Rule                    Emery           1\n",
       "1                   Rock-N-Rule                    Emery           2\n",
       "2                   Rock-N-Rule                    Emery           2\n",
       "3                   Rock-N-Rule                    Emery           1\n",
       "4  Raspberry Beret (LP Version)  Prince & The Revolution          16"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "msddf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "msddf.insert(1, 'user_id', x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>user_id</th>\n",
       "      <th>artist_name</th>\n",
       "      <th>play_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>Rock-N-Rule</td>\n",
       "      <td>67402</td>\n",
       "      <td>Emery</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>Rock-N-Rule</td>\n",
       "      <td>55644</td>\n",
       "      <td>Emery</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>Rock-N-Rule</td>\n",
       "      <td>5600</td>\n",
       "      <td>Emery</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>Rock-N-Rule</td>\n",
       "      <td>104380</td>\n",
       "      <td>Emery</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>Raspberry Beret (LP Version)</td>\n",
       "      <td>1812</td>\n",
       "      <td>Prince &amp; The Revolution</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          title  user_id              artist_name  play_count\n",
       "0                   Rock-N-Rule    67402                    Emery           1\n",
       "1                   Rock-N-Rule    55644                    Emery           2\n",
       "2                   Rock-N-Rule     5600                    Emery           2\n",
       "3                   Rock-N-Rule   104380                    Emery           1\n",
       "4  Raspberry Beret (LP Version)     1812  Prince & The Revolution          16"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "msddf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "109999"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "msddf['user_id'].max()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### merging the information from spotify & million song dataset based on a combo of title and artist since the song_ids are totally different"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "combo1= np.array(spotifydf['title']+ ' '+spotifydf['artist_name'])\n",
    "combo2= np.array(msddf['title']+ ' '+msddf['artist_name'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "spotifydf=spotifydf.drop(['title','artist_name'],axis=1)\n",
    "spotifydf.insert(1,'name', combo1)\n",
    "msddf=msddf.drop(['title','artist_name'],axis=1)\n",
    "msddf.insert(1, 'name', combo2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>acousticness</th>\n",
       "      <th>name</th>\n",
       "      <th>danceability</th>\n",
       "      <th>tempo</th>\n",
       "      <th>loudness</th>\n",
       "      <th>energy</th>\n",
       "      <th>id</th>\n",
       "      <th>instrumentalness</th>\n",
       "      <th>key</th>\n",
       "      <th>liveness</th>\n",
       "      <th>popularity</th>\n",
       "      <th>speechiness</th>\n",
       "      <th>valence</th>\n",
       "      <th>year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.995</td>\n",
       "      <td>Singende Bataillone 1. Teil Carl Woitschach</td>\n",
       "      <td>0.708</td>\n",
       "      <td>0.002377</td>\n",
       "      <td>-0.002373</td>\n",
       "      <td>0.195</td>\n",
       "      <td>6KbQ3uYMLKb5jDxLF7wYDD</td>\n",
       "      <td>0.563</td>\n",
       "      <td>10</td>\n",
       "      <td>0.151</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0506</td>\n",
       "      <td>0.779</td>\n",
       "      <td>1928</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   acousticness                                         name  danceability  \\\n",
       "0         0.995  Singende Bataillone 1. Teil Carl Woitschach         0.708   \n",
       "\n",
       "      tempo  loudness  energy                      id  instrumentalness  key  \\\n",
       "0  0.002377 -0.002373   0.195  6KbQ3uYMLKb5jDxLF7wYDD             0.563   10   \n",
       "\n",
       "   liveness  popularity  speechiness  valence  year  \n",
       "0     0.151           0       0.0506    0.779  1928  "
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spotifydf.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>name</th>\n",
       "      <th>play_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>67402</td>\n",
       "      <td>Rock-N-Rule Emery</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id               name  play_count\n",
       "0    67402  Rock-N-Rule Emery           1"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "msddf.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "final = pd.merge(spotifydf, msddf, on = 'name', how = 'outer')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>acousticness</th>\n",
       "      <th>name</th>\n",
       "      <th>danceability</th>\n",
       "      <th>tempo</th>\n",
       "      <th>loudness</th>\n",
       "      <th>energy</th>\n",
       "      <th>id</th>\n",
       "      <th>instrumentalness</th>\n",
       "      <th>key</th>\n",
       "      <th>liveness</th>\n",
       "      <th>popularity</th>\n",
       "      <th>speechiness</th>\n",
       "      <th>valence</th>\n",
       "      <th>year</th>\n",
       "      <th>user_id</th>\n",
       "      <th>play_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.995</td>\n",
       "      <td>Singende Bataillone 1. Teil Carl Woitschach</td>\n",
       "      <td>0.708</td>\n",
       "      <td>0.002377</td>\n",
       "      <td>-0.002373</td>\n",
       "      <td>0.1950</td>\n",
       "      <td>6KbQ3uYMLKb5jDxLF7wYDD</td>\n",
       "      <td>0.563</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.1510</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0506</td>\n",
       "      <td>0.7790</td>\n",
       "      <td>1928.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.994</td>\n",
       "      <td>Fantasiestücke, Op. 111: Più tosto lento Rober...</td>\n",
       "      <td>0.379</td>\n",
       "      <td>0.001685</td>\n",
       "      <td>-0.005434</td>\n",
       "      <td>0.0135</td>\n",
       "      <td>6KuQTIu1KoTTkLXKrwlLPV</td>\n",
       "      <td>0.901</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0763</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0462</td>\n",
       "      <td>0.0767</td>\n",
       "      <td>1928.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.994</td>\n",
       "      <td>Fantasiestücke, Op. 111: Più tosto lento Rober...</td>\n",
       "      <td>0.379</td>\n",
       "      <td>0.001685</td>\n",
       "      <td>-0.005434</td>\n",
       "      <td>0.0135</td>\n",
       "      <td>2YPbmbRIo7I1r9FAMqNjbZ</td>\n",
       "      <td>0.901</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0763</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0462</td>\n",
       "      <td>0.0767</td>\n",
       "      <td>1928.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.604</td>\n",
       "      <td>Chapter 1.18 - Zamek kaniowski Seweryn Goszczy...</td>\n",
       "      <td>0.749</td>\n",
       "      <td>0.002150</td>\n",
       "      <td>-0.003805</td>\n",
       "      <td>0.2200</td>\n",
       "      <td>6L63VW0PibdM1HDSBoqnoM</td>\n",
       "      <td>0.000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.1190</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.9290</td>\n",
       "      <td>0.8800</td>\n",
       "      <td>1928.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.995</td>\n",
       "      <td>Bebamos Juntos - Instrumental (Remasterizado) ...</td>\n",
       "      <td>0.781</td>\n",
       "      <td>0.002167</td>\n",
       "      <td>-0.002814</td>\n",
       "      <td>0.1300</td>\n",
       "      <td>6M94FkXd15sOAOQYRnWPN8</td>\n",
       "      <td>0.887</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.1110</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0926</td>\n",
       "      <td>0.7200</td>\n",
       "      <td>1928.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   acousticness                                               name  \\\n",
       "0         0.995        Singende Bataillone 1. Teil Carl Woitschach   \n",
       "1         0.994  Fantasiestücke, Op. 111: Più tosto lento Rober...   \n",
       "2         0.994  Fantasiestücke, Op. 111: Più tosto lento Rober...   \n",
       "3         0.604  Chapter 1.18 - Zamek kaniowski Seweryn Goszczy...   \n",
       "4         0.995  Bebamos Juntos - Instrumental (Remasterizado) ...   \n",
       "\n",
       "   danceability     tempo  loudness  energy                      id  \\\n",
       "0         0.708  0.002377 -0.002373  0.1950  6KbQ3uYMLKb5jDxLF7wYDD   \n",
       "1         0.379  0.001685 -0.005434  0.0135  6KuQTIu1KoTTkLXKrwlLPV   \n",
       "2         0.379  0.001685 -0.005434  0.0135  2YPbmbRIo7I1r9FAMqNjbZ   \n",
       "3         0.749  0.002150 -0.003805  0.2200  6L63VW0PibdM1HDSBoqnoM   \n",
       "4         0.781  0.002167 -0.002814  0.1300  6M94FkXd15sOAOQYRnWPN8   \n",
       "\n",
       "   instrumentalness   key  liveness  popularity  speechiness  valence    year  \\\n",
       "0             0.563  10.0    0.1510         0.0       0.0506   0.7790  1928.0   \n",
       "1             0.901   8.0    0.0763         0.0       0.0462   0.0767  1928.0   \n",
       "2             0.901   8.0    0.0763         0.0       0.0462   0.0767  1928.0   \n",
       "3             0.000   5.0    0.1190         0.0       0.9290   0.8800  1928.0   \n",
       "4             0.887   1.0    0.1110         0.0       0.0926   0.7200  1928.0   \n",
       "\n",
       "   user_id  play_count  \n",
       "0      NaN         NaN  \n",
       "1      NaN         NaN  \n",
       "2      NaN         NaN  \n",
       "3      NaN         NaN  \n",
       "4      NaN         NaN  "
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1784038, 16)"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Deleting all the entries which we do not have features for, we however saved entries we did not have user info for"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0         0.9950\n",
       "1         0.9940\n",
       "2         0.9940\n",
       "3         0.6040\n",
       "4         0.9950\n",
       "           ...  \n",
       "660715    0.2100\n",
       "660716    0.1730\n",
       "660717    0.0167\n",
       "660718    0.5380\n",
       "660719    0.0714\n",
       "Name: acousticness, Length: 660720, dtype: float64"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final['acousticness'].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>acousticness</th>\n",
       "      <th>name</th>\n",
       "      <th>danceability</th>\n",
       "      <th>tempo</th>\n",
       "      <th>loudness</th>\n",
       "      <th>energy</th>\n",
       "      <th>id</th>\n",
       "      <th>instrumentalness</th>\n",
       "      <th>key</th>\n",
       "      <th>liveness</th>\n",
       "      <th>popularity</th>\n",
       "      <th>speechiness</th>\n",
       "      <th>valence</th>\n",
       "      <th>year</th>\n",
       "      <th>user_id</th>\n",
       "      <th>play_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.9950</td>\n",
       "      <td>Singende Bataillone 1. Teil Carl Woitschach</td>\n",
       "      <td>0.708</td>\n",
       "      <td>0.002377</td>\n",
       "      <td>-0.002373</td>\n",
       "      <td>0.1950</td>\n",
       "      <td>6KbQ3uYMLKb5jDxLF7wYDD</td>\n",
       "      <td>0.563000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.1510</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0506</td>\n",
       "      <td>0.7790</td>\n",
       "      <td>1928.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.9940</td>\n",
       "      <td>Fantasiestücke, Op. 111: Più tosto lento Rober...</td>\n",
       "      <td>0.379</td>\n",
       "      <td>0.001685</td>\n",
       "      <td>-0.005434</td>\n",
       "      <td>0.0135</td>\n",
       "      <td>6KuQTIu1KoTTkLXKrwlLPV</td>\n",
       "      <td>0.901000</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0763</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0462</td>\n",
       "      <td>0.0767</td>\n",
       "      <td>1928.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0.9940</td>\n",
       "      <td>Fantasiestücke, Op. 111: Più tosto lento Rober...</td>\n",
       "      <td>0.379</td>\n",
       "      <td>0.001685</td>\n",
       "      <td>-0.005434</td>\n",
       "      <td>0.0135</td>\n",
       "      <td>2YPbmbRIo7I1r9FAMqNjbZ</td>\n",
       "      <td>0.901000</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0763</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0462</td>\n",
       "      <td>0.0767</td>\n",
       "      <td>1928.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0.6040</td>\n",
       "      <td>Chapter 1.18 - Zamek kaniowski Seweryn Goszczy...</td>\n",
       "      <td>0.749</td>\n",
       "      <td>0.002150</td>\n",
       "      <td>-0.003805</td>\n",
       "      <td>0.2200</td>\n",
       "      <td>6L63VW0PibdM1HDSBoqnoM</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.1190</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.9290</td>\n",
       "      <td>0.8800</td>\n",
       "      <td>1928.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>0.9950</td>\n",
       "      <td>Bebamos Juntos - Instrumental (Remasterizado) ...</td>\n",
       "      <td>0.781</td>\n",
       "      <td>0.002167</td>\n",
       "      <td>-0.002814</td>\n",
       "      <td>0.1300</td>\n",
       "      <td>6M94FkXd15sOAOQYRnWPN8</td>\n",
       "      <td>0.887000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.1110</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0926</td>\n",
       "      <td>0.7200</td>\n",
       "      <td>1928.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>660715</td>\n",
       "      <td>660715</td>\n",
       "      <td>0.2100</td>\n",
       "      <td>Ojos De Maniaco LEGADO 7 &amp; Junior H</td>\n",
       "      <td>0.795</td>\n",
       "      <td>0.001956</td>\n",
       "      <td>-0.000850</td>\n",
       "      <td>0.5850</td>\n",
       "      <td>52Cpyvd2dKb6XRn313nH87</td>\n",
       "      <td>0.000001</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.1120</td>\n",
       "      <td>68.0</td>\n",
       "      <td>0.0374</td>\n",
       "      <td>0.9340</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>660716</td>\n",
       "      <td>660716</td>\n",
       "      <td>0.1730</td>\n",
       "      <td>Skechers (feat. Tyga) - Remix DripReport &amp; Tyga</td>\n",
       "      <td>0.875</td>\n",
       "      <td>0.002007</td>\n",
       "      <td>-0.001425</td>\n",
       "      <td>0.4430</td>\n",
       "      <td>4KppkflX7I3vJQk7urOJaS</td>\n",
       "      <td>0.000032</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0891</td>\n",
       "      <td>75.0</td>\n",
       "      <td>0.1430</td>\n",
       "      <td>0.3060</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>660717</td>\n",
       "      <td>660717</td>\n",
       "      <td>0.0167</td>\n",
       "      <td>Sweeter (feat. Terrace Martin) Leon Bridges &amp; ...</td>\n",
       "      <td>0.719</td>\n",
       "      <td>0.002568</td>\n",
       "      <td>-0.002083</td>\n",
       "      <td>0.3850</td>\n",
       "      <td>1ehhGlTvjtHo2e4xJFB0SZ</td>\n",
       "      <td>0.031300</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.1110</td>\n",
       "      <td>64.0</td>\n",
       "      <td>0.0403</td>\n",
       "      <td>0.2700</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>660718</td>\n",
       "      <td>660718</td>\n",
       "      <td>0.5380</td>\n",
       "      <td>How Would I Know Kygo &amp; Oh Wonder</td>\n",
       "      <td>0.514</td>\n",
       "      <td>0.002482</td>\n",
       "      <td>-0.001782</td>\n",
       "      <td>0.5390</td>\n",
       "      <td>52eycxprLhK3lPcRLbQiVk</td>\n",
       "      <td>0.002330</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.1080</td>\n",
       "      <td>70.0</td>\n",
       "      <td>0.1050</td>\n",
       "      <td>0.1530</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>660719</td>\n",
       "      <td>660719</td>\n",
       "      <td>0.0714</td>\n",
       "      <td>I Found You Cash Cash &amp; Andy Grammer</td>\n",
       "      <td>0.646</td>\n",
       "      <td>0.002607</td>\n",
       "      <td>-0.000488</td>\n",
       "      <td>0.7610</td>\n",
       "      <td>3wYOGJYD31sLRmBgCvWxa4</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.2220</td>\n",
       "      <td>70.0</td>\n",
       "      <td>0.0385</td>\n",
       "      <td>0.4720</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>660720 rows × 17 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         index  acousticness  \\\n",
       "0            0        0.9950   \n",
       "1            1        0.9940   \n",
       "2            2        0.9940   \n",
       "3            3        0.6040   \n",
       "4            4        0.9950   \n",
       "...        ...           ...   \n",
       "660715  660715        0.2100   \n",
       "660716  660716        0.1730   \n",
       "660717  660717        0.0167   \n",
       "660718  660718        0.5380   \n",
       "660719  660719        0.0714   \n",
       "\n",
       "                                                     name  danceability  \\\n",
       "0             Singende Bataillone 1. Teil Carl Woitschach         0.708   \n",
       "1       Fantasiestücke, Op. 111: Più tosto lento Rober...         0.379   \n",
       "2       Fantasiestücke, Op. 111: Più tosto lento Rober...         0.379   \n",
       "3       Chapter 1.18 - Zamek kaniowski Seweryn Goszczy...         0.749   \n",
       "4       Bebamos Juntos - Instrumental (Remasterizado) ...         0.781   \n",
       "...                                                   ...           ...   \n",
       "660715                Ojos De Maniaco LEGADO 7 & Junior H         0.795   \n",
       "660716    Skechers (feat. Tyga) - Remix DripReport & Tyga         0.875   \n",
       "660717  Sweeter (feat. Terrace Martin) Leon Bridges & ...         0.719   \n",
       "660718                  How Would I Know Kygo & Oh Wonder         0.514   \n",
       "660719               I Found You Cash Cash & Andy Grammer         0.646   \n",
       "\n",
       "           tempo  loudness  energy                      id  instrumentalness  \\\n",
       "0       0.002377 -0.002373  0.1950  6KbQ3uYMLKb5jDxLF7wYDD          0.563000   \n",
       "1       0.001685 -0.005434  0.0135  6KuQTIu1KoTTkLXKrwlLPV          0.901000   \n",
       "2       0.001685 -0.005434  0.0135  2YPbmbRIo7I1r9FAMqNjbZ          0.901000   \n",
       "3       0.002150 -0.003805  0.2200  6L63VW0PibdM1HDSBoqnoM          0.000000   \n",
       "4       0.002167 -0.002814  0.1300  6M94FkXd15sOAOQYRnWPN8          0.887000   \n",
       "...          ...       ...     ...                     ...               ...   \n",
       "660715  0.001956 -0.000850  0.5850  52Cpyvd2dKb6XRn313nH87          0.000001   \n",
       "660716  0.002007 -0.001425  0.4430  4KppkflX7I3vJQk7urOJaS          0.000032   \n",
       "660717  0.002568 -0.002083  0.3850  1ehhGlTvjtHo2e4xJFB0SZ          0.031300   \n",
       "660718  0.002482 -0.001782  0.5390  52eycxprLhK3lPcRLbQiVk          0.002330   \n",
       "660719  0.002607 -0.000488  0.7610  3wYOGJYD31sLRmBgCvWxa4          0.000000   \n",
       "\n",
       "         key  liveness  popularity  speechiness  valence    year  user_id  \\\n",
       "0       10.0    0.1510         0.0       0.0506   0.7790  1928.0      NaN   \n",
       "1        8.0    0.0763         0.0       0.0462   0.0767  1928.0      NaN   \n",
       "2        8.0    0.0763         0.0       0.0462   0.0767  1928.0      NaN   \n",
       "3        5.0    0.1190         0.0       0.9290   0.8800  1928.0      NaN   \n",
       "4        1.0    0.1110         0.0       0.0926   0.7200  1928.0      NaN   \n",
       "...      ...       ...         ...          ...      ...     ...      ...   \n",
       "660715   8.0    0.1120        68.0       0.0374   0.9340  2020.0      NaN   \n",
       "660716   1.0    0.0891        75.0       0.1430   0.3060  2020.0      NaN   \n",
       "660717   8.0    0.1110        64.0       0.0403   0.2700  2020.0      NaN   \n",
       "660718   7.0    0.1080        70.0       0.1050   0.1530  2020.0      NaN   \n",
       "660719   1.0    0.2220        70.0       0.0385   0.4720  2020.0      NaN   \n",
       "\n",
       "        play_count  \n",
       "0              NaN  \n",
       "1              NaN  \n",
       "2              NaN  \n",
       "3              NaN  \n",
       "4              NaN  \n",
       "...            ...  \n",
       "660715         NaN  \n",
       "660716         NaN  \n",
       "660717         NaN  \n",
       "660718         NaN  \n",
       "660719         NaN  \n",
       "\n",
       "[660720 rows x 17 columns]"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final.dropna(subset = [\"acousticness\"], inplace=True)\n",
    "final.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(660720, 16)"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>acousticness</th>\n",
       "      <th>name</th>\n",
       "      <th>danceability</th>\n",
       "      <th>tempo</th>\n",
       "      <th>loudness</th>\n",
       "      <th>energy</th>\n",
       "      <th>id</th>\n",
       "      <th>instrumentalness</th>\n",
       "      <th>key</th>\n",
       "      <th>liveness</th>\n",
       "      <th>popularity</th>\n",
       "      <th>speechiness</th>\n",
       "      <th>valence</th>\n",
       "      <th>year</th>\n",
       "      <th>user_id</th>\n",
       "      <th>play_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.995</td>\n",
       "      <td>Singende Bataillone 1. Teil Carl Woitschach</td>\n",
       "      <td>0.708</td>\n",
       "      <td>0.002377</td>\n",
       "      <td>-0.002373</td>\n",
       "      <td>0.1950</td>\n",
       "      <td>6KbQ3uYMLKb5jDxLF7wYDD</td>\n",
       "      <td>0.563</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.1510</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0506</td>\n",
       "      <td>0.7790</td>\n",
       "      <td>1928.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.994</td>\n",
       "      <td>Fantasiestücke, Op. 111: Più tosto lento Rober...</td>\n",
       "      <td>0.379</td>\n",
       "      <td>0.001685</td>\n",
       "      <td>-0.005434</td>\n",
       "      <td>0.0135</td>\n",
       "      <td>6KuQTIu1KoTTkLXKrwlLPV</td>\n",
       "      <td>0.901</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0763</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0462</td>\n",
       "      <td>0.0767</td>\n",
       "      <td>1928.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.994</td>\n",
       "      <td>Fantasiestücke, Op. 111: Più tosto lento Rober...</td>\n",
       "      <td>0.379</td>\n",
       "      <td>0.001685</td>\n",
       "      <td>-0.005434</td>\n",
       "      <td>0.0135</td>\n",
       "      <td>2YPbmbRIo7I1r9FAMqNjbZ</td>\n",
       "      <td>0.901</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0763</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0462</td>\n",
       "      <td>0.0767</td>\n",
       "      <td>1928.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.604</td>\n",
       "      <td>Chapter 1.18 - Zamek kaniowski Seweryn Goszczy...</td>\n",
       "      <td>0.749</td>\n",
       "      <td>0.002150</td>\n",
       "      <td>-0.003805</td>\n",
       "      <td>0.2200</td>\n",
       "      <td>6L63VW0PibdM1HDSBoqnoM</td>\n",
       "      <td>0.000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.1190</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.9290</td>\n",
       "      <td>0.8800</td>\n",
       "      <td>1928.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.995</td>\n",
       "      <td>Bebamos Juntos - Instrumental (Remasterizado) ...</td>\n",
       "      <td>0.781</td>\n",
       "      <td>0.002167</td>\n",
       "      <td>-0.002814</td>\n",
       "      <td>0.1300</td>\n",
       "      <td>6M94FkXd15sOAOQYRnWPN8</td>\n",
       "      <td>0.887</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.1110</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0926</td>\n",
       "      <td>0.7200</td>\n",
       "      <td>1928.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   acousticness                                               name  \\\n",
       "0         0.995        Singende Bataillone 1. Teil Carl Woitschach   \n",
       "1         0.994  Fantasiestücke, Op. 111: Più tosto lento Rober...   \n",
       "2         0.994  Fantasiestücke, Op. 111: Più tosto lento Rober...   \n",
       "3         0.604  Chapter 1.18 - Zamek kaniowski Seweryn Goszczy...   \n",
       "4         0.995  Bebamos Juntos - Instrumental (Remasterizado) ...   \n",
       "\n",
       "   danceability     tempo  loudness  energy                      id  \\\n",
       "0         0.708  0.002377 -0.002373  0.1950  6KbQ3uYMLKb5jDxLF7wYDD   \n",
       "1         0.379  0.001685 -0.005434  0.0135  6KuQTIu1KoTTkLXKrwlLPV   \n",
       "2         0.379  0.001685 -0.005434  0.0135  2YPbmbRIo7I1r9FAMqNjbZ   \n",
       "3         0.749  0.002150 -0.003805  0.2200  6L63VW0PibdM1HDSBoqnoM   \n",
       "4         0.781  0.002167 -0.002814  0.1300  6M94FkXd15sOAOQYRnWPN8   \n",
       "\n",
       "   instrumentalness   key  liveness  popularity  speechiness  valence    year  \\\n",
       "0             0.563  10.0    0.1510         0.0       0.0506   0.7790  1928.0   \n",
       "1             0.901   8.0    0.0763         0.0       0.0462   0.0767  1928.0   \n",
       "2             0.901   8.0    0.0763         0.0       0.0462   0.0767  1928.0   \n",
       "3             0.000   5.0    0.1190         0.0       0.9290   0.8800  1928.0   \n",
       "4             0.887   1.0    0.1110         0.0       0.0926   0.7200  1928.0   \n",
       "\n",
       "   user_id  play_count  \n",
       "0      NaN         NaN  \n",
       "1      NaN         NaN  \n",
       "2      NaN         NaN  \n",
       "3      NaN         NaN  \n",
       "4      NaN         NaN  "
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "#final.to_csv('finale.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "acousticness          4714\n",
       "name                156593\n",
       "danceability          1232\n",
       "tempo                84548\n",
       "loudness             25313\n",
       "energy                2332\n",
       "id                  169909\n",
       "instrumentalness      5401\n",
       "key                     12\n",
       "liveness              1741\n",
       "popularity             100\n",
       "speechiness           1628\n",
       "valence               1739\n",
       "year                   100\n",
       "user_id              93229\n",
       "play_count             189\n",
       "dtype: int64"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final.nunique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generating random user id's and playcounts for entries that do not have user_info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = np.array(final['play_count'].isnull())\n",
    "final['nan']= x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "acousticness             0\n",
       "name                     0\n",
       "danceability             0\n",
       "tempo                    0\n",
       "loudness                 0\n",
       "energy                   0\n",
       "id                       0\n",
       "instrumentalness         0\n",
       "key                      0\n",
       "liveness                 0\n",
       "popularity               0\n",
       "speechiness              0\n",
       "valence                  0\n",
       "year                     0\n",
       "user_id             158221\n",
       "play_count          158221\n",
       "nan                      0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "#triplicating each entry for which the userid and play count is not available\n",
    "reps = [3 if val== True else 0 for val in final.nan]\n",
    "final=final.append(final.loc[np.repeat(final.index.values, reps)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "final = final.drop(['nan'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1135383, 16)"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "#generating a random playcount between 1 and 10 for each of those entries\n",
    "df = pd.DataFrame(final['play_count'])\n",
    "final.reset_index()\n",
    "df1 = df.applymap(lambda l: l if not np.isnan(l) else np.random.choice(range(1,10)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>play_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   play_count\n",
       "0         5.0\n",
       "1         7.0\n",
       "2         3.0\n",
       "3         7.0\n",
       "4         8.0\n",
       "5         6.0\n",
       "6         1.0\n",
       "7         8.0\n",
       "8         6.0\n",
       "9         8.0"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1135383, 1)"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "final.insert(15, 'playcount', df1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>acousticness</th>\n",
       "      <th>name</th>\n",
       "      <th>danceability</th>\n",
       "      <th>tempo</th>\n",
       "      <th>loudness</th>\n",
       "      <th>energy</th>\n",
       "      <th>id</th>\n",
       "      <th>instrumentalness</th>\n",
       "      <th>key</th>\n",
       "      <th>liveness</th>\n",
       "      <th>popularity</th>\n",
       "      <th>speechiness</th>\n",
       "      <th>valence</th>\n",
       "      <th>year</th>\n",
       "      <th>user_id</th>\n",
       "      <th>playcount</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.995</td>\n",
       "      <td>Singende Bataillone 1. Teil Carl Woitschach</td>\n",
       "      <td>0.708</td>\n",
       "      <td>0.002377</td>\n",
       "      <td>-0.002373</td>\n",
       "      <td>0.195</td>\n",
       "      <td>6KbQ3uYMLKb5jDxLF7wYDD</td>\n",
       "      <td>0.563</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.151</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0506</td>\n",
       "      <td>0.779</td>\n",
       "      <td>1928.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   acousticness                                         name  danceability  \\\n",
       "0         0.995  Singende Bataillone 1. Teil Carl Woitschach         0.708   \n",
       "\n",
       "      tempo  loudness  energy                      id  instrumentalness   key  \\\n",
       "0  0.002377 -0.002373   0.195  6KbQ3uYMLKb5jDxLF7wYDD             0.563  10.0   \n",
       "\n",
       "   liveness  popularity  speechiness  valence    year  user_id  playcount  \n",
       "0     0.151         0.0       0.0506    0.779  1928.0      NaN        5.0  "
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final = final.drop(['play_count'],axis=1)\n",
    "final.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfwithnan = pd.DataFrame(final['user_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(final['user_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "#creating a list of unique user_ids \n",
    "df.dropna(subset = [\"user_id\"], inplace=True)\n",
    "df.drop_duplicates(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "x=np.array(df['user_id'])\n",
    "y= np.array(dfwithnan['user_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "93229   1135383\n"
     ]
    }
   ],
   "source": [
    "print(len(x),\" \",len(y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "#replacing nan values of userids with a legitimate userid, random function was taking too long and giving memory error\n",
    "for i in range(0,len(y)):\n",
    "    if np.isnan(y[i]):\n",
    "        n=i/len(x)+1\n",
    "        n = int(n)\n",
    "        y[i] = x[i-n*len(x)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1135383, 1)"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1135383"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([109679.,  13441.,  25637., ...,  74627.,  12459.,  71193.])"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "final.insert(15, 'userid', y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>acousticness</th>\n",
       "      <th>name</th>\n",
       "      <th>danceability</th>\n",
       "      <th>tempo</th>\n",
       "      <th>loudness</th>\n",
       "      <th>energy</th>\n",
       "      <th>id</th>\n",
       "      <th>instrumentalness</th>\n",
       "      <th>key</th>\n",
       "      <th>liveness</th>\n",
       "      <th>popularity</th>\n",
       "      <th>speechiness</th>\n",
       "      <th>valence</th>\n",
       "      <th>year</th>\n",
       "      <th>user_id</th>\n",
       "      <th>userid</th>\n",
       "      <th>playcount</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.995</td>\n",
       "      <td>Singende Bataillone 1. Teil Carl Woitschach</td>\n",
       "      <td>0.708</td>\n",
       "      <td>0.002377</td>\n",
       "      <td>-0.002373</td>\n",
       "      <td>0.195</td>\n",
       "      <td>6KbQ3uYMLKb5jDxLF7wYDD</td>\n",
       "      <td>0.563</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.151</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0506</td>\n",
       "      <td>0.779</td>\n",
       "      <td>1928.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>109679.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   acousticness                                         name  danceability  \\\n",
       "0         0.995  Singende Bataillone 1. Teil Carl Woitschach         0.708   \n",
       "\n",
       "      tempo  loudness  energy                      id  instrumentalness   key  \\\n",
       "0  0.002377 -0.002373   0.195  6KbQ3uYMLKb5jDxLF7wYDD             0.563  10.0   \n",
       "\n",
       "   liveness  popularity  speechiness  valence    year  user_id    userid  \\\n",
       "0     0.151         0.0       0.0506    0.779  1928.0      NaN  109679.0   \n",
       "\n",
       "   playcount  \n",
       "0        5.0  "
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "final = final.drop(['user_id'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>acousticness</th>\n",
       "      <th>name</th>\n",
       "      <th>danceability</th>\n",
       "      <th>tempo</th>\n",
       "      <th>loudness</th>\n",
       "      <th>energy</th>\n",
       "      <th>id</th>\n",
       "      <th>instrumentalness</th>\n",
       "      <th>key</th>\n",
       "      <th>liveness</th>\n",
       "      <th>popularity</th>\n",
       "      <th>speechiness</th>\n",
       "      <th>valence</th>\n",
       "      <th>year</th>\n",
       "      <th>userid</th>\n",
       "      <th>playcount</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.995</td>\n",
       "      <td>Singende Bataillone 1. Teil Carl Woitschach</td>\n",
       "      <td>0.708</td>\n",
       "      <td>0.002377</td>\n",
       "      <td>-0.002373</td>\n",
       "      <td>0.1950</td>\n",
       "      <td>6KbQ3uYMLKb5jDxLF7wYDD</td>\n",
       "      <td>0.563</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.1510</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0506</td>\n",
       "      <td>0.7790</td>\n",
       "      <td>1928.0</td>\n",
       "      <td>109679.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.994</td>\n",
       "      <td>Fantasiestücke, Op. 111: Più tosto lento Rober...</td>\n",
       "      <td>0.379</td>\n",
       "      <td>0.001685</td>\n",
       "      <td>-0.005434</td>\n",
       "      <td>0.0135</td>\n",
       "      <td>6KuQTIu1KoTTkLXKrwlLPV</td>\n",
       "      <td>0.901</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0763</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0462</td>\n",
       "      <td>0.0767</td>\n",
       "      <td>1928.0</td>\n",
       "      <td>13441.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.994</td>\n",
       "      <td>Fantasiestücke, Op. 111: Più tosto lento Rober...</td>\n",
       "      <td>0.379</td>\n",
       "      <td>0.001685</td>\n",
       "      <td>-0.005434</td>\n",
       "      <td>0.0135</td>\n",
       "      <td>2YPbmbRIo7I1r9FAMqNjbZ</td>\n",
       "      <td>0.901</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0763</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0462</td>\n",
       "      <td>0.0767</td>\n",
       "      <td>1928.0</td>\n",
       "      <td>25637.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.604</td>\n",
       "      <td>Chapter 1.18 - Zamek kaniowski Seweryn Goszczy...</td>\n",
       "      <td>0.749</td>\n",
       "      <td>0.002150</td>\n",
       "      <td>-0.003805</td>\n",
       "      <td>0.2200</td>\n",
       "      <td>6L63VW0PibdM1HDSBoqnoM</td>\n",
       "      <td>0.000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.1190</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.9290</td>\n",
       "      <td>0.8800</td>\n",
       "      <td>1928.0</td>\n",
       "      <td>63548.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.995</td>\n",
       "      <td>Bebamos Juntos - Instrumental (Remasterizado) ...</td>\n",
       "      <td>0.781</td>\n",
       "      <td>0.002167</td>\n",
       "      <td>-0.002814</td>\n",
       "      <td>0.1300</td>\n",
       "      <td>6M94FkXd15sOAOQYRnWPN8</td>\n",
       "      <td>0.887</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.1110</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0926</td>\n",
       "      <td>0.7200</td>\n",
       "      <td>1928.0</td>\n",
       "      <td>10501.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   acousticness                                               name  \\\n",
       "0         0.995        Singende Bataillone 1. Teil Carl Woitschach   \n",
       "1         0.994  Fantasiestücke, Op. 111: Più tosto lento Rober...   \n",
       "2         0.994  Fantasiestücke, Op. 111: Più tosto lento Rober...   \n",
       "3         0.604  Chapter 1.18 - Zamek kaniowski Seweryn Goszczy...   \n",
       "4         0.995  Bebamos Juntos - Instrumental (Remasterizado) ...   \n",
       "\n",
       "   danceability     tempo  loudness  energy                      id  \\\n",
       "0         0.708  0.002377 -0.002373  0.1950  6KbQ3uYMLKb5jDxLF7wYDD   \n",
       "1         0.379  0.001685 -0.005434  0.0135  6KuQTIu1KoTTkLXKrwlLPV   \n",
       "2         0.379  0.001685 -0.005434  0.0135  2YPbmbRIo7I1r9FAMqNjbZ   \n",
       "3         0.749  0.002150 -0.003805  0.2200  6L63VW0PibdM1HDSBoqnoM   \n",
       "4         0.781  0.002167 -0.002814  0.1300  6M94FkXd15sOAOQYRnWPN8   \n",
       "\n",
       "   instrumentalness   key  liveness  popularity  speechiness  valence    year  \\\n",
       "0             0.563  10.0    0.1510         0.0       0.0506   0.7790  1928.0   \n",
       "1             0.901   8.0    0.0763         0.0       0.0462   0.0767  1928.0   \n",
       "2             0.901   8.0    0.0763         0.0       0.0462   0.0767  1928.0   \n",
       "3             0.000   5.0    0.1190         0.0       0.9290   0.8800  1928.0   \n",
       "4             0.887   1.0    0.1110         0.0       0.0926   0.7200  1928.0   \n",
       "\n",
       "     userid  playcount  \n",
       "0  109679.0        5.0  \n",
       "1   13441.0        7.0  \n",
       "2   25637.0        3.0  \n",
       "3   63548.0        7.0  \n",
       "4   10501.0        8.0  "
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1135383, 16)"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>acousticness</th>\n",
       "      <th>name</th>\n",
       "      <th>danceability</th>\n",
       "      <th>tempo</th>\n",
       "      <th>loudness</th>\n",
       "      <th>energy</th>\n",
       "      <th>id</th>\n",
       "      <th>instrumentalness</th>\n",
       "      <th>key</th>\n",
       "      <th>liveness</th>\n",
       "      <th>popularity</th>\n",
       "      <th>speechiness</th>\n",
       "      <th>valence</th>\n",
       "      <th>year</th>\n",
       "      <th>userid</th>\n",
       "      <th>playcount</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.9950</td>\n",
       "      <td>Singende Bataillone 1. Teil Carl Woitschach</td>\n",
       "      <td>0.708</td>\n",
       "      <td>0.002377</td>\n",
       "      <td>-0.002373</td>\n",
       "      <td>0.1950</td>\n",
       "      <td>6KbQ3uYMLKb5jDxLF7wYDD</td>\n",
       "      <td>0.56300</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.1510</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0506</td>\n",
       "      <td>0.7790</td>\n",
       "      <td>1928.0</td>\n",
       "      <td>109679.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.9940</td>\n",
       "      <td>Fantasiestücke, Op. 111: Più tosto lento Rober...</td>\n",
       "      <td>0.379</td>\n",
       "      <td>0.001685</td>\n",
       "      <td>-0.005434</td>\n",
       "      <td>0.0135</td>\n",
       "      <td>6KuQTIu1KoTTkLXKrwlLPV</td>\n",
       "      <td>0.90100</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0763</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0462</td>\n",
       "      <td>0.0767</td>\n",
       "      <td>1928.0</td>\n",
       "      <td>13441.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.9940</td>\n",
       "      <td>Fantasiestücke, Op. 111: Più tosto lento Rober...</td>\n",
       "      <td>0.379</td>\n",
       "      <td>0.001685</td>\n",
       "      <td>-0.005434</td>\n",
       "      <td>0.0135</td>\n",
       "      <td>2YPbmbRIo7I1r9FAMqNjbZ</td>\n",
       "      <td>0.90100</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0763</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0462</td>\n",
       "      <td>0.0767</td>\n",
       "      <td>1928.0</td>\n",
       "      <td>25637.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.6040</td>\n",
       "      <td>Chapter 1.18 - Zamek kaniowski Seweryn Goszczy...</td>\n",
       "      <td>0.749</td>\n",
       "      <td>0.002150</td>\n",
       "      <td>-0.003805</td>\n",
       "      <td>0.2200</td>\n",
       "      <td>6L63VW0PibdM1HDSBoqnoM</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.1190</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.9290</td>\n",
       "      <td>0.8800</td>\n",
       "      <td>1928.0</td>\n",
       "      <td>63548.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.9950</td>\n",
       "      <td>Bebamos Juntos - Instrumental (Remasterizado) ...</td>\n",
       "      <td>0.781</td>\n",
       "      <td>0.002167</td>\n",
       "      <td>-0.002814</td>\n",
       "      <td>0.1300</td>\n",
       "      <td>6M94FkXd15sOAOQYRnWPN8</td>\n",
       "      <td>0.88700</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.1110</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0926</td>\n",
       "      <td>0.7200</td>\n",
       "      <td>1928.0</td>\n",
       "      <td>10501.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>660718</td>\n",
       "      <td>0.5380</td>\n",
       "      <td>How Would I Know Kygo &amp; Oh Wonder</td>\n",
       "      <td>0.514</td>\n",
       "      <td>0.002482</td>\n",
       "      <td>-0.001782</td>\n",
       "      <td>0.5390</td>\n",
       "      <td>52eycxprLhK3lPcRLbQiVk</td>\n",
       "      <td>0.00233</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.1080</td>\n",
       "      <td>70.0</td>\n",
       "      <td>0.1050</td>\n",
       "      <td>0.1530</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>33482.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>660718</td>\n",
       "      <td>0.5380</td>\n",
       "      <td>How Would I Know Kygo &amp; Oh Wonder</td>\n",
       "      <td>0.514</td>\n",
       "      <td>0.002482</td>\n",
       "      <td>-0.001782</td>\n",
       "      <td>0.5390</td>\n",
       "      <td>52eycxprLhK3lPcRLbQiVk</td>\n",
       "      <td>0.00233</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.1080</td>\n",
       "      <td>70.0</td>\n",
       "      <td>0.1050</td>\n",
       "      <td>0.1530</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>37704.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>660719</td>\n",
       "      <td>0.0714</td>\n",
       "      <td>I Found You Cash Cash &amp; Andy Grammer</td>\n",
       "      <td>0.646</td>\n",
       "      <td>0.002607</td>\n",
       "      <td>-0.000488</td>\n",
       "      <td>0.7610</td>\n",
       "      <td>3wYOGJYD31sLRmBgCvWxa4</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.2220</td>\n",
       "      <td>70.0</td>\n",
       "      <td>0.0385</td>\n",
       "      <td>0.4720</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>74627.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>660719</td>\n",
       "      <td>0.0714</td>\n",
       "      <td>I Found You Cash Cash &amp; Andy Grammer</td>\n",
       "      <td>0.646</td>\n",
       "      <td>0.002607</td>\n",
       "      <td>-0.000488</td>\n",
       "      <td>0.7610</td>\n",
       "      <td>3wYOGJYD31sLRmBgCvWxa4</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.2220</td>\n",
       "      <td>70.0</td>\n",
       "      <td>0.0385</td>\n",
       "      <td>0.4720</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>12459.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>660719</td>\n",
       "      <td>0.0714</td>\n",
       "      <td>I Found You Cash Cash &amp; Andy Grammer</td>\n",
       "      <td>0.646</td>\n",
       "      <td>0.002607</td>\n",
       "      <td>-0.000488</td>\n",
       "      <td>0.7610</td>\n",
       "      <td>3wYOGJYD31sLRmBgCvWxa4</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.2220</td>\n",
       "      <td>70.0</td>\n",
       "      <td>0.0385</td>\n",
       "      <td>0.4720</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>71193.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1135383 rows × 16 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        acousticness                                               name  \\\n",
       "0             0.9950        Singende Bataillone 1. Teil Carl Woitschach   \n",
       "1             0.9940  Fantasiestücke, Op. 111: Più tosto lento Rober...   \n",
       "2             0.9940  Fantasiestücke, Op. 111: Più tosto lento Rober...   \n",
       "3             0.6040  Chapter 1.18 - Zamek kaniowski Seweryn Goszczy...   \n",
       "4             0.9950  Bebamos Juntos - Instrumental (Remasterizado) ...   \n",
       "...              ...                                                ...   \n",
       "660718        0.5380                  How Would I Know Kygo & Oh Wonder   \n",
       "660718        0.5380                  How Would I Know Kygo & Oh Wonder   \n",
       "660719        0.0714               I Found You Cash Cash & Andy Grammer   \n",
       "660719        0.0714               I Found You Cash Cash & Andy Grammer   \n",
       "660719        0.0714               I Found You Cash Cash & Andy Grammer   \n",
       "\n",
       "        danceability     tempo  loudness  energy                      id  \\\n",
       "0              0.708  0.002377 -0.002373  0.1950  6KbQ3uYMLKb5jDxLF7wYDD   \n",
       "1              0.379  0.001685 -0.005434  0.0135  6KuQTIu1KoTTkLXKrwlLPV   \n",
       "2              0.379  0.001685 -0.005434  0.0135  2YPbmbRIo7I1r9FAMqNjbZ   \n",
       "3              0.749  0.002150 -0.003805  0.2200  6L63VW0PibdM1HDSBoqnoM   \n",
       "4              0.781  0.002167 -0.002814  0.1300  6M94FkXd15sOAOQYRnWPN8   \n",
       "...              ...       ...       ...     ...                     ...   \n",
       "660718         0.514  0.002482 -0.001782  0.5390  52eycxprLhK3lPcRLbQiVk   \n",
       "660718         0.514  0.002482 -0.001782  0.5390  52eycxprLhK3lPcRLbQiVk   \n",
       "660719         0.646  0.002607 -0.000488  0.7610  3wYOGJYD31sLRmBgCvWxa4   \n",
       "660719         0.646  0.002607 -0.000488  0.7610  3wYOGJYD31sLRmBgCvWxa4   \n",
       "660719         0.646  0.002607 -0.000488  0.7610  3wYOGJYD31sLRmBgCvWxa4   \n",
       "\n",
       "        instrumentalness   key  liveness  popularity  speechiness  valence  \\\n",
       "0                0.56300  10.0    0.1510         0.0       0.0506   0.7790   \n",
       "1                0.90100   8.0    0.0763         0.0       0.0462   0.0767   \n",
       "2                0.90100   8.0    0.0763         0.0       0.0462   0.0767   \n",
       "3                0.00000   5.0    0.1190         0.0       0.9290   0.8800   \n",
       "4                0.88700   1.0    0.1110         0.0       0.0926   0.7200   \n",
       "...                  ...   ...       ...         ...          ...      ...   \n",
       "660718           0.00233   7.0    0.1080        70.0       0.1050   0.1530   \n",
       "660718           0.00233   7.0    0.1080        70.0       0.1050   0.1530   \n",
       "660719           0.00000   1.0    0.2220        70.0       0.0385   0.4720   \n",
       "660719           0.00000   1.0    0.2220        70.0       0.0385   0.4720   \n",
       "660719           0.00000   1.0    0.2220        70.0       0.0385   0.4720   \n",
       "\n",
       "          year    userid  playcount  \n",
       "0       1928.0  109679.0        5.0  \n",
       "1       1928.0   13441.0        7.0  \n",
       "2       1928.0   25637.0        3.0  \n",
       "3       1928.0   63548.0        7.0  \n",
       "4       1928.0   10501.0        8.0  \n",
       "...        ...       ...        ...  \n",
       "660718  2020.0   33482.0        7.0  \n",
       "660718  2020.0   37704.0        2.0  \n",
       "660719  2020.0   74627.0        6.0  \n",
       "660719  2020.0   12459.0        1.0  \n",
       "660719  2020.0   71193.0        8.0  \n",
       "\n",
       "[1135383 rows x 16 columns]"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "final.to_csv('finale.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}