Read Dumps.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6268 files processed.\n"
     ]
    }
   ],
   "source": [
    "\"\"\" Read data dumps from disk. \"\"\"\n",
    "\n",
    "import json\n",
    "import os\n",
    "\n",
    "import track\n",
    "\n",
    "# Folder to read dumps from\n",
    "folder = 'dump'\n",
    "\n",
    "# List of dictionaries\n",
    "dicts = []\n",
    "\n",
    "for file in os.listdir(folder):\n",
    "    with open('%s/%s' % (folder, file)) as inp:\n",
    "        try:\n",
    "            data = json.load(inp)\n",
    "        except ValueError:\n",
    "            print(file)\n",
    "        else:\n",
    "            try:\n",
    "                dicts.append(track.features(**data))\n",
    "            except KeyError:\n",
    "                print(file)\n",
    "\n",
    "print(\"%d files processed.\" % len(dicts))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\"\"\" Build a dataframe. \"\"\"\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "# Convert list of dicts to a dataframe\n",
    "df = pd.DataFrame(dicts)\n",
    "\n",
    "# Reorder the columns\n",
    "df = df[track.feature_list]\n",
    "\n",
    "# Convert these columns to integers\n",
    "to_int = ['Listeners', 'Playcount', 'Duration']\n",
    "df[to_int] = df[to_int].astype(int)\n",
    "\n",
    "# Sort by Listeners\n",
    "df.sort([\"Listeners\"], ascending=False, inplace=True)\n",
    "\n",
    "# Split tracks into 3 categories: \"Hit\", \"Avg\", \"Flop\"\n",
    "# Top 33% of tracks are Hits, next 33% are Avg etc.\n",
    "class_labels = np.array([\"Hit\", \"Flop\"])\n",
    "df.insert(0, 'Class', pd.qcut(range(len(df)), 2, labels=class_labels))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\"\"\" Is a science demo. \"\"\"\n",
    "\n",
    "# Select only some of the features!\n",
    "feature_list = [\"Class\", \"Name\", \"Artist\", \"MBID\", \"Listeners\", \"Playcount\", \"Duration\", \"ABL - Average Loudness\", \"ABL - Tempo (BPM)\"]\n",
    "dfn = df[feature_list]\n",
    "\n",
    "# Remove Zero duration songs!\n",
    "dfn = dfn[dfn[\"Duration\"] != 0]\n",
    "\n",
    "# Convert Tempo to categories!\n",
    "dfn.insert(len(dfn.columns), \"Tempo 70-89\", ((dfn[\"ABL - Tempo (BPM)\"] > 70) & (dfn[\"ABL - Tempo (BPM)\"] < 89)).apply(int))\n",
    "dfn.insert(len(dfn.columns), \"Tempo 90-109\", ((dfn[\"ABL - Tempo (BPM)\"] > 90) & (dfn[\"ABL - Tempo (BPM)\"] < 109)).apply(int))\n",
    "dfn.insert(len(dfn.columns), \"Tempo 110-129\", ((dfn[\"ABL - Tempo (BPM)\"] > 110) & (dfn[\"ABL - Tempo (BPM)\"] < 129)).apply(int))\n",
    "dfn.insert(len(dfn.columns), \"Tempo 130-149\", ((dfn[\"ABL - Tempo (BPM)\"] > 130) & (dfn[\"ABL - Tempo (BPM)\"] < 149)).apply(int))\n",
    "dfn.insert(len(dfn.columns), \"Tempo 150-169\", ((dfn[\"ABL - Tempo (BPM)\"] > 150) & (dfn[\"ABL - Tempo (BPM)\"] < 169)).apply(int))\n",
    "dfn.insert(len(dfn.columns), \"Tempo 170-189\", ((dfn[\"ABL - Tempo (BPM)\"] > 170) & (dfn[\"ABL - Tempo (BPM)\"] < 189)).apply(int))\n",
    "\n",
    "# Drop Tempo Column now\n",
    "dfn.drop(\"ABL - Tempo (BPM)\", axis=1, inplace=True)\n",
    "\n",
    "# Convert Duration to Categories!\n",
    "dfn.insert(len(dfn.columns), \"Duration 75-224\", ((dfn[\"Duration\"] > 75000) & (dfn[\"Duration\"] < 224000)).apply(int))\n",
    "dfn.insert(len(dfn.columns), \"Duration 225-374\", ((dfn[\"Duration\"] > 225000) & (dfn[\"Duration\"] < 374000)).apply(int))\n",
    "dfn.insert(len(dfn.columns), \"Duration 375-524\", ((dfn[\"Duration\"] > 375000) & (dfn[\"Duration\"] < 524000)).apply(int))\n",
    "dfn.insert(len(dfn.columns), \"Duration >524\", (dfn[\"Duration\"] > 524000).apply(int))\n",
    "\n",
    "# Classifier below uses a general variable 'fr'\n",
    "# Here we can assign, fr to df or dfn.\n",
    "fr = dfn\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\"\"\" Data. \"\"\"\n",
    "\n",
    "from sklearn.cross_validation import train_test_split\n",
    "\n",
    "# Skip the first 7 columns as they are mostly metadata\n",
    "features = fr.columns[7:]\n",
    "\n",
    "# Data\n",
    "X = fr[features]\n",
    "y, _ = pd.factorize(fr['Class'])\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\"\"\" SVM Classifier. \"\"\"\n",
    "\n",
    "from sklearn import svm\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "\n",
    "clf = svm.SVC(kernel='rbf')\n",
    "\n",
    "clf.fit(X_train, y_train)\n",
    "\n",
    "# clf.score(X_test, y_test)\n",
    "cross_val_score(clf, X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn import svm, datasets\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\" Random Forest Classifier. \"\"\"\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "clf = RandomForestClassifier(max_features=0.33, n_jobs=2, n_estimators=84)\n",
    "\n",
    "clf.fit(X_train, y_train)\n",
    "\n",
    "pd.crosstab(class_labels[clf.predict(X_test)], test['class'], rownames=['actual'], colnames=['pred'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\"\"\" Cross Validation. \"\"\"\n",
    "\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "\n",
    "cross_val_score(clf, X, Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\"\"\" Automatic Feature Selection. \"\"\"\n",
    "\n",
    "X = train[features]\n",
    "y, _ = pd.factorize(train['class'])\n",
    "print(X.shape)\n",
    "\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "X_new = ExtraTreesClassifier().fit_transform(X, y)\n",
    "print(X_new.shape)\n",
    "\n",
    "from sklearn.svm import LinearSVC\n",
    "X_new = LinearSVC(C=0.01, penalty=\"l1\", dual=False).fit_transform(X, y)\n",
    "print(X_new.shape)\n",
    "\n",
    "from sklearn.feature_selection import VarianceThreshold\n",
    "X_new = VarianceThreshold(threshold=(.8 * (1 - .8))).fit_transform(X, y)\n",
    "print(X_new.shape)\n",
    "\n",
    "# TODO: Find names of columns that have important features"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}