generate_d3_data2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Bad key \"lines.markeredgecolor\" on line 81 in\n",
      "C:\\Users\\harinsa\\arxiv-data-exploration\\matplotlibrc.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "df= pd.read_pickle('dblp.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('top10_cited_venues.txt') as f:\n",
    "    top_venues = json.load(f)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_top = df[df.venue.isin(top_venues)].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>authors</th>\n",
       "      <th>id</th>\n",
       "      <th>n_citation</th>\n",
       "      <th>references</th>\n",
       "      <th>title</th>\n",
       "      <th>venue</th>\n",
       "      <th>year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[N. Sai Shankar, Sunghyun Choi]</td>\n",
       "      <td>11f0bd37-ae5a-43e6-b14a-a59bc00fdd90</td>\n",
       "      <td>50</td>\n",
       "      <td>NaN</td>\n",
       "      <td>QoS Signaling for Parameterized Traffic in IEE...</td>\n",
       "      <td>Lecture Notes in Computer Science</td>\n",
       "      <td>2002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[C. Graciani Díaz, Francisco-Jesús Martín-Mate...</td>\n",
       "      <td>155dec16-36d6-44f4-976b-1afb5d1924af</td>\n",
       "      <td>50</td>\n",
       "      <td>[c17481ca-9511-4793-8dad-a2486e0b2713]</td>\n",
       "      <td>Specification of Adleman's Restricted Model Us...</td>\n",
       "      <td>Lecture Notes in Computer Science</td>\n",
       "      <td>2002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[Van-Nam Huynh, Yoshiteru Nakamori, Tu Bao Ho]</td>\n",
       "      <td>20f0a5af-395a-487f-854d-9888295f2508</td>\n",
       "      <td>4</td>\n",
       "      <td>[5fe2f07f-4093-4c1a-94e6-39666362ec48, b2e6bac...</td>\n",
       "      <td>Utility function induced by fuzzy target in pr...</td>\n",
       "      <td>Lecture Notes in Computer Science</td>\n",
       "      <td>2006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[Yisheng Guan, Teresa R. Ho, Hong Zhang]</td>\n",
       "      <td>210d708b-080c-46ce-850f-0588450de08b</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>An Integrated Robotic Hand/Simulator System fo...</td>\n",
       "      <td>international conference on robotics and autom...</td>\n",
       "      <td>2001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[Makoto Matsushita, Makoto Oshita, Hajimu Iida...</td>\n",
       "      <td>25e09f2e-c606-469f-a763-5d20b4704a86</td>\n",
       "      <td>7</td>\n",
       "      <td>[269663ea-4dc4-4e99-86da-b04240bfdf5c, 3ff297f...</td>\n",
       "      <td>Distributed Process Management System Based on...</td>\n",
       "      <td>Lecture Notes in Computer Science</td>\n",
       "      <td>1998</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             authors  \\\n",
       "0                    [N. Sai Shankar, Sunghyun Choi]   \n",
       "1  [C. Graciani Díaz, Francisco-Jesús Martín-Mate...   \n",
       "2     [Van-Nam Huynh, Yoshiteru Nakamori, Tu Bao Ho]   \n",
       "3           [Yisheng Guan, Teresa R. Ho, Hong Zhang]   \n",
       "4  [Makoto Matsushita, Makoto Oshita, Hajimu Iida...   \n",
       "\n",
       "                                     id  n_citation  \\\n",
       "0  11f0bd37-ae5a-43e6-b14a-a59bc00fdd90          50   \n",
       "1  155dec16-36d6-44f4-976b-1afb5d1924af          50   \n",
       "2  20f0a5af-395a-487f-854d-9888295f2508           4   \n",
       "3  210d708b-080c-46ce-850f-0588450de08b           0   \n",
       "4  25e09f2e-c606-469f-a763-5d20b4704a86           7   \n",
       "\n",
       "                                          references  \\\n",
       "0                                                NaN   \n",
       "1             [c17481ca-9511-4793-8dad-a2486e0b2713]   \n",
       "2  [5fe2f07f-4093-4c1a-94e6-39666362ec48, b2e6bac...   \n",
       "3                                                NaN   \n",
       "4  [269663ea-4dc4-4e99-86da-b04240bfdf5c, 3ff297f...   \n",
       "\n",
       "                                               title  \\\n",
       "0  QoS Signaling for Parameterized Traffic in IEE...   \n",
       "1  Specification of Adleman's Restricted Model Us...   \n",
       "2  Utility function induced by fuzzy target in pr...   \n",
       "3  An Integrated Robotic Hand/Simulator System fo...   \n",
       "4  Distributed Process Management System Based on...   \n",
       "\n",
       "                                               venue  year  \n",
       "0                  Lecture Notes in Computer Science  2002  \n",
       "1                  Lecture Notes in Computer Science  2002  \n",
       "2                  Lecture Notes in Computer Science  2006  \n",
       "3  international conference on robotics and autom...  2001  \n",
       "4                  Lecture Notes in Computer Science  1998  "
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_top.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_links(df):\n",
    "    df = df.copy()\n",
    "    df = df.reset_index().set_index('id')\n",
    "    link = []\n",
    "    missed = 0\n",
    "    for i, r in df.iterrows():\n",
    "        if type(r.references) is list:\n",
    "            for ref in r.references:\n",
    "                if ref not in df.index:\n",
    "                    missed += 1\n",
    "                    continue\n",
    "                link.append({\n",
    "                    'source': int(r['index']),\n",
    "                    'target': int(df.loc[ref]['index'])\n",
    "                })\n",
    "    return link"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "links = gen_links(df_top)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'source': 5, 'target': 94902},\n",
       " {'source': 6, 'target': 107007},\n",
       " {'source': 9, 'target': 10778},\n",
       " {'source': 14, 'target': 13091},\n",
       " {'source': 16, 'target': 26359},\n",
       " {'source': 24, 'target': 24830},\n",
       " {'source': 33, 'target': 9570},\n",
       " {'source': 33, 'target': 14531},\n",
       " {'source': 33, 'target': 103045},\n",
       " {'source': 33, 'target': 22952}]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "links[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "374939"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(links)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>authors</th>\n",
       "      <th>id</th>\n",
       "      <th>n_citation</th>\n",
       "      <th>references</th>\n",
       "      <th>title</th>\n",
       "      <th>venue</th>\n",
       "      <th>year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[N. Sai Shankar, Sunghyun Choi]</td>\n",
       "      <td>11f0bd37-ae5a-43e6-b14a-a59bc00fdd90</td>\n",
       "      <td>50</td>\n",
       "      <td>NaN</td>\n",
       "      <td>QoS Signaling for Parameterized Traffic in IEE...</td>\n",
       "      <td>Lecture Notes in Computer Science</td>\n",
       "      <td>2002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[C. Graciani Díaz, Francisco-Jesús Martín-Mate...</td>\n",
       "      <td>155dec16-36d6-44f4-976b-1afb5d1924af</td>\n",
       "      <td>50</td>\n",
       "      <td>[c17481ca-9511-4793-8dad-a2486e0b2713]</td>\n",
       "      <td>Specification of Adleman's Restricted Model Us...</td>\n",
       "      <td>Lecture Notes in Computer Science</td>\n",
       "      <td>2002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[Van-Nam Huynh, Yoshiteru Nakamori, Tu Bao Ho]</td>\n",
       "      <td>20f0a5af-395a-487f-854d-9888295f2508</td>\n",
       "      <td>4</td>\n",
       "      <td>[5fe2f07f-4093-4c1a-94e6-39666362ec48, b2e6bac...</td>\n",
       "      <td>Utility function induced by fuzzy target in pr...</td>\n",
       "      <td>Lecture Notes in Computer Science</td>\n",
       "      <td>2006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[Yisheng Guan, Teresa R. Ho, Hong Zhang]</td>\n",
       "      <td>210d708b-080c-46ce-850f-0588450de08b</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>An Integrated Robotic Hand/Simulator System fo...</td>\n",
       "      <td>international conference on robotics and autom...</td>\n",
       "      <td>2001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[Makoto Matsushita, Makoto Oshita, Hajimu Iida...</td>\n",
       "      <td>25e09f2e-c606-469f-a763-5d20b4704a86</td>\n",
       "      <td>7</td>\n",
       "      <td>[269663ea-4dc4-4e99-86da-b04240bfdf5c, 3ff297f...</td>\n",
       "      <td>Distributed Process Management System Based on...</td>\n",
       "      <td>Lecture Notes in Computer Science</td>\n",
       "      <td>1998</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             authors  \\\n",
       "0                    [N. Sai Shankar, Sunghyun Choi]   \n",
       "1  [C. Graciani Díaz, Francisco-Jesús Martín-Mate...   \n",
       "2     [Van-Nam Huynh, Yoshiteru Nakamori, Tu Bao Ho]   \n",
       "3           [Yisheng Guan, Teresa R. Ho, Hong Zhang]   \n",
       "4  [Makoto Matsushita, Makoto Oshita, Hajimu Iida...   \n",
       "\n",
       "                                     id  n_citation  \\\n",
       "0  11f0bd37-ae5a-43e6-b14a-a59bc00fdd90          50   \n",
       "1  155dec16-36d6-44f4-976b-1afb5d1924af          50   \n",
       "2  20f0a5af-395a-487f-854d-9888295f2508           4   \n",
       "3  210d708b-080c-46ce-850f-0588450de08b           0   \n",
       "4  25e09f2e-c606-469f-a763-5d20b4704a86           7   \n",
       "\n",
       "                                          references  \\\n",
       "0                                                NaN   \n",
       "1             [c17481ca-9511-4793-8dad-a2486e0b2713]   \n",
       "2  [5fe2f07f-4093-4c1a-94e6-39666362ec48, b2e6bac...   \n",
       "3                                                NaN   \n",
       "4  [269663ea-4dc4-4e99-86da-b04240bfdf5c, 3ff297f...   \n",
       "\n",
       "                                               title  \\\n",
       "0  QoS Signaling for Parameterized Traffic in IEE...   \n",
       "1  Specification of Adleman's Restricted Model Us...   \n",
       "2  Utility function induced by fuzzy target in pr...   \n",
       "3  An Integrated Robotic Hand/Simulator System fo...   \n",
       "4  Distributed Process Management System Based on...   \n",
       "\n",
       "                                               venue  year  \n",
       "0                  Lecture Notes in Computer Science  2002  \n",
       "1                  Lecture Notes in Computer Science  2002  \n",
       "2                  Lecture Notes in Computer Science  2006  \n",
       "3  international conference on robotics and autom...  2001  \n",
       "4                  Lecture Notes in Computer Science  1998  "
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_top.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df_top.sort_values('n_citation', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>authors</th>\n",
       "      <th>id</th>\n",
       "      <th>n_citation</th>\n",
       "      <th>references</th>\n",
       "      <th>title</th>\n",
       "      <th>venue</th>\n",
       "      <th>year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>[Stéphane Mallat]</td>\n",
       "      <td>7ccbdf09-a84e-4ad2-ab20-cb28b6c41155</td>\n",
       "      <td>24182</td>\n",
       "      <td>[093cf9fe-6fda-4495-9e89-acef614d0453, 15a2345...</td>\n",
       "      <td>A theory for multiresolution signal decomposit...</td>\n",
       "      <td>IEEE Transactions on Pattern Analysis and Mach...</td>\n",
       "      <td>1989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>[Navneet Dalal, Bill Triggs]</td>\n",
       "      <td>dd83785a-dd19-41e3-9b25-ebabbd48d336</td>\n",
       "      <td>22795</td>\n",
       "      <td>[04d8a9cb-a14d-4ccf-8b19-da1327e86b91, 3e81212...</td>\n",
       "      <td>Histograms of oriented gradients for human det...</td>\n",
       "      <td>computer vision and pattern recognition</td>\n",
       "      <td>2005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>[David L. Donoho]</td>\n",
       "      <td>f56b877b-4060-4754-b303-e8140968544c</td>\n",
       "      <td>20915</td>\n",
       "      <td>[036a19f8-fdca-4e84-a237-e54f2108dcb4, 05c85ac...</td>\n",
       "      <td>Compressed sensing</td>\n",
       "      <td>IEEE Transactions on Information Theory</td>\n",
       "      <td>2006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>[Mark A. Larkin, Gordon Blackshields, N. P. Br...</td>\n",
       "      <td>8a408df6-2ae6-45a4-afb5-ca95bc7dc664</td>\n",
       "      <td>20580</td>\n",
       "      <td>[239b9a0a-2a71-4780-960a-c565f19a1e9a, b2f2e8e...</td>\n",
       "      <td>Clustal W and Clustal X version 2.0</td>\n",
       "      <td>Bioinformatics</td>\n",
       "      <td>2007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>[David Posada, Keith A. Crandall]</td>\n",
       "      <td>e776f9db-fac3-40de-8968-19448d1693e1</td>\n",
       "      <td>19833</td>\n",
       "      <td>[]</td>\n",
       "      <td>MODELTEST: testing the model of DNA substitution.</td>\n",
       "      <td>Bioinformatics</td>\n",
       "      <td>1998</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index                                            authors  \\\n",
       "0      0                                  [Stéphane Mallat]   \n",
       "1      1                       [Navneet Dalal, Bill Triggs]   \n",
       "2      2                                  [David L. Donoho]   \n",
       "3      3  [Mark A. Larkin, Gordon Blackshields, N. P. Br...   \n",
       "4      4                  [David Posada, Keith A. Crandall]   \n",
       "\n",
       "                                     id  n_citation  \\\n",
       "0  7ccbdf09-a84e-4ad2-ab20-cb28b6c41155       24182   \n",
       "1  dd83785a-dd19-41e3-9b25-ebabbd48d336       22795   \n",
       "2  f56b877b-4060-4754-b303-e8140968544c       20915   \n",
       "3  8a408df6-2ae6-45a4-afb5-ca95bc7dc664       20580   \n",
       "4  e776f9db-fac3-40de-8968-19448d1693e1       19833   \n",
       "\n",
       "                                          references  \\\n",
       "0  [093cf9fe-6fda-4495-9e89-acef614d0453, 15a2345...   \n",
       "1  [04d8a9cb-a14d-4ccf-8b19-da1327e86b91, 3e81212...   \n",
       "2  [036a19f8-fdca-4e84-a237-e54f2108dcb4, 05c85ac...   \n",
       "3  [239b9a0a-2a71-4780-960a-c565f19a1e9a, b2f2e8e...   \n",
       "4                                                 []   \n",
       "\n",
       "                                               title  \\\n",
       "0  A theory for multiresolution signal decomposit...   \n",
       "1  Histograms of oriented gradients for human det...   \n",
       "2                                 Compressed sensing   \n",
       "3                Clustal W and Clustal X version 2.0   \n",
       "4  MODELTEST: testing the model of DNA substitution.   \n",
       "\n",
       "                                               venue  year  \n",
       "0  IEEE Transactions on Pattern Analysis and Mach...  1989  \n",
       "1            computer vision and pattern recognition  2005  \n",
       "2            IEEE Transactions on Information Theory  2006  \n",
       "3                                     Bioinformatics  2007  \n",
       "4                                     Bioinformatics  1998  "
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "N = 1000\n",
    "df_subset = df[:N].reset_index(drop=True)\n",
    "isnull = df_subset.references.isna()\n",
    "df_subset.loc[isnull, 'references'] = [ [[]] * isnull.sum() ]\n",
    "df_subset = df_subset.reset_index()\n",
    "df_subset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = {\n",
    "    'nodes': df_subset.to_dict(orient='records'),\n",
    "    'links': gen_links(df_subset)\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('paper_graph.json', 'w') as f:\n",
    "    json.dump(data, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count     1000.000000\n",
       "mean      2399.375000\n",
       "std       2609.109472\n",
       "min        932.000000\n",
       "25%       1152.000000\n",
       "50%       1502.500000\n",
       "75%       2458.000000\n",
       "max      24182.000000\n",
       "Name: n_citation, dtype: float64"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "df_subset.n_citation.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import feather"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "N=1000\n",
    "f = feather.read_dataframe('ml_papers_2.feather')\n",
    "fs = f.sort_values('n_citation', ascending=False)[:1000].reset_index(drop=True).reset_index()\n",
    "\n",
    "def gen_links2(df):\n",
    "    df = df.copy()\n",
    "    df = df.reset_index().set_index('id')\n",
    "    link = []\n",
    "    missed = 0\n",
    "    for i, r in df.iterrows():\n",
    "        for ref in list(map(lambda s: s.strip(), r.references.split(','))):\n",
    "            if ref not in df.index:\n",
    "                missed += 1\n",
    "                continue\n",
    "            link.append({\n",
    "                'source': int(r['index']),\n",
    "                'target': int(df.loc[ref]['index'])\n",
    "            })\n",
    "    return link"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = {\n",
    "    'nodes': fs.to_dict(orient='records'),\n",
    "    'links': gen_links2(fs)\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('paper_graph2.json', 'w') as f:\n",
    "    json.dump(data, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>authors</th>\n",
       "      <th>id</th>\n",
       "      <th>n_citation</th>\n",
       "      <th>references</th>\n",
       "      <th>title</th>\n",
       "      <th>venue</th>\n",
       "      <th>year</th>\n",
       "      <th>topic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Stéphane Mallat</td>\n",
       "      <td>7ccbdf09-a84e-4ad2-ab20-cb28b6c41155</td>\n",
       "      <td>24182</td>\n",
       "      <td>093cf9fe-6fda-4495-9e89-acef614d0453, 15a2345f...</td>\n",
       "      <td>A theory for multiresolution signal decomposit...</td>\n",
       "      <td>IEEE Transactions on Pattern Analysis and Mach...</td>\n",
       "      <td>1989</td>\n",
       "      <td>Pattern Recognition and Machine Learning</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Navneet Dalal, Bill Triggs</td>\n",
       "      <td>dd83785a-dd19-41e3-9b25-ebabbd48d336</td>\n",
       "      <td>22795</td>\n",
       "      <td>04d8a9cb-a14d-4ccf-8b19-da1327e86b91, 3e812129...</td>\n",
       "      <td>Histograms of oriented gradients for human det...</td>\n",
       "      <td>computer vision and pattern recognition</td>\n",
       "      <td>2005</td>\n",
       "      <td>Pattern Recognition and Machine Learning</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>David L. Donoho</td>\n",
       "      <td>f56b877b-4060-4754-b303-e8140968544c</td>\n",
       "      <td>20915</td>\n",
       "      <td>036a19f8-fdca-4e84-a237-e54f2108dcb4, 05c85ace...</td>\n",
       "      <td>Compressed sensing</td>\n",
       "      <td>IEEE Transactions on Information Theory</td>\n",
       "      <td>2006</td>\n",
       "      <td>Pattern Recognition and Machine Learning</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Mark A. Larkin, Gordon Blackshields, N. P. Bro...</td>\n",
       "      <td>8a408df6-2ae6-45a4-afb5-ca95bc7dc664</td>\n",
       "      <td>20580</td>\n",
       "      <td>239b9a0a-2a71-4780-960a-c565f19a1e9a, b2f2e8ed...</td>\n",
       "      <td>Clustal W and Clustal X version 2.0</td>\n",
       "      <td>Bioinformatics</td>\n",
       "      <td>2007</td>\n",
       "      <td>Signal and Speech Processing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>David Posada, Keith A. Crandall</td>\n",
       "      <td>e776f9db-fac3-40de-8968-19448d1693e1</td>\n",
       "      <td>19833</td>\n",
       "      <td></td>\n",
       "      <td>MODELTEST: testing the model of DNA substitution.</td>\n",
       "      <td>Bioinformatics</td>\n",
       "      <td>1998</td>\n",
       "      <td>Bioinformatics</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index                                            authors  \\\n",
       "0      0                                    Stéphane Mallat   \n",
       "1      1                         Navneet Dalal, Bill Triggs   \n",
       "2      2                                    David L. Donoho   \n",
       "3      3  Mark A. Larkin, Gordon Blackshields, N. P. Bro...   \n",
       "4      4                    David Posada, Keith A. Crandall   \n",
       "\n",
       "                                     id  n_citation  \\\n",
       "0  7ccbdf09-a84e-4ad2-ab20-cb28b6c41155       24182   \n",
       "1  dd83785a-dd19-41e3-9b25-ebabbd48d336       22795   \n",
       "2  f56b877b-4060-4754-b303-e8140968544c       20915   \n",
       "3  8a408df6-2ae6-45a4-afb5-ca95bc7dc664       20580   \n",
       "4  e776f9db-fac3-40de-8968-19448d1693e1       19833   \n",
       "\n",
       "                                          references  \\\n",
       "0  093cf9fe-6fda-4495-9e89-acef614d0453, 15a2345f...   \n",
       "1  04d8a9cb-a14d-4ccf-8b19-da1327e86b91, 3e812129...   \n",
       "2  036a19f8-fdca-4e84-a237-e54f2108dcb4, 05c85ace...   \n",
       "3  239b9a0a-2a71-4780-960a-c565f19a1e9a, b2f2e8ed...   \n",
       "4                                                      \n",
       "\n",
       "                                               title  \\\n",
       "0  A theory for multiresolution signal decomposit...   \n",
       "1  Histograms of oriented gradients for human det...   \n",
       "2                                 Compressed sensing   \n",
       "3                Clustal W and Clustal X version 2.0   \n",
       "4  MODELTEST: testing the model of DNA substitution.   \n",
       "\n",
       "                                               venue  year  \\\n",
       "0  IEEE Transactions on Pattern Analysis and Mach...  1989   \n",
       "1            computer vision and pattern recognition  2005   \n",
       "2            IEEE Transactions on Information Theory  2006   \n",
       "3                                     Bioinformatics  2007   \n",
       "4                                     Bioinformatics  1998   \n",
       "\n",
       "                                      topic  \n",
       "0  Pattern Recognition and Machine Learning  \n",
       "1  Pattern Recognition and Machine Learning  \n",
       "2  Pattern Recognition and Machine Learning  \n",
       "3              Signal and Speech Processing  \n",
       "4                            Bioinformatics  "
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fs.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>authors</th>\n",
       "      <th>id</th>\n",
       "      <th>n_citation</th>\n",
       "      <th>references</th>\n",
       "      <th>title</th>\n",
       "      <th>venue</th>\n",
       "      <th>year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>[Stéphane Mallat]</td>\n",
       "      <td>7ccbdf09-a84e-4ad2-ab20-cb28b6c41155</td>\n",
       "      <td>24182</td>\n",
       "      <td>[093cf9fe-6fda-4495-9e89-acef614d0453, 15a2345...</td>\n",
       "      <td>A theory for multiresolution signal decomposit...</td>\n",
       "      <td>IEEE Transactions on Pattern Analysis and Mach...</td>\n",
       "      <td>1989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>[Navneet Dalal, Bill Triggs]</td>\n",
       "      <td>dd83785a-dd19-41e3-9b25-ebabbd48d336</td>\n",
       "      <td>22795</td>\n",
       "      <td>[04d8a9cb-a14d-4ccf-8b19-da1327e86b91, 3e81212...</td>\n",
       "      <td>Histograms of oriented gradients for human det...</td>\n",
       "      <td>computer vision and pattern recognition</td>\n",
       "      <td>2005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>[David L. Donoho]</td>\n",
       "      <td>f56b877b-4060-4754-b303-e8140968544c</td>\n",
       "      <td>20915</td>\n",
       "      <td>[036a19f8-fdca-4e84-a237-e54f2108dcb4, 05c85ac...</td>\n",
       "      <td>Compressed sensing</td>\n",
       "      <td>IEEE Transactions on Information Theory</td>\n",
       "      <td>2006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>[Mark A. Larkin, Gordon Blackshields, N. P. Br...</td>\n",
       "      <td>8a408df6-2ae6-45a4-afb5-ca95bc7dc664</td>\n",
       "      <td>20580</td>\n",
       "      <td>[239b9a0a-2a71-4780-960a-c565f19a1e9a, b2f2e8e...</td>\n",
       "      <td>Clustal W and Clustal X version 2.0</td>\n",
       "      <td>Bioinformatics</td>\n",
       "      <td>2007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>[David Posada, Keith A. Crandall]</td>\n",
       "      <td>e776f9db-fac3-40de-8968-19448d1693e1</td>\n",
       "      <td>19833</td>\n",
       "      <td>[]</td>\n",
       "      <td>MODELTEST: testing the model of DNA substitution.</td>\n",
       "      <td>Bioinformatics</td>\n",
       "      <td>1998</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index                                            authors  \\\n",
       "0      0                                  [Stéphane Mallat]   \n",
       "1      1                       [Navneet Dalal, Bill Triggs]   \n",
       "2      2                                  [David L. Donoho]   \n",
       "3      3  [Mark A. Larkin, Gordon Blackshields, N. P. Br...   \n",
       "4      4                  [David Posada, Keith A. Crandall]   \n",
       "\n",
       "                                     id  n_citation  \\\n",
       "0  7ccbdf09-a84e-4ad2-ab20-cb28b6c41155       24182   \n",
       "1  dd83785a-dd19-41e3-9b25-ebabbd48d336       22795   \n",
       "2  f56b877b-4060-4754-b303-e8140968544c       20915   \n",
       "3  8a408df6-2ae6-45a4-afb5-ca95bc7dc664       20580   \n",
       "4  e776f9db-fac3-40de-8968-19448d1693e1       19833   \n",
       "\n",
       "                                          references  \\\n",
       "0  [093cf9fe-6fda-4495-9e89-acef614d0453, 15a2345...   \n",
       "1  [04d8a9cb-a14d-4ccf-8b19-da1327e86b91, 3e81212...   \n",
       "2  [036a19f8-fdca-4e84-a237-e54f2108dcb4, 05c85ac...   \n",
       "3  [239b9a0a-2a71-4780-960a-c565f19a1e9a, b2f2e8e...   \n",
       "4                                                 []   \n",
       "\n",
       "                                               title  \\\n",
       "0  A theory for multiresolution signal decomposit...   \n",
       "1  Histograms of oriented gradients for human det...   \n",
       "2                                 Compressed sensing   \n",
       "3                Clustal W and Clustal X version 2.0   \n",
       "4  MODELTEST: testing the model of DNA substitution.   \n",
       "\n",
       "                                               venue  year  \n",
       "0  IEEE Transactions on Pattern Analysis and Mach...  1989  \n",
       "1            computer vision and pattern recognition  2005  \n",
       "2            IEEE Transactions on Information Theory  2006  \n",
       "3                                     Bioinformatics  2007  \n",
       "4                                     Bioinformatics  1998  "
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_subset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'abc'"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "' abc'.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}