4a_text1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text analysis I\n",
    "\n",
    "In this notebook, we will:\n",
    "\n",
    "- Calculate interrater reliability.\n",
    "- Export data for coding.\n",
    "- Import coded data.\n",
    "- Score text with built-in TextBlob sentiment."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from textblob import TextBlob\n",
    "from sklearn.metrics import cohen_kappa_score\n",
    "from nltk import agreement"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Interrater reliability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>true_scores</th>\n",
       "      <th>item</th>\n",
       "      <th>r1</th>\n",
       "      <th>r2</th>\n",
       "      <th>r3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   true_scores  item  r1  r2  r3\n",
       "0            3     1   3   3   2\n",
       "1            1     1   1   1   1\n",
       "2            6     2   5   5   6\n",
       "3            2     3   2   1   1\n",
       "4            4     4   4   3   4"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Generate some data.\n",
    "true_scores = [random.randrange(1, 7) for i in range(200)]\n",
    "r1_diff = [random.randrange(-1, 1) for i in range(200)]\n",
    "r2_diff = [random.randrange(-1, 1) for i in range(200)]\n",
    "r3_diff = [random.randrange(-1, 1) for i in range(200)]\n",
    "\n",
    "ratings = pd.DataFrame(true_scores, columns=['true_scores'])\n",
    "\n",
    "ratings['item'] = range(200)\n",
    "ratings['r1'] = ratings['true_scores'] + r1_diff\n",
    "ratings['r2'] = ratings['true_scores'] + r2_diff\n",
    "ratings['r3'] = ratings['true_scores'] + r3_diff\n",
    "ratings= ratings.clip(1, 7)\n",
    "\n",
    "ratings.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.48573527611117917\n",
      "0.49135923520039226\n",
      "0.4420600858369098\n"
     ]
    }
   ],
   "source": [
    "# For two raters\n",
    "print(cohen_kappa_score(ratings['r1'], ratings['r2']))\n",
    "print(cohen_kappa_score(ratings['r2'], ratings['r3']))\n",
    "print(cohen_kappa_score(ratings['r1'], ratings['r3']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Kappa:   0.47305153238282704\n",
      "Fleiss:  0.4730607409828029\n",
      "Alpha:   0.4738458395751056\n"
     ]
    }
   ],
   "source": [
    "# NLTK\n",
    "# Note: recoding these is annoying, as you can see.\n",
    "#       We could reshape the columns and recode some stuff,\n",
    "#       but it's probably practically worse than this.\n",
    "\n",
    "ratings2_list = []\n",
    "rater = 0\n",
    "for rd in [r1_diff, r2_diff, r3_diff]:\n",
    "    values = np.array(true_scores) + np.array(rd)\n",
    "    df = pd.DataFrame(values, columns=['value'])\n",
    "    df = df.clip(1, 7)\n",
    "    df['rater'] = rater\n",
    "    df['item'] = range(200)\n",
    "    ratings2_list.append(df)\n",
    "    rater += 1\n",
    "\n",
    "ratings2 = pd.concat(ratings2_list, ignore_index=True)\n",
    "ratings_nltk = [tuple(x) for x in ratings2[['rater', 'item', 'value']].values]\n",
    "\n",
    "nltk_rate = agreement.AnnotationTask(data=ratings_nltk)\n",
    "\n",
    "print(f'Kappa:   {nltk_rate.kappa()}')\n",
    "print(f'Fleiss:  {nltk_rate.multi_kappa()}')\n",
    "print(f'Alpha:   {nltk_rate.alpha()}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exporting data for coding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>article:modified_time</th>\n",
       "      <th>article:published_time</th>\n",
       "      <th>article:section</th>\n",
       "      <th>body</th>\n",
       "      <th>h3</th>\n",
       "      <th>og:description</th>\n",
       "      <th>og:title</th>\n",
       "      <th>og:updated_time</th>\n",
       "      <th>og:url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2018-10-12T00:36:34+00:00</td>\n",
       "      <td>2018-10-11T23:16:52+00:00</td>\n",
       "      <td>Press Releases</td>\n",
       "      <td>REDMOND, Wash. — Oct. 11, 2018 — Microsoft Cor...</td>\n",
       "      <td>Microsoft to host earnings conference call web...</td>\n",
       "      <td>Microsoft to host earnings conference call web...</td>\n",
       "      <td>Microsoft announces quarterly earnings release...</td>\n",
       "      <td>2018-10-12T00:36:34+00:00</td>\n",
       "      <td>https://news.microsoft.com/2018/10/11/microsof...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2018-10-11T16:36:30+00:00</td>\n",
       "      <td>2018-10-11T14:00:25+00:00</td>\n",
       "      <td>Press Releases</td>\n",
       "      <td>INDIANAPOLIS – Oct. 11, 2018 – Today, Eric J. ...</td>\n",
       "      <td>With support provided by Walmart, and in partn...</td>\n",
       "      <td>With support provided by Walmart, and in partn...</td>\n",
       "      <td>Indiana Governor Eric J. Holcomb, the Markle F...</td>\n",
       "      <td>2018-10-11T16:36:30+00:00</td>\n",
       "      <td>https://news.microsoft.com/2018/10/11/indiana-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2018-10-11T07:05:46+00:00</td>\n",
       "      <td>2018-10-11T07:04:28+00:00</td>\n",
       "      <td>Press Releases</td>\n",
       "      <td>MUMBAI, India — Oct. 11, 2018 — Microsoft Corp...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Microsoft, Star India and Anil Kumble redefine...</td>\n",
       "      <td>Anil Kumble’s Microsoft AI-enabled Power Bat p...</td>\n",
       "      <td>2018-10-11T07:05:46+00:00</td>\n",
       "      <td>https://news.microsoft.com/2018/10/11/anil-kum...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>2018-10-11T20:29:31+00:00</td>\n",
       "      <td>2018-10-10T18:00:22+00:00</td>\n",
       "      <td>Press Releases</td>\n",
       "      <td>BOYDTON, Va. — Oct. 10, 2018 — Microsoft Corp....</td>\n",
       "      <td>NaN</td>\n",
       "      <td>BOYDTON, Va. — Oct. 10, 2018 — Microsoft Corp....</td>\n",
       "      <td>Microsoft and Lake Country Satellite roll out ...</td>\n",
       "      <td>2018-10-11T20:29:31+00:00</td>\n",
       "      <td>https://news.microsoft.com/2018/10/10/microsof...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018-10-10T15:15:58+00:00</td>\n",
       "      <td>Press Releases</td>\n",
       "      <td>Charleston, S.C. (October 10, 2018) — Blackbau...</td>\n",
       "      <td>Introduce first phase of initiative with joint...</td>\n",
       "      <td>Introduce first phase of initiative with joint...</td>\n",
       "      <td>Blackbaud and Microsoft expand partnership wit...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://news.microsoft.com/2018/10/10/blackbau...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       article:modified_time     article:published_time article:section  \\\n",
       "0  2018-10-12T00:36:34+00:00  2018-10-11T23:16:52+00:00  Press Releases   \n",
       "1  2018-10-11T16:36:30+00:00  2018-10-11T14:00:25+00:00  Press Releases   \n",
       "2  2018-10-11T07:05:46+00:00  2018-10-11T07:04:28+00:00  Press Releases   \n",
       "3  2018-10-11T20:29:31+00:00  2018-10-10T18:00:22+00:00  Press Releases   \n",
       "4                        NaN  2018-10-10T15:15:58+00:00  Press Releases   \n",
       "\n",
       "                                                body  \\\n",
       "0  REDMOND, Wash. — Oct. 11, 2018 — Microsoft Cor...   \n",
       "1  INDIANAPOLIS – Oct. 11, 2018 – Today, Eric J. ...   \n",
       "2  MUMBAI, India — Oct. 11, 2018 — Microsoft Corp...   \n",
       "3  BOYDTON, Va. — Oct. 10, 2018 — Microsoft Corp....   \n",
       "4  Charleston, S.C. (October 10, 2018) — Blackbau...   \n",
       "\n",
       "                                                  h3  \\\n",
       "0  Microsoft to host earnings conference call web...   \n",
       "1  With support provided by Walmart, and in partn...   \n",
       "2                                                NaN   \n",
       "3                                                NaN   \n",
       "4  Introduce first phase of initiative with joint...   \n",
       "\n",
       "                                      og:description  \\\n",
       "0  Microsoft to host earnings conference call web...   \n",
       "1  With support provided by Walmart, and in partn...   \n",
       "2  Microsoft, Star India and Anil Kumble redefine...   \n",
       "3  BOYDTON, Va. — Oct. 10, 2018 — Microsoft Corp....   \n",
       "4  Introduce first phase of initiative with joint...   \n",
       "\n",
       "                                            og:title  \\\n",
       "0  Microsoft announces quarterly earnings release...   \n",
       "1  Indiana Governor Eric J. Holcomb, the Markle F...   \n",
       "2  Anil Kumble’s Microsoft AI-enabled Power Bat p...   \n",
       "3  Microsoft and Lake Country Satellite roll out ...   \n",
       "4  Blackbaud and Microsoft expand partnership wit...   \n",
       "\n",
       "             og:updated_time  \\\n",
       "0  2018-10-12T00:36:34+00:00   \n",
       "1  2018-10-11T16:36:30+00:00   \n",
       "2  2018-10-11T07:05:46+00:00   \n",
       "3  2018-10-11T20:29:31+00:00   \n",
       "4                        NaN   \n",
       "\n",
       "                                              og:url  \n",
       "0  https://news.microsoft.com/2018/10/11/microsof...  \n",
       "1  https://news.microsoft.com/2018/10/11/indiana-...  \n",
       "2  https://news.microsoft.com/2018/10/11/anil-kum...  \n",
       "3  https://news.microsoft.com/2018/10/10/microsof...  \n",
       "4  https://news.microsoft.com/2018/10/10/blackbau...  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "msft_prs = pd.read_csv('./data/msft_prs.csv')\n",
    "msft_prs.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "msft_prs_export = msft_prs[['og:title']].copy()\n",
    "\n",
    "# We'd usually recover this from a filename or other metadata.\n",
    "msft_prs_export.loc[:,'id_ticker'] = 'msft'\n",
    "\n",
    "# Add empty columns for coding.\n",
    "_PR_COLS = ['cat_partner', 'cat_dividend', 'cat_award']\n",
    "for col in _PR_COLS:\n",
    "    msft_prs_export.loc[:,col] = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>og:title</th>\n",
       "      <th>id_ticker</th>\n",
       "      <th>cat_partner</th>\n",
       "      <th>cat_dividend</th>\n",
       "      <th>cat_award</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>Microsoft announces quarterly earnings release...</td>\n",
       "      <td>msft</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>Indiana Governor Eric J. Holcomb, the Markle F...</td>\n",
       "      <td>msft</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>Anil Kumble’s Microsoft AI-enabled Power Bat p...</td>\n",
       "      <td>msft</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>Microsoft and Lake Country Satellite roll out ...</td>\n",
       "      <td>msft</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>Blackbaud and Microsoft expand partnership wit...</td>\n",
       "      <td>msft</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            og:title id_ticker cat_partner  \\\n",
       "0  Microsoft announces quarterly earnings release...      msft               \n",
       "1  Indiana Governor Eric J. Holcomb, the Markle F...      msft               \n",
       "2  Anil Kumble’s Microsoft AI-enabled Power Bat p...      msft               \n",
       "3  Microsoft and Lake Country Satellite roll out ...      msft               \n",
       "4  Blackbaud and Microsoft expand partnership wit...      msft               \n",
       "\n",
       "  cat_dividend cat_award  \n",
       "0                         \n",
       "1                         \n",
       "2                         \n",
       "3                         \n",
       "4                         "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "msft_prs_export.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let's reorder to put the title at the end and export\n",
    "msft_prs_export[['id_ticker', 'cat_partner', \n",
    "                 'cat_dividend', 'cat_award',\n",
    "                 'og:title']].to_csv('./data/msft_pr_coding.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import from LIWC\n",
    "\n",
    "I ran these in LIWC for us.\n",
    "\n",
    "Note that it would be similar to import human-coded data, and we would not be dropping those columns in that case."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Source (A)</th>\n",
       "      <th>Source (B)</th>\n",
       "      <th>Source (C)</th>\n",
       "      <th>Source (D)</th>\n",
       "      <th>Source (E)</th>\n",
       "      <th>Source (F)</th>\n",
       "      <th>WC</th>\n",
       "      <th>Analytic</th>\n",
       "      <th>Clout</th>\n",
       "      <th>Authentic</th>\n",
       "      <th>...</th>\n",
       "      <th>Comma</th>\n",
       "      <th>Colon</th>\n",
       "      <th>SemiC</th>\n",
       "      <th>QMark</th>\n",
       "      <th>Exclam</th>\n",
       "      <th>Dash</th>\n",
       "      <th>Quote</th>\n",
       "      <th>Apostro</th>\n",
       "      <th>Parenth</th>\n",
       "      <th>OtherP</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>msft</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Grab forges strategic cloud partnership with M...</td>\n",
       "      <td>19</td>\n",
       "      <td>99.00</td>\n",
       "      <td>85.38</td>\n",
       "      <td>19.27</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.26</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>msft</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Redline Communications and Microsoft announce ...</td>\n",
       "      <td>16</td>\n",
       "      <td>99.00</td>\n",
       "      <td>96.95</td>\n",
       "      <td>35.37</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.25</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>msft</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Adobe, Microsoft and SAP announce the Open Dat...</td>\n",
       "      <td>18</td>\n",
       "      <td>99.00</td>\n",
       "      <td>71.09</td>\n",
       "      <td>58.07</td>\n",
       "      <td>...</td>\n",
       "      <td>5.56</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.56</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>msft</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Microsoft fortifies security and brings AI to ...</td>\n",
       "      <td>13</td>\n",
       "      <td>99.00</td>\n",
       "      <td>77.92</td>\n",
       "      <td>17.46</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.69</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>msft</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Shell expands strategic collaboration with Mic...</td>\n",
       "      <td>13</td>\n",
       "      <td>97.77</td>\n",
       "      <td>77.92</td>\n",
       "      <td>17.46</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.69</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 99 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Source (A) Source (B)  Source (C)  Source (D)  Source (E)  \\\n",
       "0           0       msft         NaN         NaN         NaN   \n",
       "1           1       msft         NaN         NaN         NaN   \n",
       "2           2       msft         NaN         NaN         NaN   \n",
       "3           3       msft         NaN         NaN         NaN   \n",
       "4           4       msft         NaN         NaN         NaN   \n",
       "\n",
       "                                          Source (F)  WC  Analytic  Clout  \\\n",
       "0  Grab forges strategic cloud partnership with M...  19     99.00  85.38   \n",
       "1  Redline Communications and Microsoft announce ...  16     99.00  96.95   \n",
       "2  Adobe, Microsoft and SAP announce the Open Dat...  18     99.00  71.09   \n",
       "3  Microsoft fortifies security and brings AI to ...  13     99.00  77.92   \n",
       "4  Shell expands strategic collaboration with Mic...  13     97.77  77.92   \n",
       "\n",
       "   Authentic  ...  Comma  Colon  SemiC  QMark  Exclam  Dash  Quote  Apostro  \\\n",
       "0      19.27  ...   0.00    0.0    0.0    0.0     0.0  5.26    0.0      0.0   \n",
       "1      35.37  ...   0.00    0.0    0.0    0.0     0.0  6.25    0.0      0.0   \n",
       "2      58.07  ...   5.56    0.0    0.0    0.0     0.0  5.56    0.0      0.0   \n",
       "3      17.46  ...   0.00    0.0    0.0    0.0     0.0  7.69    0.0      0.0   \n",
       "4      17.46  ...   0.00    0.0    0.0    0.0     0.0  7.69    0.0      0.0   \n",
       "\n",
       "   Parenth  OtherP  \n",
       "0      0.0     0.0  \n",
       "1      0.0     0.0  \n",
       "2      0.0     0.0  \n",
       "3      0.0     0.0  \n",
       "4      0.0     0.0  \n",
       "\n",
       "[5 rows x 99 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "msft_liwc = pd.read_csv('./data/msft_pr_liwc.csv')\n",
    "msft_liwc.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To merge these back together, we want to use a key that makes sense.\n",
    "We could use index, but there's a good reason that I am not.\n",
    "It's likely that Microsoft will have a new press release between the time I'm preparing materials and the time that we use them.\n",
    "So, we want to merge on something that will make sense.\n",
    "\n",
    "Here, for expediency, I'm using the title column.\n",
    "However, I often use a hash function on the document body to create a unique document ID.\n",
    "This is especially helpful in news collections where a lot of the documents are returned for more than one search.\n",
    "We can calculate hash values, remove duplicates, and export those for coding.\n",
    "This prevents rework."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>og:title</th>\n",
       "      <th>WC</th>\n",
       "      <th>Analytic</th>\n",
       "      <th>Clout</th>\n",
       "      <th>Authentic</th>\n",
       "      <th>Tone</th>\n",
       "      <th>WPS</th>\n",
       "      <th>Sixltr</th>\n",
       "      <th>Dic</th>\n",
       "      <th>function</th>\n",
       "      <th>...</th>\n",
       "      <th>Comma</th>\n",
       "      <th>Colon</th>\n",
       "      <th>SemiC</th>\n",
       "      <th>QMark</th>\n",
       "      <th>Exclam</th>\n",
       "      <th>Dash</th>\n",
       "      <th>Quote</th>\n",
       "      <th>Apostro</th>\n",
       "      <th>Parenth</th>\n",
       "      <th>OtherP</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>Grab forges strategic cloud partnership with M...</td>\n",
       "      <td>19</td>\n",
       "      <td>99.00</td>\n",
       "      <td>85.38</td>\n",
       "      <td>19.27</td>\n",
       "      <td>25.77</td>\n",
       "      <td>19.0</td>\n",
       "      <td>47.37</td>\n",
       "      <td>68.42</td>\n",
       "      <td>26.32</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.26</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>Redline Communications and Microsoft announce ...</td>\n",
       "      <td>16</td>\n",
       "      <td>99.00</td>\n",
       "      <td>96.95</td>\n",
       "      <td>35.37</td>\n",
       "      <td>1.00</td>\n",
       "      <td>16.0</td>\n",
       "      <td>43.75</td>\n",
       "      <td>81.25</td>\n",
       "      <td>25.00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.25</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>Adobe, Microsoft and SAP announce the Open Dat...</td>\n",
       "      <td>18</td>\n",
       "      <td>99.00</td>\n",
       "      <td>71.09</td>\n",
       "      <td>58.07</td>\n",
       "      <td>25.77</td>\n",
       "      <td>18.0</td>\n",
       "      <td>44.44</td>\n",
       "      <td>66.67</td>\n",
       "      <td>27.78</td>\n",
       "      <td>...</td>\n",
       "      <td>5.56</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.56</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>Microsoft fortifies security and brings AI to ...</td>\n",
       "      <td>13</td>\n",
       "      <td>99.00</td>\n",
       "      <td>77.92</td>\n",
       "      <td>17.46</td>\n",
       "      <td>99.00</td>\n",
       "      <td>13.0</td>\n",
       "      <td>30.77</td>\n",
       "      <td>61.54</td>\n",
       "      <td>30.77</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.69</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>Shell expands strategic collaboration with Mic...</td>\n",
       "      <td>13</td>\n",
       "      <td>97.77</td>\n",
       "      <td>77.92</td>\n",
       "      <td>17.46</td>\n",
       "      <td>25.77</td>\n",
       "      <td>13.0</td>\n",
       "      <td>61.54</td>\n",
       "      <td>69.23</td>\n",
       "      <td>23.08</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.69</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 94 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            og:title  WC  Analytic  Clout  \\\n",
       "0  Grab forges strategic cloud partnership with M...  19     99.00  85.38   \n",
       "1  Redline Communications and Microsoft announce ...  16     99.00  96.95   \n",
       "2  Adobe, Microsoft and SAP announce the Open Dat...  18     99.00  71.09   \n",
       "3  Microsoft fortifies security and brings AI to ...  13     99.00  77.92   \n",
       "4  Shell expands strategic collaboration with Mic...  13     97.77  77.92   \n",
       "\n",
       "   Authentic   Tone   WPS  Sixltr    Dic  function  ...  Comma  Colon  SemiC  \\\n",
       "0      19.27  25.77  19.0   47.37  68.42     26.32  ...   0.00    0.0    0.0   \n",
       "1      35.37   1.00  16.0   43.75  81.25     25.00  ...   0.00    0.0    0.0   \n",
       "2      58.07  25.77  18.0   44.44  66.67     27.78  ...   5.56    0.0    0.0   \n",
       "3      17.46  99.00  13.0   30.77  61.54     30.77  ...   0.00    0.0    0.0   \n",
       "4      17.46  25.77  13.0   61.54  69.23     23.08  ...   0.00    0.0    0.0   \n",
       "\n",
       "   QMark  Exclam  Dash  Quote  Apostro  Parenth  OtherP  \n",
       "0    0.0     0.0  5.26    0.0      0.0      0.0     0.0  \n",
       "1    0.0     0.0  6.25    0.0      0.0      0.0     0.0  \n",
       "2    0.0     0.0  5.56    0.0      0.0      0.0     0.0  \n",
       "3    0.0     0.0  7.69    0.0      0.0      0.0     0.0  \n",
       "4    0.0     0.0  7.69    0.0      0.0      0.0     0.0  \n",
       "\n",
       "[5 rows x 94 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Drop the columns we already have.\n",
    "_DROP_LIWC = [f'Source ({i})' for i in 'ABCDE']\n",
    "msft_liwc = msft_liwc.drop(_DROP_LIWC, axis=1)\n",
    "msft_liwc= msft_liwc.rename(columns={'Source (F)': 'og:title'})\n",
    "msft_liwc.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>article:modified_time</th>\n",
       "      <th>article:published_time</th>\n",
       "      <th>article:section</th>\n",
       "      <th>body_x</th>\n",
       "      <th>h3</th>\n",
       "      <th>og:description</th>\n",
       "      <th>og:title</th>\n",
       "      <th>og:updated_time</th>\n",
       "      <th>og:url</th>\n",
       "      <th>WC</th>\n",
       "      <th>...</th>\n",
       "      <th>Comma</th>\n",
       "      <th>Colon</th>\n",
       "      <th>SemiC</th>\n",
       "      <th>QMark</th>\n",
       "      <th>Exclam</th>\n",
       "      <th>Dash</th>\n",
       "      <th>Quote</th>\n",
       "      <th>Apostro</th>\n",
       "      <th>Parenth</th>\n",
       "      <th>OtherP</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2018-10-08T23:39:05+00:00</td>\n",
       "      <td>2018-10-08T23:00:25+00:00</td>\n",
       "      <td>Press Releases</td>\n",
       "      <td>SINGAPORE and REDMOND, Wash. — Oct. 8, 2018 — ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Grab will adopt Microsoft Azure as its preferr...</td>\n",
       "      <td>Grab forges strategic cloud partnership with M...</td>\n",
       "      <td>2018-10-08T23:39:05+00:00</td>\n",
       "      <td>https://news.microsoft.com/2018/10/08/grab-for...</td>\n",
       "      <td>19</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.26</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2018-10-04T14:43:59+00:00</td>\n",
       "      <td>2018-10-04T13:00:35+00:00</td>\n",
       "      <td>Press Releases</td>\n",
       "      <td>REDMOND, Wash. — Oct. 4, 2018 — On Thursday, R...</td>\n",
       "      <td>The partnership will help make broadband more ...</td>\n",
       "      <td>The partnership will help make broadband more ...</td>\n",
       "      <td>Redline Communications and Microsoft announce ...</td>\n",
       "      <td>2018-10-04T14:43:59+00:00</td>\n",
       "      <td>https://news.microsoft.com/2018/10/04/redline-...</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.25</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2018-09-24T16:11:34+00:00</td>\n",
       "      <td>2018-09-24T13:30:14+00:00</td>\n",
       "      <td>Press Releases</td>\n",
       "      <td>\\nShantanu Narayen, CEO, Adobe (left), Satya N...</td>\n",
       "      <td>Industry leaders team up to help customers con...</td>\n",
       "      <td>Industry leaders team up to help customers con...</td>\n",
       "      <td>Adobe, Microsoft and SAP announce the Open Dat...</td>\n",
       "      <td>2018-09-24T16:11:34+00:00</td>\n",
       "      <td>https://news.microsoft.com/2018/09/24/adobe-mi...</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>5.56</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.56</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018-09-24T13:01:27+00:00</td>\n",
       "      <td>Press Releases</td>\n",
       "      <td>ORLANDO, Fla. — Sept. 24, 2018 — Monday at Ign...</td>\n",
       "      <td>The company also demonstrated how AI, IoT and ...</td>\n",
       "      <td>The company also demonstrated how AI, IoT and ...</td>\n",
       "      <td>Microsoft fortifies security and brings AI to ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://news.microsoft.com/2018/09/24/microsof...</td>\n",
       "      <td>13</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.69</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>2018-09-20T15:19:45+00:00</td>\n",
       "      <td>2018-09-20T13:00:12+00:00</td>\n",
       "      <td>Press Releases</td>\n",
       "      <td>REDMOND, Wash., and HOUSTON — Sept. 20, 2018 —...</td>\n",
       "      <td>Shell appoints C3 IoT and Microsoft Azure as i...</td>\n",
       "      <td>Shell appoints C3 IoT and Microsoft Azure as i...</td>\n",
       "      <td>Shell expands strategic collaboration with Mic...</td>\n",
       "      <td>2018-09-20T15:19:45+00:00</td>\n",
       "      <td>https://news.microsoft.com/2018/09/20/shell-ex...</td>\n",
       "      <td>13</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.69</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 102 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       article:modified_time     article:published_time article:section  \\\n",
       "0  2018-10-08T23:39:05+00:00  2018-10-08T23:00:25+00:00  Press Releases   \n",
       "1  2018-10-04T14:43:59+00:00  2018-10-04T13:00:35+00:00  Press Releases   \n",
       "2  2018-09-24T16:11:34+00:00  2018-09-24T13:30:14+00:00  Press Releases   \n",
       "3                        NaN  2018-09-24T13:01:27+00:00  Press Releases   \n",
       "4  2018-09-20T15:19:45+00:00  2018-09-20T13:00:12+00:00  Press Releases   \n",
       "\n",
       "                                              body_x  \\\n",
       "0  SINGAPORE and REDMOND, Wash. — Oct. 8, 2018 — ...   \n",
       "1  REDMOND, Wash. — Oct. 4, 2018 — On Thursday, R...   \n",
       "2  \\nShantanu Narayen, CEO, Adobe (left), Satya N...   \n",
       "3  ORLANDO, Fla. — Sept. 24, 2018 — Monday at Ign...   \n",
       "4  REDMOND, Wash., and HOUSTON — Sept. 20, 2018 —...   \n",
       "\n",
       "                                                  h3  \\\n",
       "0                                                NaN   \n",
       "1  The partnership will help make broadband more ...   \n",
       "2  Industry leaders team up to help customers con...   \n",
       "3  The company also demonstrated how AI, IoT and ...   \n",
       "4  Shell appoints C3 IoT and Microsoft Azure as i...   \n",
       "\n",
       "                                      og:description  \\\n",
       "0  Grab will adopt Microsoft Azure as its preferr...   \n",
       "1  The partnership will help make broadband more ...   \n",
       "2  Industry leaders team up to help customers con...   \n",
       "3  The company also demonstrated how AI, IoT and ...   \n",
       "4  Shell appoints C3 IoT and Microsoft Azure as i...   \n",
       "\n",
       "                                            og:title  \\\n",
       "0  Grab forges strategic cloud partnership with M...   \n",
       "1  Redline Communications and Microsoft announce ...   \n",
       "2  Adobe, Microsoft and SAP announce the Open Dat...   \n",
       "3  Microsoft fortifies security and brings AI to ...   \n",
       "4  Shell expands strategic collaboration with Mic...   \n",
       "\n",
       "             og:updated_time  \\\n",
       "0  2018-10-08T23:39:05+00:00   \n",
       "1  2018-10-04T14:43:59+00:00   \n",
       "2  2018-09-24T16:11:34+00:00   \n",
       "3                        NaN   \n",
       "4  2018-09-20T15:19:45+00:00   \n",
       "\n",
       "                                              og:url  WC  ...  Comma  Colon  \\\n",
       "0  https://news.microsoft.com/2018/10/08/grab-for...  19  ...   0.00    0.0   \n",
       "1  https://news.microsoft.com/2018/10/04/redline-...  16  ...   0.00    0.0   \n",
       "2  https://news.microsoft.com/2018/09/24/adobe-mi...  18  ...   5.56    0.0   \n",
       "3  https://news.microsoft.com/2018/09/24/microsof...  13  ...   0.00    0.0   \n",
       "4  https://news.microsoft.com/2018/09/20/shell-ex...  13  ...   0.00    0.0   \n",
       "\n",
       "   SemiC  QMark  Exclam  Dash  Quote  Apostro  Parenth  OtherP  \n",
       "0    0.0    0.0     0.0  5.26    0.0      0.0      0.0     0.0  \n",
       "1    0.0    0.0     0.0  6.25    0.0      0.0      0.0     0.0  \n",
       "2    0.0    0.0     0.0  5.56    0.0      0.0      0.0     0.0  \n",
       "3    0.0    0.0     0.0  7.69    0.0      0.0      0.0     0.0  \n",
       "4    0.0    0.0     0.0  7.69    0.0      0.0      0.0     0.0  \n",
       "\n",
       "[5 rows x 102 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "msft_prs = msft_prs.merge(msft_liwc, on='og:title')\n",
    "msft_prs.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TextBlob\n",
    "\n",
    "Before applying textblob to our press releases, let's look at some examples to get a sense of how it works."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sentiment(polarity=0.375, subjectivity=0.2)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string1 = 'Text analysis is a lot of fun!'\n",
    "TextBlob(string1).sentiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sentiment(polarity=0.25, subjectivity=1.0)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string2 = 'It is not scary.'\n",
    "TextBlob(string2).sentiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sentiment(polarity=-1.0, subjectivity=1.0)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string3 = 'Manual coding is the worst.'\n",
    "TextBlob(string3).sentiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sentiment(polarity=-0.2, subjectivity=0.95)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string4 = 'Manual coding is kind of the worst.'\n",
    "TextBlob(string4).sentiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sentiment(polarity=-0.155, subjectivity=0.82)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strings = ' '.join([string1, string2, string3, string4])\n",
    "TextBlob(strings).sentiment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Like our comparison of web scraping and APIs, it's nice when we can work with data entirely inside of Python.\n",
    "\n",
    "We're going to start by defining some simple functions that we can use with pandas."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tb_subjectivity(text):\n",
    "    return TextBlob(text).sentiment.subjectivity\n",
    "\n",
    "def tb_polarity(text):\n",
    "    return TextBlob(text).sentiment.polarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "msft_prs['tb_subj'] = msft_prs['og:title'].apply(tb_subjectivity)\n",
    "msft_prs['tb_polar'] = msft_prs['og:title'].apply(tb_polarity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>article:modified_time</th>\n",
       "      <th>article:published_time</th>\n",
       "      <th>article:section</th>\n",
       "      <th>body_x</th>\n",
       "      <th>h3</th>\n",
       "      <th>og:description</th>\n",
       "      <th>og:title</th>\n",
       "      <th>og:updated_time</th>\n",
       "      <th>og:url</th>\n",
       "      <th>WC</th>\n",
       "      <th>...</th>\n",
       "      <th>SemiC</th>\n",
       "      <th>QMark</th>\n",
       "      <th>Exclam</th>\n",
       "      <th>Dash</th>\n",
       "      <th>Quote</th>\n",
       "      <th>Apostro</th>\n",
       "      <th>Parenth</th>\n",
       "      <th>OtherP</th>\n",
       "      <th>tb_subj</th>\n",
       "      <th>tb_polar</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2018-10-08T23:39:05+00:00</td>\n",
       "      <td>2018-10-08T23:00:25+00:00</td>\n",
       "      <td>Press Releases</td>\n",
       "      <td>SINGAPORE and REDMOND, Wash. — Oct. 8, 2018 — ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Grab will adopt Microsoft Azure as its preferr...</td>\n",
       "      <td>Grab forges strategic cloud partnership with M...</td>\n",
       "      <td>2018-10-08T23:39:05+00:00</td>\n",
       "      <td>https://news.microsoft.com/2018/10/08/grab-for...</td>\n",
       "      <td>19</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.26</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2018-10-04T14:43:59+00:00</td>\n",
       "      <td>2018-10-04T13:00:35+00:00</td>\n",
       "      <td>Press Releases</td>\n",
       "      <td>REDMOND, Wash. — Oct. 4, 2018 — On Thursday, R...</td>\n",
       "      <td>The partnership will help make broadband more ...</td>\n",
       "      <td>The partnership will help make broadband more ...</td>\n",
       "      <td>Redline Communications and Microsoft announce ...</td>\n",
       "      <td>2018-10-04T14:43:59+00:00</td>\n",
       "      <td>https://news.microsoft.com/2018/10/04/redline-...</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.25</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2018-09-24T16:11:34+00:00</td>\n",
       "      <td>2018-09-24T13:30:14+00:00</td>\n",
       "      <td>Press Releases</td>\n",
       "      <td>\\nShantanu Narayen, CEO, Adobe (left), Satya N...</td>\n",
       "      <td>Industry leaders team up to help customers con...</td>\n",
       "      <td>Industry leaders team up to help customers con...</td>\n",
       "      <td>Adobe, Microsoft and SAP announce the Open Dat...</td>\n",
       "      <td>2018-09-24T16:11:34+00:00</td>\n",
       "      <td>https://news.microsoft.com/2018/09/24/adobe-mi...</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.56</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.477273</td>\n",
       "      <td>0.068182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018-09-24T13:01:27+00:00</td>\n",
       "      <td>Press Releases</td>\n",
       "      <td>ORLANDO, Fla. — Sept. 24, 2018 — Monday at Ign...</td>\n",
       "      <td>The company also demonstrated how AI, IoT and ...</td>\n",
       "      <td>The company also demonstrated how AI, IoT and ...</td>\n",
       "      <td>Microsoft fortifies security and brings AI to ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://news.microsoft.com/2018/09/24/microsof...</td>\n",
       "      <td>13</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.69</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>2018-09-20T15:19:45+00:00</td>\n",
       "      <td>2018-09-20T13:00:12+00:00</td>\n",
       "      <td>Press Releases</td>\n",
       "      <td>REDMOND, Wash., and HOUSTON — Sept. 20, 2018 —...</td>\n",
       "      <td>Shell appoints C3 IoT and Microsoft Azure as i...</td>\n",
       "      <td>Shell appoints C3 IoT and Microsoft Azure as i...</td>\n",
       "      <td>Shell expands strategic collaboration with Mic...</td>\n",
       "      <td>2018-09-20T15:19:45+00:00</td>\n",
       "      <td>https://news.microsoft.com/2018/09/20/shell-ex...</td>\n",
       "      <td>13</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.69</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 104 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       article:modified_time     article:published_time article:section  \\\n",
       "0  2018-10-08T23:39:05+00:00  2018-10-08T23:00:25+00:00  Press Releases   \n",
       "1  2018-10-04T14:43:59+00:00  2018-10-04T13:00:35+00:00  Press Releases   \n",
       "2  2018-09-24T16:11:34+00:00  2018-09-24T13:30:14+00:00  Press Releases   \n",
       "3                        NaN  2018-09-24T13:01:27+00:00  Press Releases   \n",
       "4  2018-09-20T15:19:45+00:00  2018-09-20T13:00:12+00:00  Press Releases   \n",
       "\n",
       "                                              body_x  \\\n",
       "0  SINGAPORE and REDMOND, Wash. — Oct. 8, 2018 — ...   \n",
       "1  REDMOND, Wash. — Oct. 4, 2018 — On Thursday, R...   \n",
       "2  \\nShantanu Narayen, CEO, Adobe (left), Satya N...   \n",
       "3  ORLANDO, Fla. — Sept. 24, 2018 — Monday at Ign...   \n",
       "4  REDMOND, Wash., and HOUSTON — Sept. 20, 2018 —...   \n",
       "\n",
       "                                                  h3  \\\n",
       "0                                                NaN   \n",
       "1  The partnership will help make broadband more ...   \n",
       "2  Industry leaders team up to help customers con...   \n",
       "3  The company also demonstrated how AI, IoT and ...   \n",
       "4  Shell appoints C3 IoT and Microsoft Azure as i...   \n",
       "\n",
       "                                      og:description  \\\n",
       "0  Grab will adopt Microsoft Azure as its preferr...   \n",
       "1  The partnership will help make broadband more ...   \n",
       "2  Industry leaders team up to help customers con...   \n",
       "3  The company also demonstrated how AI, IoT and ...   \n",
       "4  Shell appoints C3 IoT and Microsoft Azure as i...   \n",
       "\n",
       "                                            og:title  \\\n",
       "0  Grab forges strategic cloud partnership with M...   \n",
       "1  Redline Communications and Microsoft announce ...   \n",
       "2  Adobe, Microsoft and SAP announce the Open Dat...   \n",
       "3  Microsoft fortifies security and brings AI to ...   \n",
       "4  Shell expands strategic collaboration with Mic...   \n",
       "\n",
       "             og:updated_time  \\\n",
       "0  2018-10-08T23:39:05+00:00   \n",
       "1  2018-10-04T14:43:59+00:00   \n",
       "2  2018-09-24T16:11:34+00:00   \n",
       "3                        NaN   \n",
       "4  2018-09-20T15:19:45+00:00   \n",
       "\n",
       "                                              og:url  WC  ...  SemiC  QMark  \\\n",
       "0  https://news.microsoft.com/2018/10/08/grab-for...  19  ...    0.0    0.0   \n",
       "1  https://news.microsoft.com/2018/10/04/redline-...  16  ...    0.0    0.0   \n",
       "2  https://news.microsoft.com/2018/09/24/adobe-mi...  18  ...    0.0    0.0   \n",
       "3  https://news.microsoft.com/2018/09/24/microsof...  13  ...    0.0    0.0   \n",
       "4  https://news.microsoft.com/2018/09/20/shell-ex...  13  ...    0.0    0.0   \n",
       "\n",
       "   Exclam  Dash  Quote  Apostro  Parenth  OtherP   tb_subj  tb_polar  \n",
       "0     0.0  5.26    0.0      0.0      0.0     0.0  0.000000  0.000000  \n",
       "1     0.0  6.25    0.0      0.0      0.0     0.0  0.000000  0.000000  \n",
       "2     0.0  5.56    0.0      0.0      0.0     0.0  0.477273  0.068182  \n",
       "3     0.0  7.69    0.0      0.0      0.0     0.0  0.000000  0.000000  \n",
       "4     0.0  7.69    0.0      0.0      0.0     0.0  0.000000  0.000000  \n",
       "\n",
       "[5 rows x 104 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "msft_prs.head()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}