py/Pandas-CrashCourse.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "cff6d73e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3e6b4fd1",
   "metadata": {},
   "source": [
    "# Creading DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "653bb5a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Using 2d array\n",
    "df = pd.DataFrame(\n",
    "    [[4, 7, 10],\n",
    "    [5, 8, 11],\n",
    "    [6, 9, 12]],\n",
    "    index=[1, 2, 3],\n",
    "    columns=['a', 'b', 'c'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8eaa453c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Using dictionary (Preferred)\n",
    "df = pd.DataFrame({\n",
    "    \"a\" : [3, 4 ,5, 6],\n",
    "    \"b\" : [6, np.nan, 8, 9],\n",
    "    \"c\" : [9, 10, np.nan, 12]\n",
    "    },\n",
    "    index = [1, 1, 2, 3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c5b09db6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4, 3)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape\n",
    "# len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "543ef5db",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "a      int64\n",
       "b    float64\n",
       "c    float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "24dba223",
   "metadata": {},
   "source": [
    "# Indexing/Slicing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e531f432",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    3\n",
       "1    4\n",
       "2    5\n",
       "3    6\n",
       "Name: a, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"a\"]  # one column: \"Series\"\n",
    "# pd.Series([3, 4, 5, 6], index=[1,2,3,4])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b565ee9a",
   "metadata": {},
   "source": [
    "* `loc` gets rows (and/or columns) with particular labels.\n",
    "* `iloc` gets rows (and/or columns) at integer locations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b7772b50",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "a    5.0\n",
       "b    8.0\n",
       "c    NaN\n",
       "Name: 2, dtype: float64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc[2]  # index value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e95bd94c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5.0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc[2, 'a']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8bb754a1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1     9.0\n",
       "1    10.0\n",
       "2     NaN\n",
       "3    12.0\n",
       "Name: c, dtype: float64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[:,2]  # array index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "88f2772e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9.0</td>\n",
       "      <td>12.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     b     c\n",
       "1  6.0   9.0\n",
       "1  NaN  10.0\n",
       "2  8.0   NaN\n",
       "3  9.0  12.0"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[:,[1, 2]]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "afa754c9",
   "metadata": {},
   "source": [
    "# Extrat data as numpy array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "d6ac38e4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 3.,  6.,  9.],\n",
       "       [ 4., nan, 10.],\n",
       "       [ 5.,  8., nan],\n",
       "       [ 6.,  9., 12.]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val = df.values\n",
    "val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2c4792de",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([3, 4, 5, 6], dtype=int64)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val = df['a'].values\n",
    "val"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2118b494",
   "metadata": {},
   "source": [
    "# Summarize Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "dc64d32a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "a     4.500000\n",
       "b     7.666667\n",
       "c    10.333333\n",
       "dtype: float64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.mean(axis=0)  # axis=0: columns, axis=1: rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "531aad98",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>4.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>4.500000</td>\n",
       "      <td>7.666667</td>\n",
       "      <td>10.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.290994</td>\n",
       "      <td>1.527525</td>\n",
       "      <td>1.527525</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>9.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>3.750000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>9.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>4.500000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>5.250000</td>\n",
       "      <td>8.500000</td>\n",
       "      <td>11.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>6.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>12.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              a         b          c\n",
       "count  4.000000  3.000000   3.000000\n",
       "mean   4.500000  7.666667  10.333333\n",
       "std    1.290994  1.527525   1.527525\n",
       "min    3.000000  6.000000   9.000000\n",
       "25%    3.750000  7.000000   9.500000\n",
       "50%    4.500000  8.000000  10.000000\n",
       "75%    5.250000  8.500000  11.000000\n",
       "max    6.000000  9.000000  12.000000"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c680636",
   "metadata": {},
   "source": [
    "# Making changes in DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "f327f86e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   a    b\n",
       "1  3  6.0\n",
       "1  4  NaN\n",
       "2  5  8.0\n",
       "3  6  9.0"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Subset of columns\n",
    "df[['a', 'b']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "8dcb9155",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>6.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   a    b     c\n",
       "1  3  6.0   9.0\n",
       "1  4  NaN  10.0"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Drop rows\n",
    "df.drop([2,3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "0224cdc1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>c</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>12.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   a     c\n",
       "1  3   9.0\n",
       "1  4  10.0\n",
       "2  5   NaN\n",
       "3  6  12.0"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Drop columns\n",
    "df.drop(columns=['b'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "6020cd47",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop(columns=['b'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "6eb5b868",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>c</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>12.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   a     c\n",
       "1  3   9.0\n",
       "1  4  10.0\n",
       "2  5   NaN\n",
       "3  6  12.0"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "7613cb26",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>e</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>9.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>-9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>10.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>-9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>12.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>-9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   a     c     d  e\n",
       "1  3   9.0  12.0 -9\n",
       "1  4  10.0  14.0 -9\n",
       "2  5   NaN   NaN -9\n",
       "3  6  12.0  18.0 -9"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['d'] = df['a'] + df['c']\n",
    "df['e'] = -9\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4814c980",
   "metadata": {},
   "source": [
    "# An Example: US Treasury Bond Interest Rate"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "32ff72e1",
   "metadata": {},
   "source": [
    "Download from\n",
    "https://www.federalreserve.gov/releases/H15/default.htm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "352f0395",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Series Description</th>\n",
       "      <th>Market yield on U.S. Treasury securities at 1-month   constant maturity, quoted on investment basis</th>\n",
       "      <th>Market yield on U.S. Treasury securities at 3-month   constant maturity, quoted on investment basis</th>\n",
       "      <th>Market yield on U.S. Treasury securities at 6-month   constant maturity, quoted on investment basis</th>\n",
       "      <th>Market yield on U.S. Treasury securities at 1-year   constant maturity, quoted on investment basis</th>\n",
       "      <th>Market yield on U.S. Treasury securities at 2-year   constant maturity, quoted on investment basis</th>\n",
       "      <th>Market yield on U.S. Treasury securities at 3-year   constant maturity, quoted on investment basis</th>\n",
       "      <th>Market yield on U.S. Treasury securities at 5-year   constant maturity, quoted on investment basis</th>\n",
       "      <th>Market yield on U.S. Treasury securities at 7-year   constant maturity, quoted on investment basis</th>\n",
       "      <th>Market yield on U.S. Treasury securities at 10-year   constant maturity, quoted on investment basis</th>\n",
       "      <th>Market yield on U.S. Treasury securities at 20-year   constant maturity, quoted on investment basis</th>\n",
       "      <th>Market yield on U.S. Treasury securities at 30-year   constant maturity, quoted on investment basis</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Unit:</td>\n",
       "      <td>Percent:_Per_Year</td>\n",
       "      <td>Percent:_Per_Year</td>\n",
       "      <td>Percent:_Per_Year</td>\n",
       "      <td>Percent:_Per_Year</td>\n",
       "      <td>Percent:_Per_Year</td>\n",
       "      <td>Percent:_Per_Year</td>\n",
       "      <td>Percent:_Per_Year</td>\n",
       "      <td>Percent:_Per_Year</td>\n",
       "      <td>Percent:_Per_Year</td>\n",
       "      <td>Percent:_Per_Year</td>\n",
       "      <td>Percent:_Per_Year</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Multiplier:</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Currency:</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Unique Identifier:</td>\n",
       "      <td>H15/H15/RIFLGFCM01_N.M</td>\n",
       "      <td>H15/H15/RIFLGFCM03_N.M</td>\n",
       "      <td>H15/H15/RIFLGFCM06_N.M</td>\n",
       "      <td>H15/H15/RIFLGFCY01_N.M</td>\n",
       "      <td>H15/H15/RIFLGFCY02_N.M</td>\n",
       "      <td>H15/H15/RIFLGFCY03_N.M</td>\n",
       "      <td>H15/H15/RIFLGFCY05_N.M</td>\n",
       "      <td>H15/H15/RIFLGFCY07_N.M</td>\n",
       "      <td>H15/H15/RIFLGFCY10_N.M</td>\n",
       "      <td>H15/H15/RIFLGFCY20_N.M</td>\n",
       "      <td>H15/H15/RIFLGFCY30_N.M</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Time Period</td>\n",
       "      <td>RIFLGFCM01_N.M</td>\n",
       "      <td>RIFLGFCM03_N.M</td>\n",
       "      <td>RIFLGFCM06_N.M</td>\n",
       "      <td>RIFLGFCY01_N.M</td>\n",
       "      <td>RIFLGFCY02_N.M</td>\n",
       "      <td>RIFLGFCY03_N.M</td>\n",
       "      <td>RIFLGFCY05_N.M</td>\n",
       "      <td>RIFLGFCY07_N.M</td>\n",
       "      <td>RIFLGFCY10_N.M</td>\n",
       "      <td>RIFLGFCY20_N.M</td>\n",
       "      <td>RIFLGFCY30_N.M</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Series Description  \\\n",
       "0                Unit:   \n",
       "1          Multiplier:   \n",
       "2            Currency:   \n",
       "3  Unique Identifier:    \n",
       "4          Time Period   \n",
       "\n",
       "  Market yield on U.S. Treasury securities at 1-month   constant maturity, quoted on investment basis  \\\n",
       "0                                  Percent:_Per_Year                                                    \n",
       "1                                                  1                                                    \n",
       "2                                                NaN                                                    \n",
       "3                             H15/H15/RIFLGFCM01_N.M                                                    \n",
       "4                                     RIFLGFCM01_N.M                                                    \n",
       "\n",
       "  Market yield on U.S. Treasury securities at 3-month   constant maturity, quoted on investment basis  \\\n",
       "0                                  Percent:_Per_Year                                                    \n",
       "1                                                  1                                                    \n",
       "2                                                NaN                                                    \n",
       "3                             H15/H15/RIFLGFCM03_N.M                                                    \n",
       "4                                     RIFLGFCM03_N.M                                                    \n",
       "\n",
       "  Market yield on U.S. Treasury securities at 6-month   constant maturity, quoted on investment basis  \\\n",
       "0                                  Percent:_Per_Year                                                    \n",
       "1                                                  1                                                    \n",
       "2                                                NaN                                                    \n",
       "3                             H15/H15/RIFLGFCM06_N.M                                                    \n",
       "4                                     RIFLGFCM06_N.M                                                    \n",
       "\n",
       "  Market yield on U.S. Treasury securities at 1-year   constant maturity, quoted on investment basis  \\\n",
       "0                                  Percent:_Per_Year                                                   \n",
       "1                                                  1                                                   \n",
       "2                                                NaN                                                   \n",
       "3                             H15/H15/RIFLGFCY01_N.M                                                   \n",
       "4                                     RIFLGFCY01_N.M                                                   \n",
       "\n",
       "  Market yield on U.S. Treasury securities at 2-year   constant maturity, quoted on investment basis  \\\n",
       "0                                  Percent:_Per_Year                                                   \n",
       "1                                                  1                                                   \n",
       "2                                                NaN                                                   \n",
       "3                             H15/H15/RIFLGFCY02_N.M                                                   \n",
       "4                                     RIFLGFCY02_N.M                                                   \n",
       "\n",
       "  Market yield on U.S. Treasury securities at 3-year   constant maturity, quoted on investment basis  \\\n",
       "0                                  Percent:_Per_Year                                                   \n",
       "1                                                  1                                                   \n",
       "2                                                NaN                                                   \n",
       "3                             H15/H15/RIFLGFCY03_N.M                                                   \n",
       "4                                     RIFLGFCY03_N.M                                                   \n",
       "\n",
       "  Market yield on U.S. Treasury securities at 5-year   constant maturity, quoted on investment basis  \\\n",
       "0                                  Percent:_Per_Year                                                   \n",
       "1                                                  1                                                   \n",
       "2                                                NaN                                                   \n",
       "3                             H15/H15/RIFLGFCY05_N.M                                                   \n",
       "4                                     RIFLGFCY05_N.M                                                   \n",
       "\n",
       "  Market yield on U.S. Treasury securities at 7-year   constant maturity, quoted on investment basis  \\\n",
       "0                                  Percent:_Per_Year                                                   \n",
       "1                                                  1                                                   \n",
       "2                                                NaN                                                   \n",
       "3                             H15/H15/RIFLGFCY07_N.M                                                   \n",
       "4                                     RIFLGFCY07_N.M                                                   \n",
       "\n",
       "  Market yield on U.S. Treasury securities at 10-year   constant maturity, quoted on investment basis  \\\n",
       "0                                  Percent:_Per_Year                                                    \n",
       "1                                                  1                                                    \n",
       "2                                                NaN                                                    \n",
       "3                             H15/H15/RIFLGFCY10_N.M                                                    \n",
       "4                                     RIFLGFCY10_N.M                                                    \n",
       "\n",
       "  Market yield on U.S. Treasury securities at 20-year   constant maturity, quoted on investment basis  \\\n",
       "0                                  Percent:_Per_Year                                                    \n",
       "1                                                  1                                                    \n",
       "2                                                NaN                                                    \n",
       "3                             H15/H15/RIFLGFCY20_N.M                                                    \n",
       "4                                     RIFLGFCY20_N.M                                                    \n",
       "\n",
       "  Market yield on U.S. Treasury securities at 30-year   constant maturity, quoted on investment basis  \n",
       "0                                  Percent:_Per_Year                                                   \n",
       "1                                                  1                                                   \n",
       "2                                                NaN                                                   \n",
       "3                             H15/H15/RIFLGFCY30_N.M                                                   \n",
       "4                                     RIFLGFCY30_N.M                                                   "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Do experiments with many options in the beginning\n",
    "df = pd.read_csv('FRB_H15m.csv', nrows=10)  # only first 10 rows\n",
    "df.head()  #df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "6080c827",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('FRB_H15m.csv', \n",
    "                 skiprows=5, \n",
    "                 parse_dates=['Time Period'], \n",
    "                 index_col='Time Period')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "395777a6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>RIFLGFCM01_N.M</th>\n",
       "      <th>RIFLGFCM03_N.M</th>\n",
       "      <th>RIFLGFCM06_N.M</th>\n",
       "      <th>RIFLGFCY01_N.M</th>\n",
       "      <th>RIFLGFCY02_N.M</th>\n",
       "      <th>RIFLGFCY03_N.M</th>\n",
       "      <th>RIFLGFCY05_N.M</th>\n",
       "      <th>RIFLGFCY07_N.M</th>\n",
       "      <th>RIFLGFCY10_N.M</th>\n",
       "      <th>RIFLGFCY20_N.M</th>\n",
       "      <th>RIFLGFCY30_N.M</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Time Period</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1961-01-01</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.81</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.39</td>\n",
       "      <td>3.67</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.84</td>\n",
       "      <td>3.90</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1961-02-01</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.93</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.46</td>\n",
       "      <td>3.66</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.78</td>\n",
       "      <td>3.84</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1961-03-01</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.88</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.35</td>\n",
       "      <td>3.60</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.74</td>\n",
       "      <td>3.81</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1961-04-01</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.88</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.33</td>\n",
       "      <td>3.57</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.78</td>\n",
       "      <td>3.81</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1961-05-01</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.87</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.23</td>\n",
       "      <td>3.47</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.71</td>\n",
       "      <td>3.74</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             RIFLGFCM01_N.M  RIFLGFCM03_N.M  RIFLGFCM06_N.M  RIFLGFCY01_N.M  \\\n",
       "Time Period                                                                   \n",
       "1961-01-01              NaN             NaN             NaN            2.81   \n",
       "1961-02-01              NaN             NaN             NaN            2.93   \n",
       "1961-03-01              NaN             NaN             NaN            2.88   \n",
       "1961-04-01              NaN             NaN             NaN            2.88   \n",
       "1961-05-01              NaN             NaN             NaN            2.87   \n",
       "\n",
       "             RIFLGFCY02_N.M  RIFLGFCY03_N.M  RIFLGFCY05_N.M  RIFLGFCY07_N.M  \\\n",
       "Time Period                                                                   \n",
       "1961-01-01              NaN            3.39            3.67             NaN   \n",
       "1961-02-01              NaN            3.46            3.66             NaN   \n",
       "1961-03-01              NaN            3.35            3.60             NaN   \n",
       "1961-04-01              NaN            3.33            3.57             NaN   \n",
       "1961-05-01              NaN            3.23            3.47             NaN   \n",
       "\n",
       "             RIFLGFCY10_N.M  RIFLGFCY20_N.M  RIFLGFCY30_N.M  \n",
       "Time Period                                                  \n",
       "1961-01-01             3.84            3.90             NaN  \n",
       "1961-02-01             3.78            3.84             NaN  \n",
       "1961-03-01             3.74            3.81             NaN  \n",
       "1961-04-01             3.78            3.81             NaN  \n",
       "1961-05-01             3.71            3.74             NaN  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "4eeef1a2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['RIFLGFCM01_N.M', 'RIFLGFCM03_N.M', 'RIFLGFCM06_N.M', 'RIFLGFCY01_N.M',\n",
       "       'RIFLGFCY02_N.M', 'RIFLGFCY03_N.M', 'RIFLGFCY05_N.M', 'RIFLGFCY07_N.M',\n",
       "       'RIFLGFCY10_N.M', 'RIFLGFCY20_N.M', 'RIFLGFCY30_N.M'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "331448b3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Time Period'"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.index.name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "d4d0eef5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df.rename(columns={'Time Period': 'month'}, inplace=True)\n",
    "df.columns = ['1m', '3m', '6m', '1y', '2y', '3y', '5y', '7y', '10y', '20y', '30y']\n",
    "df.index.name = 'month'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "5fe572d0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1m</th>\n",
       "      <th>3m</th>\n",
       "      <th>6m</th>\n",
       "      <th>1y</th>\n",
       "      <th>2y</th>\n",
       "      <th>3y</th>\n",
       "      <th>5y</th>\n",
       "      <th>7y</th>\n",
       "      <th>10y</th>\n",
       "      <th>20y</th>\n",
       "      <th>30y</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1961-01-01</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.81</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.39</td>\n",
       "      <td>3.67</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.84</td>\n",
       "      <td>3.90</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1961-02-01</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.93</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.46</td>\n",
       "      <td>3.66</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.78</td>\n",
       "      <td>3.84</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1961-03-01</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.88</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.35</td>\n",
       "      <td>3.60</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.74</td>\n",
       "      <td>3.81</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1961-04-01</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.88</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.33</td>\n",
       "      <td>3.57</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.78</td>\n",
       "      <td>3.81</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1961-05-01</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.87</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.23</td>\n",
       "      <td>3.47</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.71</td>\n",
       "      <td>3.74</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            1m  3m  6m    1y  2y    3y    5y  7y   10y   20y  30y\n",
       "month                                                            \n",
       "1961-01-01 NaN NaN NaN  2.81 NaN  3.39  3.67 NaN  3.84  3.90  NaN\n",
       "1961-02-01 NaN NaN NaN  2.93 NaN  3.46  3.66 NaN  3.78  3.84  NaN\n",
       "1961-03-01 NaN NaN NaN  2.88 NaN  3.35  3.60 NaN  3.74  3.81  NaN\n",
       "1961-04-01 NaN NaN NaN  2.88 NaN  3.33  3.57 NaN  3.78  3.81  NaN\n",
       "1961-05-01 NaN NaN NaN  2.87 NaN  3.23  3.47 NaN  3.71  3.74  NaN"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "0bd34230",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1m     486\n",
       "3m     248\n",
       "6m     248\n",
       "1y       0\n",
       "2y     185\n",
       "3y       0\n",
       "5y       0\n",
       "7y     102\n",
       "10y      0\n",
       "20y      0\n",
       "30y    193\n",
       "dtype: int64"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isna().sum()  #df['1y'].isna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "0dfc21a0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1m</th>\n",
       "      <th>3m</th>\n",
       "      <th>6m</th>\n",
       "      <th>1y</th>\n",
       "      <th>2y</th>\n",
       "      <th>3y</th>\n",
       "      <th>5y</th>\n",
       "      <th>7y</th>\n",
       "      <th>10y</th>\n",
       "      <th>20y</th>\n",
       "      <th>30y</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1961-01-01</th>\n",
       "      <td>-9.00</td>\n",
       "      <td>-9.00</td>\n",
       "      <td>-9.00</td>\n",
       "      <td>2.81</td>\n",
       "      <td>-9.00</td>\n",
       "      <td>3.39</td>\n",
       "      <td>3.67</td>\n",
       "      <td>-9.00</td>\n",
       "      <td>3.84</td>\n",
       "      <td>3.90</td>\n",
       "      <td>-9.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1961-02-01</th>\n",
       "      <td>-9.00</td>\n",
       "      <td>-9.00</td>\n",
       "      <td>-9.00</td>\n",
       "      <td>2.93</td>\n",
       "      <td>-9.00</td>\n",
       "      <td>3.46</td>\n",
       "      <td>3.66</td>\n",
       "      <td>-9.00</td>\n",
       "      <td>3.78</td>\n",
       "      <td>3.84</td>\n",
       "      <td>-9.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1961-03-01</th>\n",
       "      <td>-9.00</td>\n",
       "      <td>-9.00</td>\n",
       "      <td>-9.00</td>\n",
       "      <td>2.88</td>\n",
       "      <td>-9.00</td>\n",
       "      <td>3.35</td>\n",
       "      <td>3.60</td>\n",
       "      <td>-9.00</td>\n",
       "      <td>3.74</td>\n",
       "      <td>3.81</td>\n",
       "      <td>-9.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1961-04-01</th>\n",
       "      <td>-9.00</td>\n",
       "      <td>-9.00</td>\n",
       "      <td>-9.00</td>\n",
       "      <td>2.88</td>\n",
       "      <td>-9.00</td>\n",
       "      <td>3.33</td>\n",
       "      <td>3.57</td>\n",
       "      <td>-9.00</td>\n",
       "      <td>3.78</td>\n",
       "      <td>3.81</td>\n",
       "      <td>-9.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1961-05-01</th>\n",
       "      <td>-9.00</td>\n",
       "      <td>-9.00</td>\n",
       "      <td>-9.00</td>\n",
       "      <td>2.87</td>\n",
       "      <td>-9.00</td>\n",
       "      <td>3.23</td>\n",
       "      <td>3.47</td>\n",
       "      <td>-9.00</td>\n",
       "      <td>3.71</td>\n",
       "      <td>3.74</td>\n",
       "      <td>-9.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-04-01</th>\n",
       "      <td>0.02</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.16</td>\n",
       "      <td>0.35</td>\n",
       "      <td>0.86</td>\n",
       "      <td>1.31</td>\n",
       "      <td>1.64</td>\n",
       "      <td>2.20</td>\n",
       "      <td>2.30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-05-01</th>\n",
       "      <td>0.01</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.16</td>\n",
       "      <td>0.32</td>\n",
       "      <td>0.82</td>\n",
       "      <td>1.28</td>\n",
       "      <td>1.62</td>\n",
       "      <td>2.22</td>\n",
       "      <td>2.32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-06-01</th>\n",
       "      <td>0.03</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.39</td>\n",
       "      <td>0.84</td>\n",
       "      <td>1.23</td>\n",
       "      <td>1.52</td>\n",
       "      <td>2.09</td>\n",
       "      <td>2.16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-07-01</th>\n",
       "      <td>0.05</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.22</td>\n",
       "      <td>0.40</td>\n",
       "      <td>0.76</td>\n",
       "      <td>1.07</td>\n",
       "      <td>1.32</td>\n",
       "      <td>1.87</td>\n",
       "      <td>1.94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-01</th>\n",
       "      <td>0.04</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.22</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.77</td>\n",
       "      <td>1.06</td>\n",
       "      <td>1.28</td>\n",
       "      <td>1.83</td>\n",
       "      <td>1.92</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>728 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              1m    3m    6m    1y    2y    3y    5y    7y   10y   20y   30y\n",
       "month                                                                       \n",
       "1961-01-01 -9.00 -9.00 -9.00  2.81 -9.00  3.39  3.67 -9.00  3.84  3.90 -9.00\n",
       "1961-02-01 -9.00 -9.00 -9.00  2.93 -9.00  3.46  3.66 -9.00  3.78  3.84 -9.00\n",
       "1961-03-01 -9.00 -9.00 -9.00  2.88 -9.00  3.35  3.60 -9.00  3.74  3.81 -9.00\n",
       "1961-04-01 -9.00 -9.00 -9.00  2.88 -9.00  3.33  3.57 -9.00  3.78  3.81 -9.00\n",
       "1961-05-01 -9.00 -9.00 -9.00  2.87 -9.00  3.23  3.47 -9.00  3.71  3.74 -9.00\n",
       "...          ...   ...   ...   ...   ...   ...   ...   ...   ...   ...   ...\n",
       "2021-04-01  0.02  0.02  0.04  0.06  0.16  0.35  0.86  1.31  1.64  2.20  2.30\n",
       "2021-05-01  0.01  0.02  0.04  0.05  0.16  0.32  0.82  1.28  1.62  2.22  2.32\n",
       "2021-06-01  0.03  0.04  0.05  0.07  0.20  0.39  0.84  1.23  1.52  2.09  2.16\n",
       "2021-07-01  0.05  0.05  0.05  0.08  0.22  0.40  0.76  1.07  1.32  1.87  1.94\n",
       "2021-08-01  0.04  0.05  0.06  0.07  0.22  0.42  0.77  1.06  1.28  1.83  1.92\n",
       "\n",
       "[728 rows x 11 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.fillna(-9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "8a6a9c8b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1y</th>\n",
       "      <th>3y</th>\n",
       "      <th>5y</th>\n",
       "      <th>10y</th>\n",
       "      <th>20y</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1961-01-01</th>\n",
       "      <td>2.81</td>\n",
       "      <td>3.39</td>\n",
       "      <td>3.67</td>\n",
       "      <td>3.84</td>\n",
       "      <td>3.90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1961-02-01</th>\n",
       "      <td>2.93</td>\n",
       "      <td>3.46</td>\n",
       "      <td>3.66</td>\n",
       "      <td>3.78</td>\n",
       "      <td>3.84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1961-03-01</th>\n",
       "      <td>2.88</td>\n",
       "      <td>3.35</td>\n",
       "      <td>3.60</td>\n",
       "      <td>3.74</td>\n",
       "      <td>3.81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1961-04-01</th>\n",
       "      <td>2.88</td>\n",
       "      <td>3.33</td>\n",
       "      <td>3.57</td>\n",
       "      <td>3.78</td>\n",
       "      <td>3.81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1961-05-01</th>\n",
       "      <td>2.87</td>\n",
       "      <td>3.23</td>\n",
       "      <td>3.47</td>\n",
       "      <td>3.71</td>\n",
       "      <td>3.74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-04-01</th>\n",
       "      <td>0.06</td>\n",
       "      <td>0.35</td>\n",
       "      <td>0.86</td>\n",
       "      <td>1.64</td>\n",
       "      <td>2.20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-05-01</th>\n",
       "      <td>0.05</td>\n",
       "      <td>0.32</td>\n",
       "      <td>0.82</td>\n",
       "      <td>1.62</td>\n",
       "      <td>2.22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-06-01</th>\n",
       "      <td>0.07</td>\n",
       "      <td>0.39</td>\n",
       "      <td>0.84</td>\n",
       "      <td>1.52</td>\n",
       "      <td>2.09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-07-01</th>\n",
       "      <td>0.08</td>\n",
       "      <td>0.40</td>\n",
       "      <td>0.76</td>\n",
       "      <td>1.32</td>\n",
       "      <td>1.87</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-01</th>\n",
       "      <td>0.07</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.77</td>\n",
       "      <td>1.28</td>\n",
       "      <td>1.83</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>728 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              1y    3y    5y   10y   20y\n",
       "month                                   \n",
       "1961-01-01  2.81  3.39  3.67  3.84  3.90\n",
       "1961-02-01  2.93  3.46  3.66  3.78  3.84\n",
       "1961-03-01  2.88  3.35  3.60  3.74  3.81\n",
       "1961-04-01  2.88  3.33  3.57  3.78  3.81\n",
       "1961-05-01  2.87  3.23  3.47  3.71  3.74\n",
       "...          ...   ...   ...   ...   ...\n",
       "2021-04-01  0.06  0.35  0.86  1.64  2.20\n",
       "2021-05-01  0.05  0.32  0.82  1.62  2.22\n",
       "2021-06-01  0.07  0.39  0.84  1.52  2.09\n",
       "2021-07-01  0.08  0.40  0.76  1.32  1.87\n",
       "2021-08-01  0.07  0.42  0.77  1.28  1.83\n",
       "\n",
       "[728 rows x 5 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dropna(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "192f303a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1m</th>\n",
       "      <th>3m</th>\n",
       "      <th>6m</th>\n",
       "      <th>1y</th>\n",
       "      <th>2y</th>\n",
       "      <th>3y</th>\n",
       "      <th>5y</th>\n",
       "      <th>7y</th>\n",
       "      <th>10y</th>\n",
       "      <th>20y</th>\n",
       "      <th>30y</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1976-06-01</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.52</td>\n",
       "      <td>7.06</td>\n",
       "      <td>7.31</td>\n",
       "      <td>7.61</td>\n",
       "      <td>7.75</td>\n",
       "      <td>7.86</td>\n",
       "      <td>8.03</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1976-07-01</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.20</td>\n",
       "      <td>6.85</td>\n",
       "      <td>7.12</td>\n",
       "      <td>7.49</td>\n",
       "      <td>7.70</td>\n",
       "      <td>7.83</td>\n",
       "      <td>8.00</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1976-08-01</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.00</td>\n",
       "      <td>6.63</td>\n",
       "      <td>6.86</td>\n",
       "      <td>7.31</td>\n",
       "      <td>7.58</td>\n",
       "      <td>7.77</td>\n",
       "      <td>7.91</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1976-09-01</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.84</td>\n",
       "      <td>6.42</td>\n",
       "      <td>6.66</td>\n",
       "      <td>7.13</td>\n",
       "      <td>7.41</td>\n",
       "      <td>7.59</td>\n",
       "      <td>7.78</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1976-10-01</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.50</td>\n",
       "      <td>5.98</td>\n",
       "      <td>6.24</td>\n",
       "      <td>6.75</td>\n",
       "      <td>7.16</td>\n",
       "      <td>7.41</td>\n",
       "      <td>7.70</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-04-01</th>\n",
       "      <td>0.02</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.16</td>\n",
       "      <td>0.35</td>\n",
       "      <td>0.86</td>\n",
       "      <td>1.31</td>\n",
       "      <td>1.64</td>\n",
       "      <td>2.20</td>\n",
       "      <td>2.30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-05-01</th>\n",
       "      <td>0.01</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.16</td>\n",
       "      <td>0.32</td>\n",
       "      <td>0.82</td>\n",
       "      <td>1.28</td>\n",
       "      <td>1.62</td>\n",
       "      <td>2.22</td>\n",
       "      <td>2.32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-06-01</th>\n",
       "      <td>0.03</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.39</td>\n",
       "      <td>0.84</td>\n",
       "      <td>1.23</td>\n",
       "      <td>1.52</td>\n",
       "      <td>2.09</td>\n",
       "      <td>2.16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-07-01</th>\n",
       "      <td>0.05</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.22</td>\n",
       "      <td>0.40</td>\n",
       "      <td>0.76</td>\n",
       "      <td>1.07</td>\n",
       "      <td>1.32</td>\n",
       "      <td>1.87</td>\n",
       "      <td>1.94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-01</th>\n",
       "      <td>0.04</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.22</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.77</td>\n",
       "      <td>1.06</td>\n",
       "      <td>1.28</td>\n",
       "      <td>1.83</td>\n",
       "      <td>1.92</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>543 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              1m    3m    6m    1y    2y    3y    5y    7y   10y   20y   30y\n",
       "month                                                                       \n",
       "1976-06-01   NaN   NaN   NaN  6.52  7.06  7.31  7.61  7.75  7.86  8.03   NaN\n",
       "1976-07-01   NaN   NaN   NaN  6.20  6.85  7.12  7.49  7.70  7.83  8.00   NaN\n",
       "1976-08-01   NaN   NaN   NaN  6.00  6.63  6.86  7.31  7.58  7.77  7.91   NaN\n",
       "1976-09-01   NaN   NaN   NaN  5.84  6.42  6.66  7.13  7.41  7.59  7.78   NaN\n",
       "1976-10-01   NaN   NaN   NaN  5.50  5.98  6.24  6.75  7.16  7.41  7.70   NaN\n",
       "...          ...   ...   ...   ...   ...   ...   ...   ...   ...   ...   ...\n",
       "2021-04-01  0.02  0.02  0.04  0.06  0.16  0.35  0.86  1.31  1.64  2.20  2.30\n",
       "2021-05-01  0.01  0.02  0.04  0.05  0.16  0.32  0.82  1.28  1.62  2.22  2.32\n",
       "2021-06-01  0.03  0.04  0.05  0.07  0.20  0.39  0.84  1.23  1.52  2.09  2.16\n",
       "2021-07-01  0.05  0.05  0.05  0.08  0.22  0.40  0.76  1.07  1.32  1.87  1.94\n",
       "2021-08-01  0.04  0.05  0.06  0.07  0.22  0.42  0.77  1.06  1.28  1.83  1.92\n",
       "\n",
       "[543 rows x 11 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dropna(axis=0, thresh=7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "9fec067a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1m     1.217810\n",
       "3m     3.828000\n",
       "6m     4.008854\n",
       "1y     4.932637\n",
       "2y     5.096133\n",
       "3y     5.344918\n",
       "5y     5.608173\n",
       "7y     6.005527\n",
       "10y    5.945110\n",
       "20y    6.207157\n",
       "30y    6.395757\n",
       "dtype: float64"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get the column mean\n",
    "mean = df.mean(axis=0)\n",
    "mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "230725a8",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#df.fillna(mean)\n",
    "df.fillna(mean, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "cdaf7b85",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.21780992, 3.828     , 4.00885417, ..., 3.84      , 3.9       ,\n",
       "        6.39575701],\n",
       "       [1.21780992, 3.828     , 4.00885417, ..., 3.78      , 3.84      ,\n",
       "        6.39575701],\n",
       "       [1.21780992, 3.828     , 4.00885417, ..., 3.74      , 3.81      ,\n",
       "        6.39575701],\n",
       "       ...,\n",
       "       [0.03      , 0.04      , 0.05      , ..., 1.52      , 2.09      ,\n",
       "        2.16      ],\n",
       "       [0.05      , 0.05      , 0.05      , ..., 1.32      , 1.87      ,\n",
       "        1.94      ],\n",
       "       [0.04      , 0.05      , 0.06      , ..., 1.28      , 1.83      ,\n",
       "        1.92      ]])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.values  #numpy matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b616f6f",
   "metadata": {},
   "source": [
    "## Save to Feather file (Need installing feather packages)\n",
    "\n",
    "* Feather file format provides very fast read/write\n",
    "* See https://towardsdatascience.com/the-best-format-to-save-pandas-data-414dca023e0d\n",
    "* Alternative choices are `HDF5` or `pickle` format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "ac403345",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[df.index >= '2000-01']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "3a57f2ac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1m</th>\n",
       "      <th>3m</th>\n",
       "      <th>6m</th>\n",
       "      <th>1y</th>\n",
       "      <th>2y</th>\n",
       "      <th>3y</th>\n",
       "      <th>5y</th>\n",
       "      <th>7y</th>\n",
       "      <th>10y</th>\n",
       "      <th>20y</th>\n",
       "      <th>30y</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2000-01-01</th>\n",
       "      <td>1.21781</td>\n",
       "      <td>5.50</td>\n",
       "      <td>5.76</td>\n",
       "      <td>6.12</td>\n",
       "      <td>6.44</td>\n",
       "      <td>6.49</td>\n",
       "      <td>6.58</td>\n",
       "      <td>6.70</td>\n",
       "      <td>6.66</td>\n",
       "      <td>6.86</td>\n",
       "      <td>6.63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2000-02-01</th>\n",
       "      <td>1.21781</td>\n",
       "      <td>5.73</td>\n",
       "      <td>6.00</td>\n",
       "      <td>6.22</td>\n",
       "      <td>6.61</td>\n",
       "      <td>6.65</td>\n",
       "      <td>6.68</td>\n",
       "      <td>6.72</td>\n",
       "      <td>6.52</td>\n",
       "      <td>6.54</td>\n",
       "      <td>6.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2000-03-01</th>\n",
       "      <td>1.21781</td>\n",
       "      <td>5.86</td>\n",
       "      <td>6.11</td>\n",
       "      <td>6.22</td>\n",
       "      <td>6.53</td>\n",
       "      <td>6.53</td>\n",
       "      <td>6.50</td>\n",
       "      <td>6.51</td>\n",
       "      <td>6.26</td>\n",
       "      <td>6.38</td>\n",
       "      <td>6.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2000-04-01</th>\n",
       "      <td>1.21781</td>\n",
       "      <td>5.82</td>\n",
       "      <td>6.07</td>\n",
       "      <td>6.15</td>\n",
       "      <td>6.40</td>\n",
       "      <td>6.36</td>\n",
       "      <td>6.26</td>\n",
       "      <td>6.27</td>\n",
       "      <td>5.99</td>\n",
       "      <td>6.18</td>\n",
       "      <td>5.85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2000-05-01</th>\n",
       "      <td>1.21781</td>\n",
       "      <td>5.99</td>\n",
       "      <td>6.39</td>\n",
       "      <td>6.33</td>\n",
       "      <td>6.81</td>\n",
       "      <td>6.77</td>\n",
       "      <td>6.69</td>\n",
       "      <td>6.69</td>\n",
       "      <td>6.44</td>\n",
       "      <td>6.55</td>\n",
       "      <td>6.15</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 1m    3m    6m    1y    2y    3y    5y    7y   10y   20y  \\\n",
       "month                                                                       \n",
       "2000-01-01  1.21781  5.50  5.76  6.12  6.44  6.49  6.58  6.70  6.66  6.86   \n",
       "2000-02-01  1.21781  5.73  6.00  6.22  6.61  6.65  6.68  6.72  6.52  6.54   \n",
       "2000-03-01  1.21781  5.86  6.11  6.22  6.53  6.53  6.50  6.51  6.26  6.38   \n",
       "2000-04-01  1.21781  5.82  6.07  6.15  6.40  6.36  6.26  6.27  5.99  6.18   \n",
       "2000-05-01  1.21781  5.99  6.39  6.33  6.81  6.77  6.69  6.69  6.44  6.55   \n",
       "\n",
       "             30y  \n",
       "month             \n",
       "2000-01-01  6.63  \n",
       "2000-02-01  6.23  \n",
       "2000-03-01  6.05  \n",
       "2000-04-01  5.85  \n",
       "2000-05-01  6.15  "
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "2157fcbe",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>month</th>\n",
       "      <th>1m</th>\n",
       "      <th>3m</th>\n",
       "      <th>6m</th>\n",
       "      <th>1y</th>\n",
       "      <th>2y</th>\n",
       "      <th>3y</th>\n",
       "      <th>5y</th>\n",
       "      <th>7y</th>\n",
       "      <th>10y</th>\n",
       "      <th>20y</th>\n",
       "      <th>30y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>255</th>\n",
       "      <td>2021-04-01</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.16</td>\n",
       "      <td>0.35</td>\n",
       "      <td>0.86</td>\n",
       "      <td>1.31</td>\n",
       "      <td>1.64</td>\n",
       "      <td>2.20</td>\n",
       "      <td>2.30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>256</th>\n",
       "      <td>2021-05-01</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.16</td>\n",
       "      <td>0.32</td>\n",
       "      <td>0.82</td>\n",
       "      <td>1.28</td>\n",
       "      <td>1.62</td>\n",
       "      <td>2.22</td>\n",
       "      <td>2.32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257</th>\n",
       "      <td>2021-06-01</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.39</td>\n",
       "      <td>0.84</td>\n",
       "      <td>1.23</td>\n",
       "      <td>1.52</td>\n",
       "      <td>2.09</td>\n",
       "      <td>2.16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>258</th>\n",
       "      <td>2021-07-01</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.22</td>\n",
       "      <td>0.40</td>\n",
       "      <td>0.76</td>\n",
       "      <td>1.07</td>\n",
       "      <td>1.32</td>\n",
       "      <td>1.87</td>\n",
       "      <td>1.94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>259</th>\n",
       "      <td>2021-08-01</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.22</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.77</td>\n",
       "      <td>1.06</td>\n",
       "      <td>1.28</td>\n",
       "      <td>1.83</td>\n",
       "      <td>1.92</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         month    1m    3m    6m    1y    2y    3y    5y    7y   10y   20y  \\\n",
       "255 2021-04-01  0.02  0.02  0.04  0.06  0.16  0.35  0.86  1.31  1.64  2.20   \n",
       "256 2021-05-01  0.01  0.02  0.04  0.05  0.16  0.32  0.82  1.28  1.62  2.22   \n",
       "257 2021-06-01  0.03  0.04  0.05  0.07  0.20  0.39  0.84  1.23  1.52  2.09   \n",
       "258 2021-07-01  0.05  0.05  0.05  0.08  0.22  0.40  0.76  1.07  1.32  1.87   \n",
       "259 2021-08-01  0.04  0.05  0.06  0.07  0.22  0.42  0.77  1.06  1.28  1.83   \n",
       "\n",
       "      30y  \n",
       "255  2.30  \n",
       "256  2.32  \n",
       "257  2.16  \n",
       "258  1.94  \n",
       "259  1.92  "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.reset_index(inplace=True)\n",
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "bab72ac1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the procedssed dataframe into a feather file\n",
    "df.to_feather('FRB_H15m.feather')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "350b75a2",
   "metadata": {},
   "source": [
    "## Read from Feather"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "6121cc27",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1m</th>\n",
       "      <th>3m</th>\n",
       "      <th>6m</th>\n",
       "      <th>1y</th>\n",
       "      <th>2y</th>\n",
       "      <th>3y</th>\n",
       "      <th>5y</th>\n",
       "      <th>7y</th>\n",
       "      <th>10y</th>\n",
       "      <th>20y</th>\n",
       "      <th>30y</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2021-04-01</th>\n",
       "      <td>0.02</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.16</td>\n",
       "      <td>0.35</td>\n",
       "      <td>0.86</td>\n",
       "      <td>1.31</td>\n",
       "      <td>1.64</td>\n",
       "      <td>2.20</td>\n",
       "      <td>2.30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-05-01</th>\n",
       "      <td>0.01</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.16</td>\n",
       "      <td>0.32</td>\n",
       "      <td>0.82</td>\n",
       "      <td>1.28</td>\n",
       "      <td>1.62</td>\n",
       "      <td>2.22</td>\n",
       "      <td>2.32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-06-01</th>\n",
       "      <td>0.03</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.39</td>\n",
       "      <td>0.84</td>\n",
       "      <td>1.23</td>\n",
       "      <td>1.52</td>\n",
       "      <td>2.09</td>\n",
       "      <td>2.16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-07-01</th>\n",
       "      <td>0.05</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.22</td>\n",
       "      <td>0.40</td>\n",
       "      <td>0.76</td>\n",
       "      <td>1.07</td>\n",
       "      <td>1.32</td>\n",
       "      <td>1.87</td>\n",
       "      <td>1.94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-01</th>\n",
       "      <td>0.04</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.22</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.77</td>\n",
       "      <td>1.06</td>\n",
       "      <td>1.28</td>\n",
       "      <td>1.83</td>\n",
       "      <td>1.92</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              1m    3m    6m    1y    2y    3y    5y    7y   10y   20y   30y\n",
       "month                                                                       \n",
       "2021-04-01  0.02  0.02  0.04  0.06  0.16  0.35  0.86  1.31  1.64  2.20  2.30\n",
       "2021-05-01  0.01  0.02  0.04  0.05  0.16  0.32  0.82  1.28  1.62  2.22  2.32\n",
       "2021-06-01  0.03  0.04  0.05  0.07  0.20  0.39  0.84  1.23  1.52  2.09  2.16\n",
       "2021-07-01  0.05  0.05  0.05  0.08  0.22  0.40  0.76  1.07  1.32  1.87  1.94\n",
       "2021-08-01  0.04  0.05  0.06  0.07  0.22  0.42  0.77  1.06  1.28  1.83  1.92"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_feather('FRB_H15m.feather')\n",
    "df.set_index('month', inplace=True)\n",
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "03f05266",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1m</th>\n",
       "      <th>3m</th>\n",
       "      <th>6m</th>\n",
       "      <th>1y</th>\n",
       "      <th>2y</th>\n",
       "      <th>3y</th>\n",
       "      <th>5y</th>\n",
       "      <th>7y</th>\n",
       "      <th>10y</th>\n",
       "      <th>20y</th>\n",
       "      <th>30y</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2000-01-01</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2000-02-01</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2000-03-01</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2000-04-01</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2000-05-01</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-04-01</th>\n",
       "      <td>0.077500</td>\n",
       "      <td>0.090000</td>\n",
       "      <td>0.104167</td>\n",
       "      <td>0.117500</td>\n",
       "      <td>0.150000</td>\n",
       "      <td>0.217500</td>\n",
       "      <td>0.440833</td>\n",
       "      <td>0.713333</td>\n",
       "      <td>0.960833</td>\n",
       "      <td>1.499167</td>\n",
       "      <td>1.693333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-05-01</th>\n",
       "      <td>0.070000</td>\n",
       "      <td>0.080833</td>\n",
       "      <td>0.095000</td>\n",
       "      <td>0.108333</td>\n",
       "      <td>0.149167</td>\n",
       "      <td>0.225833</td>\n",
       "      <td>0.480833</td>\n",
       "      <td>0.775833</td>\n",
       "      <td>1.040000</td>\n",
       "      <td>1.590833</td>\n",
       "      <td>1.771667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-06-01</th>\n",
       "      <td>0.061667</td>\n",
       "      <td>0.070833</td>\n",
       "      <td>0.084167</td>\n",
       "      <td>0.099167</td>\n",
       "      <td>0.150000</td>\n",
       "      <td>0.240000</td>\n",
       "      <td>0.522500</td>\n",
       "      <td>0.832500</td>\n",
       "      <td>1.105833</td>\n",
       "      <td>1.659167</td>\n",
       "      <td>1.827500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-07-01</th>\n",
       "      <td>0.056667</td>\n",
       "      <td>0.064167</td>\n",
       "      <td>0.076667</td>\n",
       "      <td>0.093333</td>\n",
       "      <td>0.155833</td>\n",
       "      <td>0.259167</td>\n",
       "      <td>0.562500</td>\n",
       "      <td>0.883333</td>\n",
       "      <td>1.164167</td>\n",
       "      <td>1.724167</td>\n",
       "      <td>1.880000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-01</th>\n",
       "      <td>0.053333</td>\n",
       "      <td>0.060000</td>\n",
       "      <td>0.071667</td>\n",
       "      <td>0.088333</td>\n",
       "      <td>0.162500</td>\n",
       "      <td>0.280833</td>\n",
       "      <td>0.604167</td>\n",
       "      <td>0.933333</td>\n",
       "      <td>1.216667</td>\n",
       "      <td>1.781667</td>\n",
       "      <td>1.926667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>260 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  1m        3m        6m        1y        2y        3y  \\\n",
       "month                                                                    \n",
       "2000-01-01       NaN       NaN       NaN       NaN       NaN       NaN   \n",
       "2000-02-01       NaN       NaN       NaN       NaN       NaN       NaN   \n",
       "2000-03-01       NaN       NaN       NaN       NaN       NaN       NaN   \n",
       "2000-04-01       NaN       NaN       NaN       NaN       NaN       NaN   \n",
       "2000-05-01       NaN       NaN       NaN       NaN       NaN       NaN   \n",
       "...              ...       ...       ...       ...       ...       ...   \n",
       "2021-04-01  0.077500  0.090000  0.104167  0.117500  0.150000  0.217500   \n",
       "2021-05-01  0.070000  0.080833  0.095000  0.108333  0.149167  0.225833   \n",
       "2021-06-01  0.061667  0.070833  0.084167  0.099167  0.150000  0.240000   \n",
       "2021-07-01  0.056667  0.064167  0.076667  0.093333  0.155833  0.259167   \n",
       "2021-08-01  0.053333  0.060000  0.071667  0.088333  0.162500  0.280833   \n",
       "\n",
       "                  5y        7y       10y       20y       30y  \n",
       "month                                                         \n",
       "2000-01-01       NaN       NaN       NaN       NaN       NaN  \n",
       "2000-02-01       NaN       NaN       NaN       NaN       NaN  \n",
       "2000-03-01       NaN       NaN       NaN       NaN       NaN  \n",
       "2000-04-01       NaN       NaN       NaN       NaN       NaN  \n",
       "2000-05-01       NaN       NaN       NaN       NaN       NaN  \n",
       "...              ...       ...       ...       ...       ...  \n",
       "2021-04-01  0.440833  0.713333  0.960833  1.499167  1.693333  \n",
       "2021-05-01  0.480833  0.775833  1.040000  1.590833  1.771667  \n",
       "2021-06-01  0.522500  0.832500  1.105833  1.659167  1.827500  \n",
       "2021-07-01  0.562500  0.883333  1.164167  1.724167  1.880000  \n",
       "2021-08-01  0.604167  0.933333  1.216667  1.781667  1.926667  \n",
       "\n",
       "[260 rows x 11 columns]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "### Rolling average\n",
    "df.rolling(12).mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3998f6ed",
   "metadata": {},
   "source": [
    "# Creating panel datda by Pivot"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e9b658cc",
   "metadata": {},
   "source": [
    "`ReturnMonthly2020.csv.gz` contains montly US stock return in 2020.\n",
    "\n",
    "* `PERMNO`: company identification number\n",
    "* `date`: end of the month\n",
    "* `SICCD`: industry classification\n",
    "* `VOL`: trade volume\n",
    "* `RET`: stock return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "6794a6cf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PERMNO</th>\n",
       "      <th>date</th>\n",
       "      <th>PRC</th>\n",
       "      <th>VOL</th>\n",
       "      <th>RET</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10026</td>\n",
       "      <td>20200131</td>\n",
       "      <td>165.84000</td>\n",
       "      <td>22433</td>\n",
       "      <td>-0.100016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10026</td>\n",
       "      <td>20200228</td>\n",
       "      <td>160.82001</td>\n",
       "      <td>18648</td>\n",
       "      <td>-0.030270</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10026</td>\n",
       "      <td>20200331</td>\n",
       "      <td>121.00000</td>\n",
       "      <td>39302</td>\n",
       "      <td>-0.244031</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PERMNO      date        PRC    VOL       RET\n",
       "0   10026  20200131  165.84000  22433 -0.100016\n",
       "1   10026  20200228  160.82001  18648 -0.030270\n",
       "2   10026  20200331  121.00000  39302 -0.244031"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# You can directly read compressed csv (.gz or .zip)\n",
    "df = pd.read_csv('ReturnMonthly2020.csv.gz', nrows=3)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "af3697b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\n",
    "    'ReturnMonthly2020.csv.gz', parse_dates = ['date'],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "a236d461",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PERMNO             int64\n",
       "date      datetime64[ns]\n",
       "PRC              float64\n",
       "VOL              float64\n",
       "RET               object\n",
       "dtype: object"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "a1720a0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_panel = df.pivot(index='date', columns='PERMNO', values='RET')\n",
    "#df_panel.sort_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "552aaf65",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>PERMNO</th>\n",
       "      <th>10026</th>\n",
       "      <th>10028</th>\n",
       "      <th>10032</th>\n",
       "      <th>10044</th>\n",
       "      <th>10051</th>\n",
       "      <th>10065</th>\n",
       "      <th>10104</th>\n",
       "      <th>10107</th>\n",
       "      <th>10113</th>\n",
       "      <th>10138</th>\n",
       "      <th>...</th>\n",
       "      <th>93421</th>\n",
       "      <th>93422</th>\n",
       "      <th>93423</th>\n",
       "      <th>93424</th>\n",
       "      <th>93425</th>\n",
       "      <th>93426</th>\n",
       "      <th>93427</th>\n",
       "      <th>93429</th>\n",
       "      <th>93434</th>\n",
       "      <th>93436</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2020-01-31</th>\n",
       "      <td>-0.100016</td>\n",
       "      <td>0.607407</td>\n",
       "      <td>-0.075643</td>\n",
       "      <td>-0.098592</td>\n",
       "      <td>-0.115176</td>\n",
       "      <td>0.005707</td>\n",
       "      <td>-0.005474</td>\n",
       "      <td>0.079455</td>\n",
       "      <td>-0.001206</td>\n",
       "      <td>0.095946</td>\n",
       "      <td>...</td>\n",
       "      <td>0.011010</td>\n",
       "      <td>-0.295556</td>\n",
       "      <td>-0.154733</td>\n",
       "      <td>-0.025676</td>\n",
       "      <td>-0.127578</td>\n",
       "      <td>0.015882</td>\n",
       "      <td>-0.027761</td>\n",
       "      <td>0.026833</td>\n",
       "      <td>0.023810</td>\n",
       "      <td>0.555160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-02-28</th>\n",
       "      <td>-0.030270</td>\n",
       "      <td>0.225806</td>\n",
       "      <td>-0.067070</td>\n",
       "      <td>-0.064904</td>\n",
       "      <td>-0.055669</td>\n",
       "      <td>-0.090164</td>\n",
       "      <td>-0.057007</td>\n",
       "      <td>-0.045292</td>\n",
       "      <td>-0.082666</td>\n",
       "      <td>-0.116229</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.020689</td>\n",
       "      <td>-0.290221</td>\n",
       "      <td>-0.337005</td>\n",
       "      <td>-0.045770</td>\n",
       "      <td>-0.110500</td>\n",
       "      <td>-0.204690</td>\n",
       "      <td>-0.125635</td>\n",
       "      <td>-0.071904</td>\n",
       "      <td>0.372093</td>\n",
       "      <td>0.026776</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-03-31</th>\n",
       "      <td>-0.244031</td>\n",
       "      <td>-0.052632</td>\n",
       "      <td>-0.177694</td>\n",
       "      <td>-0.373368</td>\n",
       "      <td>-0.324664</td>\n",
       "      <td>-0.124478</td>\n",
       "      <td>-0.022847</td>\n",
       "      <td>-0.026541</td>\n",
       "      <td>-0.182996</td>\n",
       "      <td>-0.164901</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.086212</td>\n",
       "      <td>-0.842444</td>\n",
       "      <td>-0.494066</td>\n",
       "      <td>-0.058140</td>\n",
       "      <td>-0.512361</td>\n",
       "      <td>-0.269021</td>\n",
       "      <td>-0.010160</td>\n",
       "      <td>-0.217105</td>\n",
       "      <td>-0.305085</td>\n",
       "      <td>-0.215557</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-04-30</th>\n",
       "      <td>0.049835</td>\n",
       "      <td>0.448413</td>\n",
       "      <td>0.149010</td>\n",
       "      <td>-0.052083</td>\n",
       "      <td>0.178434</td>\n",
       "      <td>0.125496</td>\n",
       "      <td>0.100972</td>\n",
       "      <td>0.136326</td>\n",
       "      <td>0.139936</td>\n",
       "      <td>0.184127</td>\n",
       "      <td>...</td>\n",
       "      <td>0.059810</td>\n",
       "      <td>1.947683</td>\n",
       "      <td>0.595694</td>\n",
       "      <td>-0.072531</td>\n",
       "      <td>-0.110266</td>\n",
       "      <td>0.151394</td>\n",
       "      <td>0.150110</td>\n",
       "      <td>0.113501</td>\n",
       "      <td>-0.073171</td>\n",
       "      <td>0.492137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-05-29</th>\n",
       "      <td>0.012595</td>\n",
       "      <td>-0.016438</td>\n",
       "      <td>0.024406</td>\n",
       "      <td>-0.057143</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.043754</td>\n",
       "      <td>0.015103</td>\n",
       "      <td>0.025389</td>\n",
       "      <td>0.067522</td>\n",
       "      <td>0.045576</td>\n",
       "      <td>...</td>\n",
       "      <td>0.012518</td>\n",
       "      <td>-0.134280</td>\n",
       "      <td>0.148426</td>\n",
       "      <td>0.013311</td>\n",
       "      <td>0.393162</td>\n",
       "      <td>0.021626</td>\n",
       "      <td>0.018964</td>\n",
       "      <td>0.074864</td>\n",
       "      <td>0.121053</td>\n",
       "      <td>0.067939</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-06-30</th>\n",
       "      <td>-0.007191</td>\n",
       "      <td>0.699164</td>\n",
       "      <td>0.098723</td>\n",
       "      <td>-0.009324</td>\n",
       "      <td>-0.098039</td>\n",
       "      <td>0.011533</td>\n",
       "      <td>0.027897</td>\n",
       "      <td>0.110559</td>\n",
       "      <td>0.079446</td>\n",
       "      <td>0.028950</td>\n",
       "      <td>...</td>\n",
       "      <td>0.023660</td>\n",
       "      <td>0.511246</td>\n",
       "      <td>-0.164056</td>\n",
       "      <td>0.018062</td>\n",
       "      <td>0.103272</td>\n",
       "      <td>0.040643</td>\n",
       "      <td>-0.023772</td>\n",
       "      <td>-0.123802</td>\n",
       "      <td>0.070422</td>\n",
       "      <td>0.293186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-07-31</th>\n",
       "      <td>-0.031464</td>\n",
       "      <td>-0.306557</td>\n",
       "      <td>0.052863</td>\n",
       "      <td>-0.218824</td>\n",
       "      <td>0.054348</td>\n",
       "      <td>0.051643</td>\n",
       "      <td>0.007599</td>\n",
       "      <td>0.007371</td>\n",
       "      <td>0.071523</td>\n",
       "      <td>0.118219</td>\n",
       "      <td>...</td>\n",
       "      <td>0.058000</td>\n",
       "      <td>0.139535</td>\n",
       "      <td>-0.094742</td>\n",
       "      <td>-0.054839</td>\n",
       "      <td>0.049120</td>\n",
       "      <td>0.035395</td>\n",
       "      <td>0.163569</td>\n",
       "      <td>-0.059820</td>\n",
       "      <td>0.021930</td>\n",
       "      <td>0.325011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-08-31</th>\n",
       "      <td>0.104118</td>\n",
       "      <td>-0.082742</td>\n",
       "      <td>0.023960</td>\n",
       "      <td>-0.018072</td>\n",
       "      <td>0.131730</td>\n",
       "      <td>0.078444</td>\n",
       "      <td>0.031921</td>\n",
       "      <td>0.102580</td>\n",
       "      <td>-0.001907</td>\n",
       "      <td>0.008038</td>\n",
       "      <td>...</td>\n",
       "      <td>0.010831</td>\n",
       "      <td>-0.115646</td>\n",
       "      <td>0.249569</td>\n",
       "      <td>0.076792</td>\n",
       "      <td>0.030919</td>\n",
       "      <td>-0.021218</td>\n",
       "      <td>-0.039240</td>\n",
       "      <td>0.051425</td>\n",
       "      <td>0.008584</td>\n",
       "      <td>0.741452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-09-30</th>\n",
       "      <td>-0.036668</td>\n",
       "      <td>0.105670</td>\n",
       "      <td>-0.071513</td>\n",
       "      <td>-0.177914</td>\n",
       "      <td>-0.199393</td>\n",
       "      <td>-0.038553</td>\n",
       "      <td>0.043341</td>\n",
       "      <td>-0.067397</td>\n",
       "      <td>-0.023567</td>\n",
       "      <td>-0.072480</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.016637</td>\n",
       "      <td>-0.305538</td>\n",
       "      <td>-0.065808</td>\n",
       "      <td>0.038827</td>\n",
       "      <td>-0.087404</td>\n",
       "      <td>0.016459</td>\n",
       "      <td>-0.096733</td>\n",
       "      <td>-0.044122</td>\n",
       "      <td>0.055319</td>\n",
       "      <td>-0.139087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-10-30</th>\n",
       "      <td>0.039727</td>\n",
       "      <td>-0.058275</td>\n",
       "      <td>-0.015432</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.104298</td>\n",
       "      <td>-0.040099</td>\n",
       "      <td>-0.056114</td>\n",
       "      <td>-0.037370</td>\n",
       "      <td>-0.034728</td>\n",
       "      <td>-0.012167</td>\n",
       "      <td>...</td>\n",
       "      <td>0.002009</td>\n",
       "      <td>-0.003102</td>\n",
       "      <td>0.065025</td>\n",
       "      <td>0.016781</td>\n",
       "      <td>-0.109859</td>\n",
       "      <td>-0.056477</td>\n",
       "      <td>-0.047755</td>\n",
       "      <td>-0.073513</td>\n",
       "      <td>-0.080645</td>\n",
       "      <td>-0.095499</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-11-30</th>\n",
       "      <td>0.072435</td>\n",
       "      <td>0.143564</td>\n",
       "      <td>0.074346</td>\n",
       "      <td>0.593284</td>\n",
       "      <td>0.298798</td>\n",
       "      <td>0.128535</td>\n",
       "      <td>0.028694</td>\n",
       "      <td>0.060058</td>\n",
       "      <td>0.081895</td>\n",
       "      <td>0.132244</td>\n",
       "      <td>...</td>\n",
       "      <td>0.041595</td>\n",
       "      <td>0.788889</td>\n",
       "      <td>0.421369</td>\n",
       "      <td>0.061515</td>\n",
       "      <td>0.257384</td>\n",
       "      <td>0.224362</td>\n",
       "      <td>0.138121</td>\n",
       "      <td>0.128552</td>\n",
       "      <td>0.144737</td>\n",
       "      <td>0.462736</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-12-31</th>\n",
       "      <td>0.072598</td>\n",
       "      <td>0.125541</td>\n",
       "      <td>0.046848</td>\n",
       "      <td>-0.051522</td>\n",
       "      <td>-0.030851</td>\n",
       "      <td>0.036571</td>\n",
       "      <td>0.120755</td>\n",
       "      <td>0.039006</td>\n",
       "      <td>0.054937</td>\n",
       "      <td>0.061920</td>\n",
       "      <td>...</td>\n",
       "      <td>0.034439</td>\n",
       "      <td>0.484472</td>\n",
       "      <td>0.109665</td>\n",
       "      <td>0.101060</td>\n",
       "      <td>0.080537</td>\n",
       "      <td>0.076239</td>\n",
       "      <td>0.135851</td>\n",
       "      <td>0.019711</td>\n",
       "      <td>0.122605</td>\n",
       "      <td>0.243252</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12 rows × 8438 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "PERMNO          10026      10028      10032      10044      10051      10065  \\\n",
       "date                                                                           \n",
       "2020-01-31  -0.100016   0.607407  -0.075643  -0.098592  -0.115176   0.005707   \n",
       "2020-02-28  -0.030270   0.225806  -0.067070  -0.064904  -0.055669  -0.090164   \n",
       "2020-03-31  -0.244031  -0.052632  -0.177694  -0.373368  -0.324664  -0.124478   \n",
       "2020-04-30   0.049835   0.448413   0.149010  -0.052083   0.178434   0.125496   \n",
       "2020-05-29   0.012595  -0.016438   0.024406  -0.057143   0.000000   0.043754   \n",
       "2020-06-30  -0.007191   0.699164   0.098723  -0.009324  -0.098039   0.011533   \n",
       "2020-07-31  -0.031464  -0.306557   0.052863  -0.218824   0.054348   0.051643   \n",
       "2020-08-31   0.104118  -0.082742   0.023960  -0.018072   0.131730   0.078444   \n",
       "2020-09-30  -0.036668   0.105670  -0.071513  -0.177914  -0.199393  -0.038553   \n",
       "2020-10-30   0.039727  -0.058275  -0.015432   0.000000   0.104298  -0.040099   \n",
       "2020-11-30   0.072435   0.143564   0.074346   0.593284   0.298798   0.128535   \n",
       "2020-12-31   0.072598   0.125541   0.046848  -0.051522  -0.030851   0.036571   \n",
       "\n",
       "PERMNO          10104      10107      10113      10138  ...      93421  \\\n",
       "date                                                    ...              \n",
       "2020-01-31  -0.005474   0.079455  -0.001206   0.095946  ...   0.011010   \n",
       "2020-02-28  -0.057007  -0.045292  -0.082666  -0.116229  ...  -0.020689   \n",
       "2020-03-31  -0.022847  -0.026541  -0.182996  -0.164901  ...  -0.086212   \n",
       "2020-04-30   0.100972   0.136326   0.139936   0.184127  ...   0.059810   \n",
       "2020-05-29   0.015103   0.025389   0.067522   0.045576  ...   0.012518   \n",
       "2020-06-30   0.027897   0.110559   0.079446   0.028950  ...   0.023660   \n",
       "2020-07-31   0.007599   0.007371   0.071523   0.118219  ...   0.058000   \n",
       "2020-08-31   0.031921   0.102580  -0.001907   0.008038  ...   0.010831   \n",
       "2020-09-30   0.043341  -0.067397  -0.023567  -0.072480  ...  -0.016637   \n",
       "2020-10-30  -0.056114  -0.037370  -0.034728  -0.012167  ...   0.002009   \n",
       "2020-11-30   0.028694   0.060058   0.081895   0.132244  ...   0.041595   \n",
       "2020-12-31   0.120755   0.039006   0.054937   0.061920  ...   0.034439   \n",
       "\n",
       "PERMNO          93422      93423      93424      93425      93426      93427  \\\n",
       "date                                                                           \n",
       "2020-01-31  -0.295556  -0.154733  -0.025676  -0.127578   0.015882  -0.027761   \n",
       "2020-02-28  -0.290221  -0.337005  -0.045770  -0.110500  -0.204690  -0.125635   \n",
       "2020-03-31  -0.842444  -0.494066  -0.058140  -0.512361  -0.269021  -0.010160   \n",
       "2020-04-30   1.947683   0.595694  -0.072531  -0.110266   0.151394   0.150110   \n",
       "2020-05-29  -0.134280   0.148426   0.013311   0.393162   0.021626   0.018964   \n",
       "2020-06-30   0.511246  -0.164056   0.018062   0.103272   0.040643  -0.023772   \n",
       "2020-07-31   0.139535  -0.094742  -0.054839   0.049120   0.035395   0.163569   \n",
       "2020-08-31  -0.115646   0.249569   0.076792   0.030919  -0.021218  -0.039240   \n",
       "2020-09-30  -0.305538  -0.065808   0.038827  -0.087404   0.016459  -0.096733   \n",
       "2020-10-30  -0.003102   0.065025   0.016781  -0.109859  -0.056477  -0.047755   \n",
       "2020-11-30   0.788889   0.421369   0.061515   0.257384   0.224362   0.138121   \n",
       "2020-12-31   0.484472   0.109665   0.101060   0.080537   0.076239   0.135851   \n",
       "\n",
       "PERMNO          93429      93434      93436  \n",
       "date                                         \n",
       "2020-01-31   0.026833   0.023810   0.555160  \n",
       "2020-02-28  -0.071904   0.372093   0.026776  \n",
       "2020-03-31  -0.217105  -0.305085  -0.215557  \n",
       "2020-04-30   0.113501  -0.073171   0.492137  \n",
       "2020-05-29   0.074864   0.121053   0.067939  \n",
       "2020-06-30  -0.123802   0.070422   0.293186  \n",
       "2020-07-31  -0.059820   0.021930   0.325011  \n",
       "2020-08-31   0.051425   0.008584   0.741452  \n",
       "2020-09-30  -0.044122   0.055319  -0.139087  \n",
       "2020-10-30  -0.073513  -0.080645  -0.095499  \n",
       "2020-11-30   0.128552   0.144737   0.462736  \n",
       "2020-12-31   0.019711   0.122605   0.243252  \n",
       "\n",
       "[12 rows x 8438 columns]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_panel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b220a397",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}