code/analysis/main.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, sys\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import glob\n",
    "import pyarrow as pa\n",
    "import pyarrow.parquet as pq\n",
    "import pyarrow.dataset as ds\n",
    "import statsmodels.formula.api as smf\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.dates as mdates\n",
    "import seaborn as sns\n",
    "import itertools\n",
    "from tqdm.auto import tqdm\n",
    "from multiprocessing import Pool\n",
    "from stargazer.stargazer import Stargazer\n",
    "from adjustText import adjust_text\n",
    "import ipywidgets as widgets\n",
    "\n",
    "sns.set_context(\"paper\", font_scale=1.7)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Stock info\n",
    "info_df = pd.read_feather('../../../HFZoo/data/keys/stock_universe.feather')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 6.11 s, sys: 8.48 s, total: 14.6 s\n",
      "Wall time: 31.6 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# Folder with clean prices for all stocks\n",
    "data_folder = '../../data/proc/clean_prices/'\n",
    "\n",
    "# Read all files in data folder\n",
    "filter_ds = pq.ParquetDataset(\n",
    "    glob.glob(data_folder + '*.parquet'),\n",
    "    metadata = pq.read_metadata(data_folder + '_metadata')\n",
    ")\n",
    "rawdata_df = filter_ds.read_pandas().to_pandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 11.3 s, sys: 4.58 s, total: 15.9 s\n",
      "Wall time: 15.8 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# Filter data\n",
    "data_df = rawdata_df.sort_values(by = ['date', 'permno', 'datetime']).copy()\n",
    "data_df['datetime'] = pd.to_datetime(data_df['datetime'])\n",
    "data_df['ticker'] = data_df['symbol'].astype(str).str.replace('.', '', regex=False)\n",
    "\n",
    "# Set aside returns with merged CRSP overnight returns\n",
    "# and set main returns to just intradaily\n",
    "data_df['return'] = np.log(1+data_df['return'])\n",
    "data_df = data_df.rename(columns={\"return\": \"return_merge\"})\n",
    "data_df[\"return\"] = np.where(\n",
    "    data_df[\"datetime\"].dt.time.astype(str) == \"09:30:00\",\n",
    "    np.nan, data_df['return_merge']\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.64 s, sys: 2.78 s, total: 4.42 s\n",
      "Wall time: 4.45 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "## Deal with missing data for holidays\n",
    "# Get list of half-days in the data\n",
    "temp_df = data_df.query('symbol in (\"SPY\", \"SPY.\")').copy()\n",
    "temp_df[\"retdiff\"] = temp_df[\"return\"].diff(1)\n",
    "half_days = (\n",
    "    temp_df.query(\"retdiff==0\")\n",
    "    .groupby([\"date\"])[\"retdiff\"]\n",
    "    .count()\n",
    "    .sort_values()\n",
    "    .reset_index()\n",
    "    .query(\"retdiff >= 20\")[\"date\"]\n",
    ")\n",
    "\n",
    "# Drop these days from return series\n",
    "data_df[\"return\"] = np.where(\n",
    "    data_df[\"date\"].isin(half_days),\n",
    "    np.nan,\n",
    "    data_df[\"return\"],\n",
    ")\n",
    "\n",
    "# Add time aggregator\n",
    "time_agg_df = pd.DataFrame(np.sort(data_df['date'].unique()), columns = ['date']).reset_index()\n",
    "time_agg_df['trading_week_id'] = np.floor(time_agg_df['index'] / 5).astype(int)\n",
    "time_agg_df['trading_month_id'] = np.floor(time_agg_df['index'] / 21).astype(int)\n",
    "time_agg_df['trading_week_datetime'] = time_agg_df.groupby(['trading_week_id'])['date'].last()\n",
    "time_agg_df['trading_month_datetime'] = time_agg_df.groupby(['trading_month_id'])['date'].last()\n",
    "data_df = data_df.merge(time_agg_df.drop(['index'], axis = 1), on = ['date'], how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.02 s, sys: 1.3 s, total: 2.31 s\n",
      "Wall time: 2.2 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Save ETF results\n",
    "etf_tickers = [\"SPY\", \"SPYV\", \"VFVA\", \"VFMO\", \"QUAL\", \"MTUM\", \"SIZE\", \"IWM\"]\n",
    "etf_permnos = (\n",
    "    info_df.query(\"ticker in @etf_tickers\")\n",
    "    .groupby([\"permno\"])[[\"ticker\", \"dt\", \"shrcd\", \"exchcd\"]]\n",
    "    .last()\n",
    "    .query(\"shrcd == 73\")\n",
    "    .index\n",
    ").astype(int).astype(str).astype('category')\n",
    "data_df.query('permno in @etf_permnos').reset_index(drop = True).to_feather('../../data/temp/etf_prices.feather')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Set up Covid Clusters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>permno</th>\n",
       "      <th>ticker</th>\n",
       "      <th>class</th>\n",
       "      <th>is_faang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>19561</td>\n",
       "      <td>BA</td>\n",
       "      <td>bad</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>34833</td>\n",
       "      <td>OXY</td>\n",
       "      <td>bad</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>17830</td>\n",
       "      <td>RTX</td>\n",
       "      <td>bad</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>14277</td>\n",
       "      <td>SLB</td>\n",
       "      <td>bad</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>80100</td>\n",
       "      <td>SPG</td>\n",
       "      <td>bad</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>76841</td>\n",
       "      <td>BIIB</td>\n",
       "      <td>good</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>86783</td>\n",
       "      <td>BKNG</td>\n",
       "      <td>good</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>12308</td>\n",
       "      <td>CHTR</td>\n",
       "      <td>good</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>77274</td>\n",
       "      <td>GILD</td>\n",
       "      <td>good</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>84788</td>\n",
       "      <td>AMZN</td>\n",
       "      <td>home</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>89393</td>\n",
       "      <td>NFLX</td>\n",
       "      <td>home</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>13407</td>\n",
       "      <td>FB</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>14593</td>\n",
       "      <td>AAPL</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>14542</td>\n",
       "      <td>GOOG</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   permno ticker class  is_faang\n",
       "0   19561     BA   bad         0\n",
       "1   34833    OXY   bad         0\n",
       "2   17830    RTX   bad         0\n",
       "3   14277    SLB   bad         0\n",
       "4   80100    SPG   bad         0\n",
       "5   76841   BIIB  good         0\n",
       "6   86783   BKNG  good         0\n",
       "7   12308   CHTR  good         0\n",
       "8   77274   GILD  good         0\n",
       "9   84788   AMZN  home         1\n",
       "10  89393   NFLX  home         1\n",
       "11  13407     FB  None         1\n",
       "12  14593   AAPL  None         1\n",
       "13  14542   GOOG  None         1"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Cluster by ticker\n",
    "cluster_bad_permnos = [\n",
    "    19561, # \"BA\"  \n",
    "    34833, # \"OXY\" \n",
    "    17830, # \"RTX\" \n",
    "    14277, # \"SLB\" \n",
    "    80100, # \"SPG\"\n",
    "]\n",
    "cluster_good_permnos = [76841, # \"BIIB\" \n",
    "                        86783, # \"BKNG\" \n",
    "                        12308, # \"CHTR\" \n",
    "                        77274  # \"GILD\"\n",
    "                       ]\n",
    "cluster_home_permnos = [84788, # \"AMZN\" \n",
    "                        89393  # \"NFLX\"\n",
    "                       ]\n",
    "other_permnos = [13407, # \"FB\" \n",
    "                 84788, # \"AMZN\"\n",
    "                 14593, # \"AAPL\"\n",
    "                 89393, # \"NFLX\" \n",
    "                 14542  # \"GOOG\"\n",
    "                ]\n",
    "\n",
    "# Create dataframe of cluster classifications\n",
    "cluster_class_df = pd.DataFrame(\n",
    "    cluster_bad_permnos + cluster_good_permnos + cluster_home_permnos + other_permnos, columns=[\"permno\"]\n",
    ").drop_duplicates().astype({'permno': 'str'})\n",
    "\n",
    "# Add PERMNOs\n",
    "cluster_class_df = cluster_class_df.merge(\n",
    "    data_df.groupby([\"permno\"])[[\"ticker\", \"symbol\"]].last().reset_index(),\n",
    "    on=[\"permno\"],\n",
    "    how=\"left\",\n",
    ")\n",
    "\n",
    "# Add cluster types\n",
    "cluster_class_df[\"class\"] = np.select(\n",
    "    [\n",
    "        cluster_class_df[\"permno\"].astype(int).isin(cluster_bad_permnos),\n",
    "        cluster_class_df[\"permno\"].astype(int).isin(cluster_home_permnos),\n",
    "        cluster_class_df[\"permno\"].astype(int).isin(cluster_good_permnos),\n",
    "    ],\n",
    "    [\"bad\", \"home\", \"good\"],\n",
    "    default=None,\n",
    ")\n",
    "cluster_class_df['is_faang'] = np.where(cluster_class_df['ticker'].str.contains('FB|AAPL|AMZN|NFLX|GOOG'), 1, 0)\n",
    "cluster_class_df.drop(['symbol'], axis= 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create portfolios"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Portfolios\n",
    "temp_df = data_df.merge(\n",
    "    cluster_class_df[[\"permno\", \"class\", \"is_faang\"]], on=[\"permno\"], how=\"left\"\n",
    ")\n",
    "\n",
    "# COVID\n",
    "covid_port_df = (\n",
    "    temp_df.groupby([\"class\", \"datetime\"])[[\"return\", \"return_merge\"]]\n",
    "    .mean()\n",
    "    .reset_index()\n",
    "    .rename(columns={\"class\": \"ticker\"})\n",
    ")\n",
    "covid_port_df[\"ticker\"] = covid_port_df[\"ticker\"].str.title()\n",
    "covid_port_df[\"permno\"] = covid_port_df[\"ticker\"].str.title()\n",
    "\n",
    "# FAANG\n",
    "faang_port_df = (\n",
    "    temp_df.groupby([\"is_faang\", \"datetime\"])[[\"return\", \"return_merge\"]]\n",
    "    .mean()\n",
    "    .reset_index()\n",
    "    .rename(columns={\"class\": \"ticker\"})\n",
    ")\n",
    "faang_port_df = faang_port_df.query(\"is_faang == 1\")[[\"datetime\", \"return\", \"return_merge\"]]\n",
    "faang_port_df[[\"ticker\", \"permno\"]] = \"FAANG\"\n",
    "\n",
    "# SPY\n",
    "spy_df = temp_df.query('permno == \"84398\"').copy()\n",
    "spy_df = spy_df[[\"datetime\", \"permno\", \"return\", \"return_merge\"]]\n",
    "spy_df[\"ticker\"] = \"SPY\"\n",
    "\n",
    "port_df = pd.concat([covid_port_df, faang_port_df, spy_df], axis=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_df.reset_index(drop = True).to_feather('../../data/proc/all_hf.feather')\n",
    "port_df.reset_index(drop = True).to_feather('../../data/proc/port_hf.feather')\n",
    "cluster_class_df.to_feather('../../data/proc/cluster_classifications.feather')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Process Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Compute Volatility Measures"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### Main"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.52 s, sys: 4.47 s, total: 6.98 s\n",
      "Wall time: 6.34 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Add intermediate columns to speed up calculations\n",
    "data_df['return_pos'] = data_df['return']*(data_df['return'] > 0)\n",
    "data_df['return_neg'] = data_df['return']*(data_df['return'] < 0)\n",
    "data_df['return_sq'] = np.square(data_df['return'])\n",
    "data_df['return_pos_sq'] = np.square(data_df['return']*(data_df['return'] > 0))\n",
    "data_df['return_neg_sq'] = np.square(data_df['return']*(data_df['return'] < 0))\n",
    "\n",
    "# Get daily results\n",
    "daily_data_df = (\n",
    "    data_df.groupby(['permno', pd.Grouper(key = 'datetime', freq = '1d', label = 'right')], observed = True)\n",
    "    [['return_sq', 'return_pos_sq', 'return_neg_sq', 'return_merge']]\n",
    "    .sum()\n",
    "    .reset_index()\n",
    ")\n",
    "daily_data_df.rename(\n",
    "    columns = {'return_sq': 'real_var', 'return_pos_sq': 'real_var_pos', 'return_neg_sq': 'real_var_neg', 'return_merge': 'log_return'}, \n",
    "    inplace = True\n",
    ")\n",
    "\n",
    "# Add datetime info\n",
    "daily_data_df['date'] = pd.to_datetime(daily_data_df['datetime'].dt.date)\n",
    "daily_data_df = daily_data_df.merge(time_agg_df.drop(['index'], axis = 1), on = ['date'], how = 'left')\n",
    "\n",
    "\n",
    "# Add weekly and monthly RV\n",
    "daily_data_df = daily_data_df.sort_values(by = ['permno', 'datetime'])\n",
    "daily_data_df['real_var_weekly'] = daily_data_df.groupby(['permno'])['real_var'].transform(\n",
    "    lambda x: x.rolling(5, min_periods = 5).sum())\n",
    "daily_data_df['real_var_monthly'] = daily_data_df.groupby(['permno'])['real_var'].transform(\n",
    "    lambda x: x.rolling(21, min_periods = 21).sum())\n",
    "daily_data_df['log_return_weekly'] = daily_data_df.groupby(['permno'])['log_return'].transform(\n",
    "    lambda x: x.rolling(5, min_periods = 5).sum())\n",
    "daily_data_df['log_return_monthly'] = daily_data_df.groupby(['permno'])['log_return'].transform(\n",
    "    lambda x: x.rolling(21, min_periods = 21).sum())\n",
    "\n",
    "# Add other RV based values\n",
    "daily_data_df['real_var_lead'] = daily_data_df.groupby(['permno'])['real_var'].shift(-1)\n",
    "daily_data_df['log_return_lead'] = daily_data_df.groupby(['permno'])['log_return'].shift(-1)\n",
    "daily_data_df['real_vol'] = np.sqrt(daily_data_df['real_var'])\n",
    "daily_data_df['real_vol_pos'] = np.sqrt(daily_data_df['real_var_pos'])\n",
    "daily_data_df['real_vol_neg'] = np.sqrt(daily_data_df['real_var_neg'])\n",
    "daily_data_df['SJ'] = daily_data_df['real_var_pos'] - daily_data_df['real_var_neg']\n",
    "daily_data_df['SJs'] = daily_data_df['real_vol_pos'] - daily_data_df['real_vol_neg']\n",
    "daily_data_df['SJ_10'] = daily_data_df['SJ']*10\n",
    "\n",
    "# Save SPY data for other notebook\n",
    "daily_data_df.query('permno == \"84398\"').reset_index(drop = True).to_feather('../../data/proc/SPY_daily.feather')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 31.2 ms, sys: 0 ns, total: 31.2 ms\n",
      "Wall time: 40.8 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Get weekly results\n",
    "weekly_data_df = (\n",
    "    daily_data_df.groupby(['permno',  pd.Grouper(key = 'datetime', freq = '1w')], observed = True)\n",
    "    [['real_var', 'real_var_pos', 'real_var_neg', 'log_return']]\n",
    "    .mean()\n",
    "    .reset_index()\n",
    ")\n",
    "# weekly_data_df['datetime'] = pd.to_datetime(weekly_data_df['trading_week_datetime'], format = '%Y%m%d')\n",
    "\n",
    "# Add weekly and weekly RV\n",
    "weekly_data_df = weekly_data_df.sort_values(by = ['permno', 'datetime'])\n",
    "# weekly_data_df['real_var_weekly'] = weekly_data_df.groupby(['permno'])['real_var'].transform(\n",
    "#     lambda x: x.rolling(5, min_periods = 5).sum())\n",
    "# weekly_data_df['real_var_monthly'] = weekly_data_df.groupby(['permno'])['real_var'].transform(\n",
    "#     lambda x: x.rolling(22, min_periods = 22).sum())\n",
    "# weekly_data_df['log_return_weekly'] = weekly_data_df.groupby(['permno'])['log_return'].transform(\n",
    "#     lambda x: x.rolling(5, min_periods = 5).sum())\n",
    "# weekly_data_df['log_return_monthly'] = weekly_data_df.groupby(['permno'])['log_return'].transform(\n",
    "#     lambda x: x.rolling(22, min_periods = 22).sum())\n",
    "\n",
    "# Add other RV based values\n",
    "weekly_data_df['real_var_lead'] = weekly_data_df.groupby(['permno'])['real_var'].shift(-1)\n",
    "weekly_data_df['log_return_lead'] = weekly_data_df.groupby(['permno'])['log_return'].shift(-1)\n",
    "weekly_data_df['real_vol'] = np.sqrt(weekly_data_df['real_var'])\n",
    "weekly_data_df['real_vol_pos'] = np.sqrt(weekly_data_df['real_var_pos'])\n",
    "weekly_data_df['real_vol_neg'] = np.sqrt(weekly_data_df['real_var_neg'])\n",
    "weekly_data_df['SJ'] = weekly_data_df['real_var_pos'] - weekly_data_df['real_var_neg']\n",
    "weekly_data_df['SJs'] = weekly_data_df['real_vol_pos'] - weekly_data_df['real_vol_neg']\n",
    "weekly_data_df['SJ_10'] = weekly_data_df['SJ']*10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 31.2 ms, sys: 15.6 ms, total: 46.9 ms\n",
      "Wall time: 28.6 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Get monthly results\n",
    "monthly_data_df = (\n",
    "    daily_data_df.groupby(['permno', pd.Grouper(key = 'datetime', freq = '1m')], observed = True)\n",
    "    [['real_var', 'real_var_pos', 'real_var_neg', 'log_return']]\n",
    "    .mean()\n",
    "    .reset_index()\n",
    ")\n",
    "# monthly_data_df['datetime'] = pd.to_datetime(monthly_data_df['trading_month_datetime'], format = '%Y%m%d')\n",
    "monthly_data_df.rename(\n",
    "    columns = {'return_sq': 'real_var', 'return_pos_sq': 'real_var_pos', 'return_neg_sq': 'real_var_neg', 'return_merge': 'log_return'}, \n",
    "    inplace = True\n",
    ")\n",
    "\n",
    "# Add weekly and monthly RV\n",
    "monthly_data_df = monthly_data_df.sort_values(by = ['permno', 'datetime'])\n",
    "# monthly_data_df['real_var_weekly'] = monthly_data_df.groupby(['permno'])['real_var'].transform(\n",
    "#     lambda x: x.rolling(5, min_periods = 5).sum())\n",
    "# monthly_data_df['real_var_monthly'] = monthly_data_df.groupby(['permno'])['real_var'].transform(\n",
    "#     lambda x: x.rolling(22, min_periods = 22).sum())\n",
    "# monthly_data_df['log_return_weekly'] = monthly_data_df.groupby(['permno'])['log_return'].transform(\n",
    "#     lambda x: x.rolling(5, min_periods = 5).sum())\n",
    "# monthly_data_df['log_return_monthly'] = monthly_data_df.groupby(['permno'])['log_return'].transform(\n",
    "#     lambda x: x.rolling(22, min_periods = 22).sum())\n",
    "\n",
    "# Add other RV based values\n",
    "monthly_data_df['real_var_lead'] = monthly_data_df.groupby(['permno'])['real_var'].shift(-1)\n",
    "monthly_data_df['log_return_lead'] = monthly_data_df.groupby(['permno'])['log_return'].shift(-1)\n",
    "monthly_data_df['real_vol'] = np.sqrt(monthly_data_df['real_var'])\n",
    "monthly_data_df['real_vol_pos'] = np.sqrt(monthly_data_df['real_var_pos'])\n",
    "monthly_data_df['real_vol_neg'] = np.sqrt(monthly_data_df['real_var_neg'])\n",
    "monthly_data_df['SJ'] = monthly_data_df['real_var_pos'] - monthly_data_df['real_var_neg']\n",
    "monthly_data_df['SJs'] = monthly_data_df['real_vol_pos'] - monthly_data_df['real_vol_neg']\n",
    "monthly_data_df['SJ_10'] = monthly_data_df['SJ']*10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.42 s, sys: 2.14 s, total: 4.56 s\n",
      "Wall time: 4.57 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Get hourly results\n",
    "hourly_data_df = (\n",
    "    data_df.groupby(['permno', pd.Grouper(key = 'datetime', freq = '1h', label = 'right')], observed = True)\n",
    "    [['return_sq', 'return_pos_sq', 'return_neg_sq', 'return']]\n",
    "    .sum()\n",
    "    .reset_index()\n",
    ")\n",
    "hourly_data_df['datetime'] = pd.to_datetime(hourly_data_df['datetime'], format = '%Y%m%d')\n",
    "hourly_data_df.rename(\n",
    "    columns = {'return_sq': 'real_var', 'return_pos_sq': 'real_var_pos', 'return_neg_sq': 'real_var_neg', 'return': 'log_return'}, \n",
    "    inplace = True\n",
    ")\n",
    "\n",
    "# Add weekly and hourly RV\n",
    "hourly_data_df = hourly_data_df.sort_values(by = ['permno', 'datetime'])\n",
    "# hourly_data_df['real_var_5'] = hourly_data_df.groupby(['permno'])['real_var'].transform(\n",
    "#     lambda x: x.rolling(5, min_periods = 5).sum())\n",
    "# hourly_data_df['real_var_22'] = hourly_data_df.groupby(['permno'])['real_var'].transform(\n",
    "#     lambda x: x.rolling(22, min_periods = 22).sum())\n",
    "# hourly_data_df['log_return_5'] = hourly_data_df.groupby(['permno'])['log_return'].transform(\n",
    "#     lambda x: x.rolling(5, min_periods = 5).sum())\n",
    "# hourly_data_df['log_return_22'] = hourly_data_df.groupby(['permno'])['log_return'].transform(\n",
    "#     lambda x: x.rolling(22, min_periods = 22).sum())\n",
    "\n",
    "# Add other RV based values\n",
    "hourly_data_df['real_var_lead'] = hourly_data_df.groupby(['permno'])['real_var'].shift(-1)\n",
    "hourly_data_df['log_return_lead'] = hourly_data_df.groupby(['permno'])['log_return'].shift(-1)\n",
    "hourly_data_df['real_vol'] = np.sqrt(hourly_data_df['real_var'])\n",
    "hourly_data_df['real_vol_pos'] = np.sqrt(hourly_data_df['real_var_pos'])\n",
    "hourly_data_df['real_vol_neg'] = np.sqrt(hourly_data_df['real_var_neg'])\n",
    "hourly_data_df['SJ'] = hourly_data_df['real_var_pos'] - hourly_data_df['real_var_neg']\n",
    "hourly_data_df['SJs'] = hourly_data_df['real_vol_pos'] - hourly_data_df['real_vol_neg']\n",
    "hourly_data_df['SJ_10'] = hourly_data_df['SJ']*10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### Portfolios"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 516 ms, sys: 484 ms, total: 1 s\n",
      "Wall time: 951 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Add intermediate columns to speed up calculations\n",
    "port_df['return_pos'] = port_df['return']*(port_df['return'] > 0)\n",
    "port_df['return_neg'] = port_df['return']*(port_df['return'] < 0)\n",
    "port_df['return_sq'] = np.square(port_df['return'])\n",
    "port_df['return_pos_sq'] = np.square(port_df['return']*(port_df['return'] > 0))\n",
    "port_df['return_neg_sq'] = np.square(port_df['return']*(port_df['return'] < 0))\n",
    "\n",
    "# Get daily results\n",
    "daily_port_df = (\n",
    "    port_df.groupby(['permno', pd.Grouper(key = 'datetime', freq = '1d', label = 'right')], observed = True)\n",
    "    [['return_sq', 'return_pos_sq', 'return_neg_sq', 'return_merge']]\n",
    "    .sum()\n",
    "    .reset_index()\n",
    ")\n",
    "daily_port_df.rename(\n",
    "    columns = {'return_sq': 'real_var', 'return_pos_sq': 'real_var_pos', 'return_neg_sq': 'real_var_neg', 'return_merge': 'log_return'}, \n",
    "    inplace = True\n",
    ")\n",
    "\n",
    "# Add weekly and monthly RV\n",
    "daily_port_df = daily_port_df.sort_values(by = ['permno', 'datetime'])\n",
    "daily_port_df['real_var_weekly'] = daily_port_df.groupby(['permno'])['real_var'].transform(\n",
    "    lambda x: x.rolling(5, min_periods = 5).sum())\n",
    "daily_port_df['real_var_monthly'] = daily_port_df.groupby(['permno'])['real_var'].transform(\n",
    "    lambda x: x.rolling(22, min_periods = 22).sum())\n",
    "daily_port_df['log_return_weekly'] = daily_port_df.groupby(['permno'])['log_return'].transform(\n",
    "    lambda x: x.rolling(5, min_periods = 5).sum())\n",
    "daily_port_df['log_return_monthly'] = daily_port_df.groupby(['permno'])['log_return'].transform(\n",
    "    lambda x: x.rolling(22, min_periods = 22).sum())\n",
    "\n",
    "# Add other RV based values\n",
    "daily_port_df['real_var_lead'] = daily_port_df.groupby(['permno'])['real_var'].shift(-1)\n",
    "daily_port_df['log_return_lead'] = daily_port_df.groupby(['permno'])['log_return'].shift(-1)\n",
    "daily_port_df['real_vol'] = np.sqrt(daily_port_df['real_var'])\n",
    "daily_port_df['real_vol_pos'] = np.sqrt(daily_port_df['real_var_pos'])\n",
    "daily_port_df['real_vol_neg'] = np.sqrt(daily_port_df['real_var_neg'])\n",
    "daily_port_df['SJ'] = daily_port_df['real_var_pos'] - daily_port_df['real_var_neg']\n",
    "daily_port_df['SJs'] = daily_port_df['real_vol_pos'] - daily_port_df['real_vol_neg']\n",
    "daily_port_df['SJ_10'] = daily_port_df['SJ']*10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "daily_data_df.to_feather('../../data/proc/all_daily.feather')\n",
    "daily_port_df.to_feather('../../data/proc/portfolios_daily.feather')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "true",
    "tags": []
   },
   "source": [
    "# Visualize"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "## SPY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "spy_daily_data_df = daily_data_df.query('permno == \"84398\"')\n",
    "spy_monthly_data_df = monthly_data_df.query('permno == \"84398\"')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axs = plt.subplots(figsize = (19,15), nrows = 2)\n",
    "\n",
    "ax = axs[0]\n",
    "spy_daily_data_df.plot('datetime', 'real_vol', color = 'k', ls = '--', alpha = 0.5, ax = ax)\n",
    "spy_daily_data_df.plot('datetime', 'real_vol_pos', color = 'b', alpha = 0.5, ax = ax)\n",
    "spy_daily_data_df.plot('datetime', 'real_vol_neg', color = 'r', alpha = 0.5, ax = ax)\n",
    "spy_daily_data_df.plot('datetime', 'SJ_10', color = 'g', alpha = 0.5, ax = ax)\n",
    "\n",
    "ax = axs[1]\n",
    "spy_monthly_data_df.plot('datetime', 'real_vol', color = 'k', ls = '--', alpha = 0.5, ax = ax)\n",
    "spy_monthly_data_df.plot('datetime', 'real_vol_pos', color = 'b', alpha = 0.5, ax = ax)\n",
    "spy_monthly_data_df.plot('datetime', 'real_vol_neg', color = 'r', alpha = 0.5, ax = ax)\n",
    "spy_monthly_data_df.plot('datetime', 'SJ_10', color = 'g', alpha = 0.5, ax = ax)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "true",
    "tags": []
   },
   "source": [
    "## BA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_daily_data_df = daily_data_df.query('permno == \"19561\"')\n",
    "sample_monthly_data_df = monthly_data_df.query('permno == \"19561\"')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axs = plt.subplots(figsize = (19,15), nrows = 2)\n",
    "\n",
    "ax = axs[0]\n",
    "sample_daily_data_df.plot('datetime', 'real_vol', color = 'k', ls = '--', alpha = 0.5, ax = ax)\n",
    "sample_daily_data_df.plot('datetime', 'real_vol_pos', color = 'b', alpha = 0.5, ax = ax)\n",
    "sample_daily_data_df.plot('datetime', 'real_vol_neg', color = 'r', alpha = 0.5, ax = ax)\n",
    "sample_daily_data_df.plot('datetime', 'SJ_10', color = 'g', alpha = 0.5, ax = ax)\n",
    "\n",
    "ax = axs[1]\n",
    "sample_monthly_data_df.plot('datetime', 'real_vol', color = 'k', ls = '--', alpha = 0.5, ax = ax)\n",
    "sample_monthly_data_df.plot('datetime', 'real_vol_pos', color = 'b', alpha = 0.5, ax = ax)\n",
    "sample_monthly_data_df.plot('datetime', 'real_vol_neg', color = 'r', alpha = 0.5, ax = ax)\n",
    "sample_monthly_data_df.plot('datetime', 'SJ_10', color = 'g', alpha = 0.5, ax = ax)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "# SHAR Regs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": "true",
    "tags": []
   },
   "source": [
    "## SPY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "spy_daily_data_df = daily_data_df.query('permno == \"84398\"')\n",
    "spy_daily_data_df = spy_daily_data_df.loc[spy_daily_data_df['datetime'].between('2002', '2019')]\n",
    "fit = smf.ols('real_var_lead ~ SJ + real_var_weekly + real_var_monthly', \n",
    "       data = spy_daily_data_df).fit(cov_type = 'HAC', cov_kwds={\"maxlags\": int(0.75*len(spy_daily_data_df)**(1/3))})\n",
    "fit.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Excluding 2020\n",
    "spy_daily_data_df = daily_data_df.query('permno == \"84398\"')\n",
    "spy_daily_data_df = spy_daily_data_df.loc[spy_daily_data_df['datetime'].between('2002', '2020')]\n",
    "fit1 = smf.ols('real_var_lead ~ real_var + real_var_weekly + real_var_monthly', \n",
    "       data = spy_daily_data_df).fit(cov_type = 'HAC', cov_kwds={\"maxlags\": int(0.75*len(spy_daily_data_df)**(1/3))})\n",
    "fit2 = smf.ols('real_var_lead ~ real_var_pos + real_var_neg + real_var_weekly + real_var_monthly', \n",
    "       data = spy_daily_data_df).fit(cov_type = 'HAC', cov_kwds={\"maxlags\": int(0.75*len(spy_daily_data_df)**(1/3))})\n",
    "fit3 = smf.ols('real_var_lead ~ SJ + real_var_weekly + real_var_monthly', \n",
    "       data = spy_daily_data_df).fit(cov_type = 'HAC', cov_kwds={\"maxlags\": int(0.75*len(spy_daily_data_df)**(1/3))})\n",
    "\n",
    "stargazer = Stargazer([fit1, fit2, fit3])\n",
    "stargazer.covariate_order(['real_var', 'real_var_pos', 'real_var_neg', 'SJ', 'real_var_weekly', 'real_var_monthly'])\n",
    "print(stargazer.render_latex()\n",
    "      .replace('SJ ', '$SJ_t$')\n",
    "      .replace('real_var_weekly', '$RV_{t:t-4}$')\n",
    "      .replace('real_var_monthly', '$RV_{t:t-22}$')\n",
    "      .replace('real_var_pos', '$RV_{t}^+$')\n",
    "      .replace('real_var_neg', '$RV_{t}^-$')\n",
    "      .replace('real_var', '$RV_{t}$')\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2002-2020\n",
    "spy_daily_data_df = daily_data_df.query('permno == \"84398\"')\n",
    "spy_daily_data_df = spy_daily_data_df.loc[spy_daily_data_df['datetime'].between('2002', '2021')]\n",
    "fit1 = smf.ols('real_var_lead ~ real_var + real_var_weekly + real_var_monthly', \n",
    "       data = spy_daily_data_df).fit(cov_type = 'HAC', cov_kwds={\"maxlags\": int(0.75*len(spy_daily_data_df)**(1/3))})\n",
    "fit2 = smf.ols('real_var_lead ~ real_var_pos + real_var_neg + real_var_weekly + real_var_monthly', \n",
    "       data = spy_daily_data_df).fit(cov_type = 'HAC', cov_kwds={\"maxlags\": int(0.75*len(spy_daily_data_df)**(1/3))})\n",
    "fit3 = smf.ols('real_var_lead ~ SJ + real_var_weekly + real_var_monthly', \n",
    "       data = spy_daily_data_df).fit(cov_type = 'HAC', cov_kwds={\"maxlags\": int(0.75*len(spy_daily_data_df)**(1/3))})\n",
    "\n",
    "stargazer = Stargazer([fit1, fit2, fit3])\n",
    "stargazer.covariate_order(['real_var', 'real_var_pos', 'real_var_neg', 'SJ', 'real_var_weekly', 'real_var_monthly'])\n",
    "print(stargazer.render_latex()\n",
    "      .replace('SJ ', '$SJ_t$')\n",
    "      .replace('real_var_weekly', '$RV_{t:t-4}$')\n",
    "      .replace('real_var_monthly', '$RV_{t:t-22}$')\n",
    "      .replace('real_var_pos', '$RV_{t}^+$')\n",
    "      .replace('real_var_neg', '$RV_{t}^-$')\n",
    "      .replace('real_var', '$RV_{t}$')\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2020 only\n",
    "spy_daily_data_df = daily_data_df.query('permno == \"84398\"')\n",
    "spy_daily_data_df = spy_daily_data_df.loc[spy_daily_data_df['datetime'].between('2020', '2021')]\n",
    "fit1 = smf.ols('real_var_lead ~ real_var + real_var_weekly + real_var_monthly', \n",
    "       data = spy_daily_data_df).fit(cov_type = 'HAC', cov_kwds={\"maxlags\": int(0.75*len(spy_daily_data_df)**(1/3))})\n",
    "fit2 = smf.ols('real_var_lead ~ real_var_pos + real_var_neg + real_var_weekly + real_var_monthly', \n",
    "       data = spy_daily_data_df).fit(cov_type = 'HAC', cov_kwds={\"maxlags\": int(0.75*len(spy_daily_data_df)**(1/3))})\n",
    "fit3 = smf.ols('real_var_lead ~ SJ + real_var_weekly + real_var_monthly', \n",
    "       data = spy_daily_data_df).fit(cov_type = 'HAC', cov_kwds={\"maxlags\": int(0.75*len(spy_daily_data_df)**(1/3))})\n",
    "\n",
    "stargazer = Stargazer([fit1, fit2, fit3])\n",
    "stargazer.covariate_order(['real_var', 'real_var_pos', 'real_var_neg', 'SJ', 'real_var_weekly', 'real_var_monthly'])\n",
    "print(stargazer.render_latex()\n",
    "      .replace('SJ ', '$SJ_t$')\n",
    "      .replace('real_var_weekly', '$RV_{t:t-4}$')\n",
    "      .replace('real_var_monthly', '$RV_{t:t-22}$')\n",
    "      .replace('real_var_pos', '$RV_{t}^+$')\n",
    "      .replace('real_var_neg', '$RV_{t}^-$')\n",
    "      .replace('real_var', '$RV_{t}$')\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Clusters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fits\n",
    "fit_names = []\n",
    "shar_fits = []\n",
    "\n",
    "for i, row in cluster_class_df.iterrows():\n",
    "    \n",
    "    if not row['permno']:\n",
    "        print('Skipping ', row['ticker'])\n",
    "        continue\n",
    "    \n",
    "    # Run OLS\n",
    "    sample_df = daily_data_df.query(f'permno == \"{row[\"permno\"]}\"')\n",
    "    sample_df = sample_df.loc[sample_df['datetime'].between('2002', '2020')]\n",
    "    fit_names.append(row['ticker'])\n",
    "    fit = smf.ols('real_var_lead ~ real_var_pos + real_var_neg + real_var_weekly + real_var_monthly', \n",
    "           data = sample_df).fit(cov_type = 'HAC', cov_kwds={\"maxlags\": int(0.75*len(sample_df)**(1/3))})\n",
    "    shar_fits.append(fit)\n",
    "    fit_sj = smf.ols('real_var_lead ~ SJ + real_var_weekly + real_var_monthly', \n",
    "           data = sample_df).fit(cov_type = 'HAC', cov_kwds={\"maxlags\": int(0.75*len(sample_df)**(1/3))})\n",
    "    \n",
    "    # Save results to dataframe\n",
    "    cluster_class_df.loc[i, 'coeff_phi_pos_d'] = fit.params['real_var_pos']\n",
    "    cluster_class_df.loc[i, 'coeff_phi_neg_d'] = fit.params['real_var_neg']\n",
    "    cluster_class_df.loc[i, 't_phi_pos_d'] = np.abs(fit.tvalues['real_var_pos'])\n",
    "    cluster_class_df.loc[i, 't_phi_neg_d'] = np.abs(fit.tvalues['real_var_neg'])\n",
    "    cluster_class_df.loc[i, 'coeff_SJ'] = fit_sj.params['SJ']\n",
    "    cluster_class_df.loc[i, 't_SJ'] = np.abs(fit_sj.tvalues['SJ'])\n",
    "    \n",
    "## SPY\n",
    "spy_class_df = pd.DataFrame({'permno': '84398', 'ticker': 'SPY'}, index = [0])\n",
    "\n",
    "# Run OLS\n",
    "i = 0\n",
    "row = spy_class_df.iloc[i]\n",
    "sample_df = daily_data_df.query(f'permno == \"{row[\"permno\"]}\"')\n",
    "sample_df = sample_df.loc[sample_df['datetime'].between('2002', '2020')]\n",
    "fit_names.append(row['ticker'])\n",
    "fit = smf.ols('real_var_lead ~ real_var_pos + real_var_neg + real_var_weekly + real_var_monthly', \n",
    "       data = sample_df).fit(cov_type = 'HAC', cov_kwds={\"maxlags\": int(0.75*len(sample_df)**(1/3))})\n",
    "shar_fits.append(fit)\n",
    "fit_sj = smf.ols('real_var_lead ~ SJ + real_var_weekly + real_var_monthly', \n",
    "       data = sample_df).fit(cov_type = 'HAC', cov_kwds={\"maxlags\": int(0.75*len(sample_df)**(1/3))})\n",
    "\n",
    "# Save results to dataframe\n",
    "spy_class_df.loc[i, 'coeff_phi_pos_d'] = fit.params['real_var_pos']\n",
    "spy_class_df.loc[i, 'coeff_phi_neg_d'] = fit.params['real_var_neg']\n",
    "spy_class_df.loc[i, 't_phi_pos_d'] = np.abs(fit.tvalues['real_var_pos'])\n",
    "spy_class_df.loc[i, 't_phi_neg_d'] = np.abs(fit.tvalues['real_var_neg'])\n",
    "spy_class_df.loc[i, 'coeff_SJ'] = fit_sj.params['SJ']\n",
    "spy_class_df.loc[i, 't_SJ'] = np.abs(fit_sj.tvalues['SJ'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stargazer = Stargazer(shar_fits)\n",
    "stargazer.custom_columns(fit_names, [1]*14)\n",
    "stargazer.show_model_numbers(False)\n",
    "stargazer.covariate_order(['real_var_pos', 'real_var_neg', 'real_var_weekly', 'real_var_monthly'])\n",
    "stargazer.significant_digits(2)\n",
    "# stargazer.significance_levels([1e-999, 1e-999, 1e-999])\n",
    "print(stargazer.render_latex()\n",
    "      .replace('real_var_weekly', '$RV_{t:t-4}$')\n",
    "      .replace('real_var_monthly', '$RV_{t:t-22}$')\n",
    "      .replace('real_var_pos', '$RV_{t}^+$')\n",
    "      .replace('real_var_neg', '$RV_{t}^-$')\n",
    ")\n",
    "stargazer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Coeffs (SemiRV)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize = (16,11))\n",
    "p1 = cluster_class_df.plot.scatter('coeff_phi_pos_d', 'coeff_phi_neg_d', s = 100, ax= ax)\n",
    "texts = []\n",
    "\n",
    "# Cluster\n",
    "for line in range(0,cluster_class_df.shape[0]):\n",
    "     texts.append(p1.text(cluster_class_df.loc[line, 'coeff_phi_pos_d']+0.01, cluster_class_df.loc[line, 'coeff_phi_neg_d']+0.01, \n",
    "     cluster_class_df.loc[line, 'ticker'], horizontalalignment='left', weight = ('bold' if cluster_class_df.loc[line,'is_faang'] else 'normal'),\n",
    "     size=14, color='black'))\n",
    "        \n",
    "# SPY\n",
    "p2 = spy_class_df.plot.scatter('coeff_phi_pos_d', 'coeff_phi_neg_d', s = 150, ax= ax, color = 'tab:green')\n",
    "texts.append(p2.text(spy_class_df.loc[0, 'coeff_phi_pos_d']+0.01, spy_class_df.loc[0, 'coeff_phi_neg_d']+0.01, \n",
    "spy_class_df.loc[0, 'ticker'], horizontalalignment='left', weight = 'bold', size=14, color='black'))  \n",
    "    \n",
    "# adjust_text(texts, force_text=0.05)\n",
    "ax.set_xlim(-0.6, 0.5)\n",
    "ax.set_ylim(-0.1, 1.3)\n",
    "# ax.autoscale()\n",
    "ax.axhline(0, lw = 2, ls = '--', color = 'gray')\n",
    "ax.axvline(0, lw = 2, ls = '--', color = 'gray')\n",
    "ax.plot([-100, 100], [-100, 100], lw = 2, ls = '--', color = 'darkgray')\n",
    "ax.set_ylabel('Coefficient on $RV_t^-$ ($\\\\phi_D^{-}$)')\n",
    "ax.set_xlabel('Coefficient on $RV_t^+$ ($\\\\phi_D^{+}$)')\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.savefig('../../exhibits/semivar_coeffs.pdf')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "### T-stats (SemiRV)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize = (14,11))\n",
    "p1 = cluster_class_df.plot.scatter('t_phi_pos_d', 't_phi_neg_d', s = 100, ax= ax)\n",
    "texts = []\n",
    "\n",
    "for line in range(0,cluster_class_df.shape[0]):\n",
    "     texts.append(p1.text(cluster_class_df.loc[line, 't_phi_pos_d']+0.03, cluster_class_df.loc[line, 't_phi_neg_d'], \n",
    "     cluster_class_df.loc[line, 'ticker'], horizontalalignment='left', weight = ('bold' if cluster_class_df.loc[line,'is_faang'] else 'normal'),\n",
    "     size='medium', color='black'))\n",
    "        \n",
    "# SPY\n",
    "p2 = spy_class_df.plot.scatter('t_phi_pos_d', 't_phi_neg_d', s = 150, ax= ax, color = 'tab:green')\n",
    "texts.append(p2.text(spy_class_df.loc[0, 't_phi_pos_d']+0.03, spy_class_df.loc[0, 't_phi_neg_d']+0.01, \n",
    "spy_class_df.loc[0, 'ticker'], horizontalalignment='left', weight = 'bold', size='medium', color='black'))  \n",
    "\n",
    "# adjust_text(texts, force_text=0.01)\n",
    "ax.set_xlim(0, 2.75)\n",
    "ax.axhline(1.96, lw = 2, ls = '--', color = 'k', alpha = 0.5)\n",
    "ax.axvline(1.96, lw = 2, ls = '--', color = 'k', alpha = 0.5)\n",
    "ax.set_ylabel('T-Stat on Real SemiVar Negative ($\\\\phi_D^{-}$)')\n",
    "ax.set_xlabel('T-Stat on Real SemiVar Positive ($\\\\phi_D^{+}}$)')\n",
    "ax.text(1, 5, '$\\\\phi_D^{-}$ significant', horizontalalignment = 'center', verticalalignment = 'center' , color = 'tab:blue')\n",
    "ax.text(2.3, 5, '$\\{\\\\phi_D^{-}, \\\\phi_D^{+}\\}$ significant', horizontalalignment = 'center', verticalalignment = 'center' , color = 'tab:blue')\n",
    "ax.text(2.3, 1, '$\\\\phi_D^{+}$ significant', horizontalalignment = 'center', verticalalignment = 'center' , color = 'tab:blue')\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.savefig('../../exhibits/semivar_tstats.pdf')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Coeffs (SJV)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize = (16,11))\n",
    "p1 = cluster_class_df.plot.scatter('coeff_SJ', 't_SJ', s = 100, ax= ax)\n",
    "texts = []\n",
    "\n",
    "for line in range(0,cluster_class_df.shape[0]):\n",
    "     texts.append(p1.text(cluster_class_df.loc[line, 'coeff_SJ']+0.01, cluster_class_df.loc[line, 't_SJ'], \n",
    "     cluster_class_df.loc[line, 'ticker'], horizontalalignment='left', weight = ('bold' if cluster_class_df.loc[line,'is_faang'] else 'normal'),\n",
    "     size=14, color='black'))\n",
    "        \n",
    "# SPY\n",
    "p2 = spy_class_df.plot.scatter('coeff_SJ', 't_SJ', s = 150, ax= ax, color = 'tab:green')\n",
    "texts.append(p2.text(spy_class_df.loc[0, 'coeff_SJ']+0.01, spy_class_df.loc[0, 't_SJ']+0.01, \n",
    "spy_class_df.loc[0, 'ticker'], horizontalalignment='left', weight = 'bold', size='medium', color='black'))  \n",
    "\n",
    "adjust_text(texts, force_text=0.05)\n",
    "ax.axhline(1.96, lw = 2, ls = '-.', color = 'g')\n",
    "ax.axvline(0, lw = 2, ls = '--', color = 'gray')\n",
    "ax.set_ylabel('T-stat on Signed Jump Variation')\n",
    "ax.set_xlabel('Coeff on Signed Jump Variation')\n",
    "# ax.set_title('Coeff versus T-stat For Signed Jump Variation', fontsize = 20)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.savefig('../../exhibits/semivar_sj_results.pdf')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}