getdata.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import bs4 as bs\n",
    "import requests\n",
    "import pandas as pd\n",
    "import datetime as dt\n",
    "import os\n",
    "pd.core.common.is_list_like = pd.api.types.is_list_like\n",
    "import pandas_datareader.data as web\n",
    "from tiingo import TiingoClient\n",
    "\n",
    "# auth \n",
    "yourtoken = os.getenv(\"TIINGO_API_KEY\")\n",
    "\n",
    "config = {}\n",
    "config['session'] = True\n",
    "config['api_key'] = yourtoken\n",
    "\n",
    "client = TiingoClient(config)\n",
    "\n",
    "# since requests doesn't work correctly on JS objects, we will just manually get the DOW30 tickers + DJI\n",
    "def DJIA_tickers() -> list[str]:\n",
    "    return ['MMM', 'AXP', 'AMGN', 'AAPL', 'BA', 'CAT', 'CVX', 'CSCO', 'KO', 'DIS', 'DOW', 'XOM', 'GS', 'HD', 'HON', 'IBM', 'INTC', 'JNJ', 'JPM', 'MCD', 'MRK', 'MSFT', 'NKE', 'PG', 'CRM', 'TRV', 'UNH', 'VZ', 'V', 'WBA', 'WMT', 'DIA', 'SPY']"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TODO: try this with PP - pricing power \n",
    "# TODO: try this with QQQ - technologies \n",
    "# TODO: try this with GUNR / other stable ETFs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dataReader(ticker, d_start, d_end, frequency):\n",
    "    d_end = d_end.strftime(\"%Y-%m-%d\")\n",
    "    d_start = d_start.strftime(\"%Y-%m-%d\")\n",
    "    print(\"Getting Data from Tiingo\")\n",
    "    df = client.get_dataframe(ticker, startDate=d_start, endDate=d_end, frequency=frequency)\n",
    "    # print(f\"head: {df.head()}\")\n",
    "    #print(f\"columns: {df.columns}\")\n",
    "    # print(f\"shape: {df.shape}\")\n",
    "    #print(f\"index: {df.index}\")\n",
    "    # print(f\"index name: {df.index.name}\")\n",
    "    #print(f\"info: {df.info()}\")\n",
    "    \n",
    "      # Check for existance of Adj Close column\n",
    "      # If not, check for existance of Close column\n",
    "      # If not -> throw error\n",
    "      # If no Adj Close, but Close, copy Close to Adj close\n",
    "\n",
    "    if not \"adjClose\" in df.columns:\n",
    "        if \"close\" in df.columns:\n",
    "                df[\"adjClose\"] = df[\"close\"]\n",
    "        else:\n",
    "                print(\"Error: No Close information\")\n",
    "        return\n",
    "      \n",
    "    # Convert ISO date format to Pandas DateTime\n",
    "    #df[\"date\"] = pd.to_datetime(df[\"date\"])\n",
    "    # Align Column Names to previous DataReader names\n",
    "    df.index = pd.to_datetime(df.index)\n",
    "    print(f\"index: {df.index}\")\n",
    "    #print(f\"info: {df.info()}\")\n",
    "\n",
    "    df = df.rename(\n",
    "        columns={\n",
    "        'date': \"Date\",\n",
    "        \"open\": \"Open\",\n",
    "        \"adjClose\": \"Adj Close\",\n",
    "        \"volume\": \"Volume\",\n",
    "        \"high\": \"High\",\n",
    "        \"low\": \"Low\",\n",
    "        \"close\": \"Close\",\n",
    "        }\n",
    "    )\n",
    "    # print(df)\n",
    "    #a = df.set_index([\"Date\"])\n",
    "    # print(a)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def DJIA_tickers():\n",
    "# \t'''\n",
    "\n",
    "# \tParses Slickcharts Webpage to obtain the tickers for each current listed stock in the DJIA.\n",
    "\n",
    "# \t'''\n",
    "\n",
    "# \tresp = requests.get('https://money.cnn.com/data/dow30/') # https://www.cnbc.com/dow-30/\n",
    "# \tsoup = bs.BeautifulSoup(resp.text, \"lxml\")\n",
    "# \ttable = soup.find('table', {'class': 'wsod_dataTable wsod_dataTableBig'}) # BasicTable-table\n",
    "# \ttickers = []\n",
    "# \tfor row in table.findAll('tr')[1:]:\n",
    "# \t\tticker = row.findAll('td')[0].text\n",
    "# \t\tticker = ticker.split('\\xa0', 1)[0]\n",
    "# \t\ttickers.append(ticker)\n",
    "\n",
    "# \treturn tickers\n",
    "\n",
    "# DJIA_tickers()\n",
    "# print(DJIA_tickers())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def API_data():\n",
    "\t'''\n",
    "\n",
    "\tUses Tiingo API to access all the historical data for the current stocks listed in the DJIA.\n",
    "\n",
    "\t'''\n",
    "\t# DJIA = 'DIA'\n",
    "\ttickers = DJIA_tickers()\n",
    "\t# tickers.append(DJIA)\n",
    "\n",
    "\tif not os.path.exists('stocks_DJIA'):\n",
    "\t\tos.makedirs('stocks_DJIA')\n",
    "\n",
    "\tstart = dt.datetime(2000,1,1)\n",
    "\tend = dt.datetime.now()\n",
    "\tprint(f\"tickers: {tickers}\")\n",
    "\t\n",
    "\tfor ticker in tickers:\n",
    "\t\tif not os.path.exists('stock_DJIA/{}.csv'.format(ticker)):\n",
    "\t\t\tdf = dataReader(ticker, start, end, frequency='daily')\n",
    "\t\t\tdf.to_csv('stocks_DJIA/{}.csv'.format(ticker))\n",
    "\t\t\tprint(df)\n",
    "\t\telse:\n",
    "\t\t\tprint('Information already acquired for {}'.format(ticker))\n",
    "\n",
    "print(f\"api_data: {API_data()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compile_data():\n",
    "\t'''\n",
    "\n",
    "\tCreates a dataframe with the compiled adjusted closing for all the stocks in the DJIA.\n",
    "\n",
    "\t'''\n",
    "\n",
    "\ttickers = DJIA_tickers()\n",
    "\n",
    "\tmain_df = pd.DataFrame()\n",
    "\n",
    "\tfor count,ticker in enumerate(tickers):\n",
    "\t\tdf = pd.read_csv('stocks_DJIA/{}.csv'.format(ticker))\n",
    "\t\tdf.set_index('date', inplace= True)\n",
    "\n",
    "\t\tdf.rename(columns = {'Adj Close': ticker}, inplace=True)\n",
    "\n",
    "\t\tdf.drop(['Open', 'High', 'Low', 'Close', 'Volume', 'adjHigh', 'adjLow', 'adjOpen', 'adjVolume', 'divCash', 'splitFactor'], 1, inplace=True)\n",
    "\n",
    "\t\tif main_df.empty:\n",
    "\t\t\tmain_df = df\n",
    "\t\telse:\n",
    "\t\t\tmain_df = main_df.join(df, how='outer')\n",
    "\n",
    "\t\tif count % 10 == 0:\n",
    "\t\t\tprint(count)\n",
    "\n",
    "\tprint(main_df.head())\n",
    "\tmain_df.to_csv('DJIA_adjcloses.csv')\n",
    "\n",
    "print(f\"compiling data...\")\n",
    "compile_data()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pipenv",
   "language": "python",
   "name": "pipenv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}