optimize_us.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Background"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Created to optimize algo and helper codes for decreased execution time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json\n",
    "from pillaralgos.helpers import data_handler as dh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pillaralgos import algo1,algo2,algo3_0,algo3_5,brain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = json.load(open('data/lg_962598044.json'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Timeit results as of `April 13, 2021 18:31 EST` for file `962598044.json`\n",
    "\n",
    "| algo1   | algo2        | algo3_0 | algo3_5 | brain |\n",
    "|---------|--------------|---------|---------|-------|\n",
    "|1.59 sec | 1 min 6 sec  |22.0 sec | 11.6 sec| xx    |"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%timeit\n",
    "\n",
    "json1 = algo1.run(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%timeit\n",
    "\n",
    "json2 = algo2.run(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%timeit\n",
    "\n",
    "json3_0 = algo3_0.run(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%timeit\n",
    "\n",
    "json3_5 = algo3_5.run(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%timeit\n",
    "\n",
    "brain_r = brain.run(data, clip_length=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# organize_twitch_chat()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class dictExtractor:\n",
    "    def __init__(self, my_series, label = ''):\n",
    "        '''\n",
    "        Extracts dictionaries from series into a new dict using the\n",
    "        longest dictionary's keys. Converts new dict into df, stored\n",
    "        as `self.result`.\n",
    "        \n",
    "        input\n",
    "        -----\n",
    "        my_series: pd.Series\n",
    "            A column from twitch dataframe where each row is a dict\n",
    "        label: str\n",
    "            What will be appended to the end of each col\n",
    "        '''\n",
    "        # find max length of dicts\n",
    "        length = my_series.apply(lambda x: len(x))\n",
    "        y = 0\n",
    "        for x in length:\n",
    "            if x > y:\n",
    "                y = x\n",
    "        # find index of max keys dict\n",
    "        ind = length[length == 8].index[0]\n",
    "        max_d = my_series.iloc[ind].keys()\n",
    "        self.max_d = max_d\n",
    "        # initiate new dict\n",
    "        self.new_dict = {}\n",
    "        for k in max_d:\n",
    "            self.new_dict[k] = []\n",
    "        # extract dict values into new dict\n",
    "        my_series.apply(lambda x: self.keys_iterator(x))\n",
    "        # store as df\n",
    "        self.result = pd.DataFrame.from_dict(self.new_dict)\n",
    "        # df.add_suffix() is actually 0.25 seconds slower\n",
    "        self.result.columns = [col + label for col in self.result.columns]\n",
    "        \n",
    "    def keys_iterator(self, my_dict):\n",
    "        '''\n",
    "        Checks that all of the `max_d` are in the given dictionary. If not,\n",
    "        appends np.nan. Otherwise appends the value.\n",
    "        '''\n",
    "        for k in self.max_d:\n",
    "            if k not in my_dict.keys():\n",
    "                self.new_dict[k].append(np.nan)\n",
    "            else:\n",
    "                self.new_dict[k].append(my_dict[k])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def organize_twitch_chat(data):\n",
    "    '''\n",
    "    Turns json into dataframe. Expands lists of lists into own columns. Selects only relevant columns.\n",
    "    \n",
    "    input\n",
    "    -----\n",
    "    data: list\n",
    "        list of dictionaries in json format, loaded with the `open` context manager.\n",
    "        \n",
    "    output\n",
    "    ------\n",
    "    df: pd.DataFrame\n",
    "        Dataframe with the following columns: \n",
    "            ['created_at', 'updated_at', 'display_name', '_id', 'name', 'type',\n",
    "             'bio', 'logo', 'body', 'is_action', 'user_badges', 'emoticons']\n",
    "    '''\n",
    "    data = pd.DataFrame.from_records(data) # convert to df\n",
    "    df = data[['created_at','updated_at','commenter','message']].add_suffix('_mess')\n",
    "    \n",
    "    h = dictExtractor(df['message_mess'],label='_mess')\n",
    "    messages = h.result\n",
    "    g = dictExtractor(df['commenter_mess'], label='_id')\n",
    "    users = g.result\n",
    "    \n",
    "    df = df.drop(['message_mess','commenter_mess'], axis=1) # duplicate info\n",
    "    df = pd.concat([df,users,messages],axis=1)\n",
    "    \n",
    "    cols = list(df.columns)\n",
    "    cols.sort()\n",
    "    \n",
    "    bad_cols = [\n",
    "                'display_name_id',\n",
    "                'logo_id',\n",
    "                'name_id',\n",
    "                'user_badges_mess',\n",
    "                'user_color_mess',\n",
    "                'user_notice_params_mess'\n",
    "            ]\n",
    "    df = df[cols]\n",
    "    df = df.drop(bad_cols, axis=1)\n",
    "    # all vars were loaded as str. Change type to datetime/int/bool\n",
    "    df = df.astype( {'_id_id':int,\n",
    "                    'bio_id':'category',\n",
    "                    'created_at_id':'datetime64[ns]',\n",
    "                    'created_at_mess':'datetime64[ns]',\n",
    "                    'updated_at_id':'datetime64[ns]',\n",
    "                    'updated_at_mess':'datetime64[ns]',\n",
    "                    'is_action_mess':bool,\n",
    "                    'type_id':'category'}\n",
    "                  )    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Pillar Env",
   "language": "python",
   "name": "pillar_env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}