Testbot1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Import basic packages\n",
    "import os\n",
    "from urllib.request import urlretrieve, urlopen\n",
    "import time\n",
    "\n",
    "# Import data science packages\n",
    "import pandas as pd\n",
    "\n",
    "# Import reddit related packages\n",
    "import praw\n",
    "import pdb\n",
    "import re\n",
    "\n",
    "# Import CNN\n",
    "#from neuralnetwork import food_CNN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Config parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set download destinations \n",
    "pic_dest = \"C:/project/imgbot/pics/\"\n",
    "thumbnail_dest = \"C:/project/imgbot/thumbnails/\"\n",
    "\n",
    "# Set up reddit instance\n",
    "reddit = praw.Reddit('bot1')\n",
    "\n",
    "# Check or create csv file to save data\n",
    "csv_file = 'replies.csv'\n",
    "col_list = ['Post_ID', 'Post_title', 'Post_URL', 'Subreddit', 'Post_Date', 'Post_Score', 'Bot_Reply', 'Reply_Date', 'Classification', 'Calories', 'Reply_Score']\n",
    "\n",
    "if os.path.isfile('replies.csv'):\n",
    "   # if there is a log, load it as a pandas dataframe    \n",
    "    reply_log = pd.read_csv(csv_file)\n",
    "else:\n",
    "    reply_log = pd.DataFrame(columns=col_list)\n",
    "\n",
    "reply_log.set_index('Post_ID', inplace=True)\n",
    "\n",
    "# Set up list of subreddits to monitor\n",
    "subreddit_list = 'food+HealthyFood'\n",
    "subreddit = reddit.subreddit('pythonforengineers')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Download helper that checks if file already exists\n",
    "def download(url, filename, destination):\n",
    "    \"\"\"\n",
    "    Download file from <url>\n",
    "    :param url: URL to file\n",
    "    :param filename: name that file is saved as (also checks for duplications)\n",
    "    :param destination: Local file save path\n",
    "    \"\"\"\n",
    "    full_destination = destination + filename\n",
    "    \n",
    "    if not url:\n",
    "        return print(\"invalid URL. Skipping...\")\n",
    "    \n",
    "    if not os.path.isfile(full_destination):\n",
    "                urlretrieve(url, full_destination) \n",
    "    \n",
    "    return print(\"Downloaded \", url)\n",
    "\n",
    "\n",
    "# Helper to check if submission is an image\n",
    "def submission_is_image(url):\n",
    "    \"\"\"\n",
    "    Check if the submission is an image. Returns boolean\n",
    "    :param url: URL of Reddit submission\n",
    "    \"\"\"   \n",
    "    response = urlopen(url)\n",
    "    type = response.headers.get_content_maintype()    \n",
    "    if type == 'image':   \n",
    "        return True    \n",
    "    else:    \n",
    "        return False \n",
    "\n",
    "\n",
    "# Helper to save a new reply entry or update existing entry in dataframe and csv    \n",
    "def save_reply(data, save_dataframe, csv_file):\n",
    "    \"\"\"\n",
    "    Save submission and reply data to reply_log dataframe and csv file\n",
    "    :param data: list of data in same format/structure as reply_log\n",
    "    :param save_dataframe: the target dataframe to update with new data\n",
    "    :param csv_file: filename of the csv_file for saving\n",
    "    \"\"\"\n",
    "    col_list = list(save_dataframe.columns.values)        \n",
    "    new_entry = pd.DataFrame([new_row], columns=col_list)\n",
    "    new_entry.set_index('Post_ID', inplace=True)\n",
    "    \n",
    "    if new_entry.index[0] in save_dataframe.index:\n",
    "        save_dataframe.update(new_entry)\n",
    "    else:\n",
    "        save_dataframe = save_dataframe.append(new_entry)\n",
    "    \n",
    "    save_dataframe.to_csv(csv_file, encoding='utf-8', index=False) # Save dataframe as .csv file   \n",
    "    return print(\"Reply added to dataframe and saved to {}\".format(csv_file)) \n",
    "\n",
    "\n",
    "# Helper to reply to a submission with classification data, and return new entry to log        \n",
    "def send_reply(text, submission, classification=None, calories=None):\n",
    "    \"\"\"\n",
    "    Save submission and reply data to reply_log dataframe and csv file\n",
    "    :param text: list of data in same format/structure as reply_log\n",
    "    :param submission: the target dataframe to update with new data\n",
    "    :param classification: food type / classification to save in record\n",
    "    :param calories: calories score to save in record. Should be same as posted in reply text\n",
    "    \"\"\"       \n",
    "    reply_date = time.time()      \n",
    "    reply_id = submission.reply(text)       \n",
    "    reply = reddit.comment(reply_id)\n",
    "    reply_score = None\n",
    "    new_row = [submission.id,\n",
    "               submission.title,\n",
    "               submission.url,\n",
    "               submission.subreddit.name,\n",
    "               submission.created_utc,\n",
    "               submission.score,\n",
    "               reply.id,\n",
    "               reply_date,\n",
    "               classification,\n",
    "               calories,\n",
    "               reply_score]\n",
    "    return new_row"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Subreddit loop and reply function\n",
    "#### Scans through subreddits and classifies food images "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloaded  https://www.reddit.com/r/pythonforengineers/comments/7fxac4/testing80/\n",
      "invalid URL. Skipping...\n"
     ]
    },
    {
     "ename": "UnboundLocalError",
     "evalue": "local variable 'reply_log' referenced before assignment",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mUnboundLocalError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-20-ff9a674fb141>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     20\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     21\u001b[0m             \u001b[1;31m# Save reply in dataframe and csv file\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 22\u001b[1;33m             \u001b[0msave_reply\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnew_reply\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     23\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     24\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-17-2f086a6cc6a4>\u001b[0m in \u001b[0;36msave_reply\u001b[1;34m(data)\u001b[0m\n\u001b[0;32m     38\u001b[0m     \u001b[1;33m:\u001b[0m\u001b[0mparam\u001b[0m \u001b[0mdata\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mlist\u001b[0m \u001b[0mof\u001b[0m \u001b[0mdata\u001b[0m \u001b[1;32min\u001b[0m \u001b[0msame\u001b[0m \u001b[0mformat\u001b[0m\u001b[1;33m/\u001b[0m\u001b[0mstructure\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mreply_log\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     39\u001b[0m     \"\"\"\n\u001b[1;32m---> 40\u001b[1;33m     \u001b[0mcol_list\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mreply_log\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     41\u001b[0m     \u001b[0mnew_entry\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mnew_row\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcol_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     42\u001b[0m     \u001b[1;31m#TODO: CHANGE THIS DATAFRAME APPEND TO DATAFRAME MERGE ON SUBMISSION ID\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mUnboundLocalError\u001b[0m: local variable 'reply_log' referenced before assignment"
     ]
    }
   ],
   "source": [
    "for submission in subreddit.hot(limit=5):\n",
    "    \n",
    "    if submission.id not in reply_log.Post_ID.values:  # Check that the bot hasn't already replied to this post\n",
    "        if submission.stickied: # Ignore stickied posts e.g. rules\n",
    "            continue\n",
    "        \n",
    "        # Download and save picture and thumbnail \n",
    "        if submission_is_image:\n",
    "            filename = submission.id + \".jpg\"\n",
    "            download(submission.url, filename, pic_dest)\n",
    "            download(submission.thumbnail, filename, thumbnail_dest)\n",
    "            \n",
    "            # Call CNN to classify and estimate calories\n",
    "            #classification, calories = food_CNN(pic_dest, filename)\n",
    "            classification, calories = (None, None)                \n",
    "                \n",
    "            # Reply to post and generate new log entry\n",
    "            text = \"Hi\" # TODO: learn to format \n",
    "            new_reply = send_reply(text, submission, classification, calories)\n",
    " \n",
    "            # Save reply in dataframe and csv file\n",
    "            save_reply(new_reply, reply_log, csv_file)\n",
    "            \n",
    "        else:\n",
    "            print(\"Already replied to post\")\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reply score loop function\n",
    "#### Scans previous replies and records score for any reply made more than 24 hours ago"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in reply_log.index:   \n",
    "    \n",
    "    elapsed_hours = (time.time() - reply_log['Reply_Date'][i]) / 3600 # 3600 seconds per hour \n",
    "    saved_reply_score = reply_log['Reply_Score'][i]\n",
    "    \n",
    "    if saved_reply_score == None and elapsed_hours >= 24:\n",
    "        reply = reddit.comment(reply_log['Bot_Reply'][i])\n",
    "        score_update = reply.score\n",
    "        reply_log['Reply_Score'][i] = score_update\n",
    "        reply_log.to_csv(csv_file, encoding='utf-8', index=False) # Save dataframe as .csv file \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}