Caching_and_Performance.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n",
      "/usr/local/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "import redis\n",
    "import pymongo\n",
    "import json\n",
    "import pandas as pd\n",
    "from bson.json_util import dumps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_EXPIRE_DURATION = 24 * 3600 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def configure(max_redis_connections = 2): \n",
    "    client = pymongo.MongoClient()\n",
    "    globals()['client'] = client\n",
    "    cache_pool = redis.ConnectionPool(host = 'localhost', port = 6379, db = 0, max_connections = max_redis_connections)\n",
    "    globals()['cache_pool'] = cache_pool"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "configure()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_database_connection():  \n",
    "    '''\n",
    "    This method should be called to get a connection to the MongoDB server\n",
    "    '''\n",
    "    return globals()['client']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_cache_connection():  \n",
    "    '''\n",
    "    This method should be called to get a connection to the Redis server  \n",
    "    '''\n",
    "    return redis.StrictRedis(connection_pool = globals()['cache_pool'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def search(query):  \n",
    "    '''\n",
    "    The function perform the query search. \n",
    "    It first checks the cache to see whether the key exists. \n",
    "    If yes, then the query result corresponding to cache are retrieved from Redis, \n",
    "    else an query is fired to database.\n",
    "    '''\n",
    "    cache = get_cache_connection()\n",
    "    if cache.exists(query):\n",
    "        print('Exist in Cache')\n",
    "        results = lookup_cache_key(query)\n",
    "        return(results)\n",
    "    else:\n",
    "        print('Do not exist in Cache')\n",
    "        results = database_lookup(query)\n",
    "        return(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lookup_cache_key(cache_key):  \n",
    "    '''\n",
    "    Lookup a key in Redis cache\n",
    "    '''\n",
    "    cache = get_cache_connection()\n",
    "    ids = cache.get(cache_key)\n",
    "    return ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def database_lookup(query):  \n",
    "    '''\n",
    "    This will run when cache is not available\n",
    "    This function will search the results in MongoDB\n",
    "    and saved the cache to the Redis\n",
    "    '''\n",
    "    myquery = {\"content\":{\"$regex\":query,\"$options\" :'i'}}\n",
    "    client = get_database_connection()\n",
    "    db = client[\"tweet_database\"]\n",
    "    tweets = db.tweets_collection\n",
    "    twts = list(tweets.find(myquery))\n",
    "    save_query_to_cache(query, twts)\n",
    "    return twts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_query_to_cache(cache_key, twts):  \n",
    "    '''\n",
    "    Save the query result of into Redis cache\n",
    "    Also, given the expiry date for the duration\n",
    "    '''\n",
    "    cache = get_cache_connection()\n",
    "    serializedObj = dumps(twts)\n",
    "    cache.set(cache_key, serializedObj)\n",
    "    cache.expire(cache_key, MAX_EXPIRE_DURATION)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check the performance for searching a keyword with cache and without cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Do not exist in Cache\n",
      "--- 0.22595620155334473 seconds ---\n"
     ]
    }
   ],
   "source": [
    "## Run the query first time\n",
    "import time\n",
    "start_time = time.time()\n",
    "search(\"covid19\")\n",
    "print(\"--- %s seconds ---\" % (time.time() - start_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Exist in Cache\n",
      "--- 0.003137350082397461 seconds ---\n"
     ]
    }
   ],
   "source": [
    "## Run the same query again, change the time difference\n",
    "import time\n",
    "start_time = time.time()\n",
    "search(\"covid19\")\n",
    "print(\"--- %s seconds ---\" % (time.time() - start_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## For now we have delete the query from database\n",
    "redis.StrictRedis(connection_pool = globals()['cache_pool']).delete('covid19')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check the performance for searching a Hashtag with cache and without cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def database_lookup(query):  \n",
    "    '''\n",
    "    This will run when cache is not available\n",
    "    This function will search the results in MongoDB\n",
    "    and saved the cache to the Redis\n",
    "    '''\n",
    "    myquery = {\"content\":{\"$regex\":query,\"$options\" :'i'}}\n",
    "    client = get_database_connection()\n",
    "    db = client[\"tweet_database\"]\n",
    "    tweets = db.tweets_collection\n",
    "    twts = list(tweets.find(myquery))\n",
    "    save_query_to_cache(query, twts)\n",
    "    return twts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Do not exist in Cache\n",
      "--- 0.1576838493347168 seconds ---\n"
     ]
    }
   ],
   "source": [
    "## Run the query first time\n",
    "import time\n",
    "start_time = time.time()\n",
    "search(\"#trump\")\n",
    "print(\"--- %s seconds ---\" % (time.time() - start_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Exist in Cache\n",
      "--- 0.003531932830810547 seconds ---\n"
     ]
    }
   ],
   "source": [
    "## Run the query first time\n",
    "import time\n",
    "start_time = time.time()\n",
    "search(\"#trump\")\n",
    "print(\"--- %s seconds ---\" % (time.time() - start_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check the performance for searching a username with cache and without cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def database_lookup(query):  \n",
    "    '''\n",
    "    This will run when cache is not available\n",
    "    This function will search the results in MongoDB\n",
    "    and saved the cache to the Redis\n",
    "    '''\n",
    "    myquery = {\"user_name\":query}\n",
    "    client = get_database_connection()\n",
    "    db = client[\"tweet_database\"]\n",
    "    tweets = db.tweets_collection\n",
    "    twts = list(tweets.find(myquery))\n",
    "    save_query_to_cache(query, twts)\n",
    "    return twts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Do not exist in Cache\n",
      "--- 0.021148681640625 seconds ---\n"
     ]
    }
   ],
   "source": [
    "## Run the query first time\n",
    "import time\n",
    "start_time = time.time()\n",
    "search(\"Howard\")\n",
    "print(\"--- %s seconds ---\" % (time.time() - start_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Exist in Cache\n",
      "--- 0.0027000904083251953 seconds ---\n"
     ]
    }
   ],
   "source": [
    "## Run the query second time\n",
    "import time\n",
    "start_time = time.time()\n",
    "search(\"Howard\")\n",
    "print(\"--- %s seconds ---\" % (time.time() - start_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}