sw_07_large_network_complexity.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "2f443f71",
   "metadata": {},
   "source": [
    "# Explore how to create and compute large networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c4350417",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import networkx as nx\n",
    "import pickle \n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9bd20c06",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(12053805, 14)\n",
      "Index(['user', 'duration', 'ini_dat', 'lon_medoid', 'lat_medoid', 'GEOID',\n",
      "       'home_lon_med', 'home_lat_med', 'GEOID_home', 'quant', 'fsq_id', 'cat',\n",
      "       'distPOI', 'disthome'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "with open('./data/boston_activities/raw/boston_activities.pickle', 'rb') as f:\n",
    "    df = pickle.load(f)\n",
    "    \n",
    "print(df.shape)\n",
    "print(df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "53337c33",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "699.2867678403854"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get size 'MB'\n",
    "sys.getsizeof(df) / 8 / 1024 / 1024 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "dc4005c6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "72803\n",
      "3115\n"
     ]
    }
   ],
   "source": [
    "# nodes for people and places\n",
    "user_nodes = list(np.unique(df.user))\n",
    "geoid_nodes = list(np.unique(df.GEOID))\n",
    "\n",
    "print(len(user_nodes))\n",
    "print(len(geoid_nodes)) # the graph size is HUGE."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c6624a1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# people-place matrix\n",
    "weight_df = df.groupby(['user','GEOID']).count().iloc[:, 0]\n",
    "people_place_df = weight_df.unstack()\n",
    "# people_place_w.values[people_place_w.isna().values] = 0.0\n",
    "people_place_m = people_place_df.values\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bc4cd59b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Kernel DEAD through this step. \n",
    "people_m = np.matmul(people_place_df.values, people_place_df.T.values)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d8fab8b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# sparse \n",
    "from scipy import sparse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c9814330",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7.62939453125e-06"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# size of the compressed matrix in scipy!!! Crap! \n",
    "sys.getsizeof(sparse.csr_matrix(people_place_m)) / 8 / 1024 / 1024"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "c7691d7e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3115, 72803)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "people_place_m.T.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b5b95c02",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(72803, 3115)\n",
      "(3115, 72803)\n"
     ]
    }
   ],
   "source": [
    "print(people_place_m.shape)\n",
    "print(people_place_m.T.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ec1c853",
   "metadata": {},
   "outputs": [],
   "source": [
    "# it computed for two hours. No result...\n",
    "sparse.csr_matrix(people_place_m).dot(sparse.csr_matrix(people_place_m.T))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "8659bd2b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3115, 72803)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "people_place_m.T.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2fdeedcb",
   "metadata": {},
   "outputs": [],
   "source": [
    "people_array\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a20e0e2c",
   "metadata": {},
   "outputs": [],
   "source": [
    "people_m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f45fdee7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3115, 72803)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "people_place_w.T.values.shape "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2c8bcd8f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "217.308746636"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get size of in python\n",
    "sys.getsizeof(people_place_w) * 1.192*(10)**(-7)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "70b1277a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# test\n",
    "large_m = np.random.rand(72803, 72803)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "63efa9bd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5054.73787689209"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sys.getsizeof(large_m) / 8 / 1024 / 1024\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0f7a4e71",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "user                                                              GEOID       \n",
       "0000ff45f7f170db960e4e601167975f7559c5be147d69e75460123855ab0eab  250173684002    1\n",
       "000277100d5593fec35a151e228f6a485210a3fa87cda78d79c465bbebb6e71e  250173734002    1\n",
       "                                                                  250173738003    1\n",
       "                                                                  250173739001    2\n",
       "                                                                  250214011003    1\n",
       "                                                                                 ..\n",
       "fffe58b2fb58c5940b85ba55b07eb4d1d11b36185a4afb2f64a95d49a2025d6f  330151061011    1\n",
       "                                                                  330151061012    1\n",
       "                                                                  330151061021    1\n",
       "                                                                  330151061023    1\n",
       "                                                                  330151075002    1\n",
       "Name: duration, Length: 1446228, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "df.groupby(['user','GEOID']).count().iloc[:, 0]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "58609123",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19b2724f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a6e99edf",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a1ebb2f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "90ca63c2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d5f60a86",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0910c151",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e95d5db4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "84a21e4f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1d1ac793",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b66988a2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9781ec0b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}