Mapbox Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import dask.dataframe as dd\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import dask\n",
    "\n",
    "from dask.distributed import Client\n",
    "from datetime import datetime\n",
    "\n",
    "client = Client(n_workers=8)\n",
    "\n",
    "# CADES configurationt\n",
    "# from dask_jobqueue import SLURMCluster\n",
    "# cluster = SLURMCluster(project='birthright', queue='high_mem_cd', cores=30, memory='150 GB', job_extra=[\"-N 1 -t 4:00:00\"], interface=\"ib0\")\n",
    "# client = Client(cluster)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>geography</th>\n",
       "      <th>agg_day_period</th>\n",
       "      <th>activity_index_total</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>USA248021</td>\n",
       "      <td>2020-03-01</td>\n",
       "      <td>0.017817</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>USA224005</td>\n",
       "      <td>2020-03-01</td>\n",
       "      <td>0.089171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>USA237035</td>\n",
       "      <td>2020-03-01</td>\n",
       "      <td>0.027586</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>USA205045</td>\n",
       "      <td>2020-03-01</td>\n",
       "      <td>0.030865</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>USA220149</td>\n",
       "      <td>2020-03-01</td>\n",
       "      <td>0.005965</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   geography agg_day_period  activity_index_total\n",
       "0  USA248021     2020-03-01              0.017817\n",
       "1  USA224005     2020-03-01              0.089171\n",
       "2  USA237035     2020-03-01              0.027586\n",
       "3  USA205045     2020-03-01              0.030865\n",
       "4  USA220149     2020-03-01              0.005965"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counties = dd.read_csv(\"data/mapbox/production_v02/county_daily_24_0/US/**/*.csv\", sep='|')\n",
    "counties.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_geography(row):\n",
    "    geog = row.geography\n",
    "    row['state_fips'] = geog[4:6]\n",
    "    row['county_fips'] = geog[-3:]\n",
    "    return row\n",
    "\n",
    "counties = counties.apply(split_geography, axis=1, meta={'geography': 'str', 'agg_day_period': 'object', 'activity_index_total': 'float64', 'state_fips': 'str', 'county_fips': 'str'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "state_fips = dd.read_csv(\"data/reference/state_fips.csv\", dtype={'state_fips': 'str'}).set_index('state_fips')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "counties = counties.join(state_fips[['state_name']], on='state_fips')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/Users/raac/Development/covid/mobility-analysis/data/output/Mabbox-State-Mobility.csv/0.part']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counties.groupby(['state_name', 'agg_day_period']).mean().reset_index().to_csv(\"data/output/Mabbox-State-Mobility.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "c2 = counties.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Metadata inference failed in `ge`.\n\nOriginal error is below:\n------------------------\nTypeError(\"'>=' not supported between instances of 'str' and 'datetime.datetime'\")\n\nTraceback:\n---------\n  File \"/Users/raac/Development/covid/mobility-analysis/venv/lib/python3.7/site-packages/dask/dataframe/utils.py\", line 172, in raise_on_meta_error\n    yield\n  File \"/Users/raac/Development/covid/mobility-analysis/venv/lib/python3.7/site-packages/dask/dataframe/core.py\", line 4660, in elemwise\n    meta = partial_by_order(*parts, function=op, other=other)\n  File \"/Users/raac/Development/covid/mobility-analysis/venv/lib/python3.7/site-packages/dask/utils.py\", line 1089, in partial_by_order\n    return function(*args2, **kwargs)\n  File \"/Users/raac/Development/covid/mobility-analysis/venv/lib/python3.7/site-packages/pandas/core/ops/common.py\", line 64, in new_method\n    return method(self, other)\n  File \"/Users/raac/Development/covid/mobility-analysis/venv/lib/python3.7/site-packages/pandas/core/ops/__init__.py\", line 529, in wrapper\n    res_values = comparison_op(lvalues, rvalues, op)\n  File \"/Users/raac/Development/covid/mobility-analysis/venv/lib/python3.7/site-packages/pandas/core/ops/array_ops.py\", line 247, in comparison_op\n    res_values = comp_method_OBJECT_ARRAY(op, lvalues, rvalues)\n  File \"/Users/raac/Development/covid/mobility-analysis/venv/lib/python3.7/site-packages/pandas/core/ops/array_ops.py\", line 57, in comp_method_OBJECT_ARRAY\n    result = libops.scalar_compare(x.ravel(), y, op)\n  File \"pandas/_libs/ops.pyx\", line 96, in pandas._libs.ops.scalar_compare\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m~/Development/covid/mobility-analysis/venv/lib/python3.7/site-packages/dask/dataframe/utils.py\u001b[0m in \u001b[0;36mraise_on_meta_error\u001b[0;34m(funcname, udf)\u001b[0m\n\u001b[1;32m    171\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 172\u001b[0;31m         \u001b[0;32myield\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    173\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Development/covid/mobility-analysis/venv/lib/python3.7/site-packages/dask/dataframe/core.py\u001b[0m in \u001b[0;36melemwise\u001b[0;34m(op, *args, **kwargs)\u001b[0m\n\u001b[1;32m   4659\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mraise_on_meta_error\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfuncname\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 4660\u001b[0;31m             \u001b[0mmeta\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpartial_by_order\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mparts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunction\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mother\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mother\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   4661\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Development/covid/mobility-analysis/venv/lib/python3.7/site-packages/dask/utils.py\u001b[0m in \u001b[0;36mpartial_by_order\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m   1088\u001b[0m         \u001b[0margs2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minsert\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1089\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mfunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1090\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Development/covid/mobility-analysis/venv/lib/python3.7/site-packages/pandas/core/ops/common.py\u001b[0m in \u001b[0;36mnew_method\u001b[0;34m(self, other)\u001b[0m\n\u001b[1;32m     63\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 64\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mother\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     65\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Development/covid/mobility-analysis/venv/lib/python3.7/site-packages/pandas/core/ops/__init__.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(self, other)\u001b[0m\n\u001b[1;32m    528\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 529\u001b[0;31m         \u001b[0mres_values\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcomparison_op\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    530\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Development/covid/mobility-analysis/venv/lib/python3.7/site-packages/pandas/core/ops/array_ops.py\u001b[0m in \u001b[0;36mcomparison_op\u001b[0;34m(left, right, op)\u001b[0m\n\u001b[1;32m    246\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mis_object_dtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlvalues\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 247\u001b[0;31m         \u001b[0mres_values\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcomp_method_OBJECT_ARRAY\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrvalues\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    248\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Development/covid/mobility-analysis/venv/lib/python3.7/site-packages/pandas/core/ops/array_ops.py\u001b[0m in \u001b[0;36mcomp_method_OBJECT_ARRAY\u001b[0;34m(op, x, y)\u001b[0m\n\u001b[1;32m     56\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 57\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlibops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mscalar_compare\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mravel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     58\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/ops.pyx\u001b[0m in \u001b[0;36mpandas._libs.ops.scalar_compare\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: '>=' not supported between instances of 'str' and 'datetime.datetime'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-169-5d85f3c9ff49>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[0md\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdatetime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoday\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mtimedelta\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdays\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m \u001b[0mc2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mc2\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mc2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdate\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0md\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     22\u001b[0m \u001b[0;31m# c2.groupby(['geography']).apply(lambda x: x.activity_index_total.transform(print_it), meta=('trend', 'object')).reset_index().head()\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m \u001b[0mc2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgroupby\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'geography'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mactivity_index_total\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhead\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Development/covid/mobility-analysis/venv/lib/python3.7/site-packages/dask/dataframe/core.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(self, other)\u001b[0m\n\u001b[1;32m   1393\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mother\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0melemwise\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mother\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1394\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1395\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mother\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0melemwise\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mother\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1396\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1397\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mrolling\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwindow\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmin_periods\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcenter\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwin_type\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Development/covid/mobility-analysis/venv/lib/python3.7/site-packages/dask/dataframe/core.py\u001b[0m in \u001b[0;36melemwise\u001b[0;34m(op, *args, **kwargs)\u001b[0m\n\u001b[1;32m   4658\u001b[0m         ]\n\u001b[1;32m   4659\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mraise_on_meta_error\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfuncname\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 4660\u001b[0;31m             \u001b[0mmeta\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpartial_by_order\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mparts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunction\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mother\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mother\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   4661\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4662\u001b[0m     \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnew_dd_object\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgraph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmeta\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdivisions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/Cellar/python/3.7.7/Frameworks/Python.framework/Versions/3.7/lib/python3.7/contextlib.py\u001b[0m in \u001b[0;36m__exit__\u001b[0;34m(self, type, value, traceback)\u001b[0m\n\u001b[1;32m    128\u001b[0m                 \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    129\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 130\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgen\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mthrow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraceback\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    131\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    132\u001b[0m                 \u001b[0;31m# Suppress StopIteration *unless* it's the same exception that\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Development/covid/mobility-analysis/venv/lib/python3.7/site-packages/dask/dataframe/utils.py\u001b[0m in \u001b[0;36mraise_on_meta_error\u001b[0;34m(funcname, udf)\u001b[0m\n\u001b[1;32m    191\u001b[0m         )\n\u001b[1;32m    192\u001b[0m         \u001b[0mmsg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmsg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\" in `{0}`\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfuncname\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mfuncname\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m\"\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrepr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 193\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    194\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    195\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Metadata inference failed in `ge`.\n\nOriginal error is below:\n------------------------\nTypeError(\"'>=' not supported between instances of 'str' and 'datetime.datetime'\")\n\nTraceback:\n---------\n  File \"/Users/raac/Development/covid/mobility-analysis/venv/lib/python3.7/site-packages/dask/dataframe/utils.py\", line 172, in raise_on_meta_error\n    yield\n  File \"/Users/raac/Development/covid/mobility-analysis/venv/lib/python3.7/site-packages/dask/dataframe/core.py\", line 4660, in elemwise\n    meta = partial_by_order(*parts, function=op, other=other)\n  File \"/Users/raac/Development/covid/mobility-analysis/venv/lib/python3.7/site-packages/dask/utils.py\", line 1089, in partial_by_order\n    return function(*args2, **kwargs)\n  File \"/Users/raac/Development/covid/mobility-analysis/venv/lib/python3.7/site-packages/pandas/core/ops/common.py\", line 64, in new_method\n    return method(self, other)\n  File \"/Users/raac/Development/covid/mobility-analysis/venv/lib/python3.7/site-packages/pandas/core/ops/__init__.py\", line 529, in wrapper\n    res_values = comparison_op(lvalues, rvalues, op)\n  File \"/Users/raac/Development/covid/mobility-analysis/venv/lib/python3.7/site-packages/pandas/core/ops/array_ops.py\", line 247, in comparison_op\n    res_values = comp_method_OBJECT_ARRAY(op, lvalues, rvalues)\n  File \"/Users/raac/Development/covid/mobility-analysis/venv/lib/python3.7/site-packages/pandas/core/ops/array_ops.py\", line 57, in comp_method_OBJECT_ARRAY\n    result = libops.scalar_compare(x.ravel(), y, op)\n  File \"pandas/_libs/ops.pyx\", line 96, in pandas._libs.ops.scalar_compare\n"
     ]
    }
   ],
   "source": [
    "from scipy.stats import linregress\n",
    "from datetime import datetime, timedelta\n",
    "\n",
    "def compute_regression(values):\n",
    "    y = np.array(values)\n",
    "    x = np.arange(len(values))\n",
    "    slope, intercept, r_value, p_value, std_err = linregress(x, y)\n",
    "    return slope\n",
    "\n",
    "def filter_dates(row):\n",
    "    return  #.apply(np.array).apply(compute_regression)\n",
    "\n",
    "def print_it(vals):\n",
    "    print(len(vals))\n",
    "    return vals.sum()\n",
    "\n",
    "c2['date'] = c2['agg_day_period'].apply(lambda x: datetime.strptime(x, '%Y-%m-%d').date(), meta=('date', 'object'))\n",
    "c2 = c2.drop(['agg_day_period'], axis=1)\n",
    "d = datetime.today() - timedelta(days=3)\n",
    "\n",
    "c2 = c2[c2.date >= d]\n",
    "# c2.groupby(['geography']).apply(lambda x: x.activity_index_total.transform(print_it), meta=('trend', 'object')).reset_index().head()\n",
    "c2.groupby(['geography']).activity_index_total.sum().head()\n",
    "\n",
    "# c2.groupby(['geography']).apply(lambda x: x.activity_index_total.rolling(7).sum()).reset_index().head()\n",
    "# c2['hello'] = c2.groupby(['geography']).activity_index_total.transform(lambda x: 1).head() #.rolling(7).apply(lambda x: len(x)).head()\n",
    "# s = c2.groupby(['geography']).activity_index_total.sum()\n",
    "#ID = AID.set_index('AID').drop('ANumOfF', axis=1).join(s).reset_index()\n",
    "# c2['summed'] = c2.geography.map(s)\n",
    "# c2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>geography</th>\n",
       "      <th>agg_day_period</th>\n",
       "      <th>activity_index_total</th>\n",
       "      <th>state_fips</th>\n",
       "      <th>county_fips</th>\n",
       "      <th>state_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>USA248021</td>\n",
       "      <td>2020-03-01</td>\n",
       "      <td>0.017817</td>\n",
       "      <td>48</td>\n",
       "      <td>021</td>\n",
       "      <td>Texas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>USA224005</td>\n",
       "      <td>2020-03-01</td>\n",
       "      <td>0.089171</td>\n",
       "      <td>24</td>\n",
       "      <td>005</td>\n",
       "      <td>Maryland</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>USA237035</td>\n",
       "      <td>2020-03-01</td>\n",
       "      <td>0.027586</td>\n",
       "      <td>37</td>\n",
       "      <td>035</td>\n",
       "      <td>North Carolina</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>USA205045</td>\n",
       "      <td>2020-03-01</td>\n",
       "      <td>0.030865</td>\n",
       "      <td>05</td>\n",
       "      <td>045</td>\n",
       "      <td>Arkansas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>USA220149</td>\n",
       "      <td>2020-03-01</td>\n",
       "      <td>0.005965</td>\n",
       "      <td>20</td>\n",
       "      <td>149</td>\n",
       "      <td>Kansas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   geography agg_day_period  activity_index_total state_fips county_fips  \\\n",
       "0  USA248021     2020-03-01              0.017817         48         021   \n",
       "1  USA224005     2020-03-01              0.089171         24         005   \n",
       "2  USA237035     2020-03-01              0.027586         37         035   \n",
       "3  USA205045     2020-03-01              0.030865         05         045   \n",
       "4  USA220149     2020-03-01              0.005965         20         149   \n",
       "\n",
       "       state_name  \n",
       "0           Texas  \n",
       "1        Maryland  \n",
       "2  North Carolina  \n",
       "3        Arkansas  \n",
       "4          Kansas  "
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "    "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}