0.merge_original_data_with_missing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Merge the original data with the missing data, then store in directory \"Merged\"\n",
    "\n",
    "Input:\n",
    "* Transactions/patients_1.txt ~ patients_50.txt\n",
    "* Missing/patients_1.txt ~ patients_50.txt\n",
    "\n",
    "Output:\n",
    "* Merged/patient_1.txt ~ patient_50.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import pickle\n",
    "\n",
    "def load_data(filename):\n",
    "    with open(filename, 'rb') as fin:\n",
    "        return pickle.load(fin)\n",
    "    \n",
    "def save_data(obj, filename):\n",
    "    with open(filename, 'wb+') as fout:\n",
    "        pickle.dump(obj, fout)\n",
    "        \n",
    "import os\n",
    "root_path = 'C:/Users/yuanl4/Documents/MelbDatathon2017/'\n",
    "\n",
    "def check_directory(directory):\n",
    "    assert os.path.isdir(directory), \"Directory doesn't exist: %s\" % directory\n",
    "\n",
    "check_directory(root_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C:/Users/yuanl4/Documents/MelbDatathon2017/Merged has been created.\n"
     ]
    }
   ],
   "source": [
    "check_directory(root_path + 'Transactions')\n",
    "check_directory(root_path + 'Missing')\n",
    "\n",
    "if os.path.isdir(root_path + 'Merged') is False:\n",
    "    os.makedirs(root_path + 'Merged')\n",
    "    print(root_path + 'Merged', 'has been created.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def merge_patients(i):\n",
    "    df1 = pd.read_csv(root_path + 'Transactions/patients_%d.txt' % i, sep='\\t')\n",
    "    df2 = pd.read_csv(root_path + 'Missing/missing_patients_%d.txt' % i, sep='\\t')\n",
    "    df = pd.concat([df1, df2])\n",
    "    df.drop_duplicates(inplace=True)\n",
    "    df.sort_values(['Patient_ID', 'Dispense_Week'], inplace=True)\n",
    "    df.set_index(np.arange(df.shape[0]), inplace=True)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Very slow... because we store everything, but only do this once."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 finished.\n",
      "Wall time: 14min 42s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "for i in range(1, 51):\n",
    "    print(i, end=' ')\n",
    "    merge_patients(i).to_csv(root_path + 'Merged/patient_%d.txt'%i, index=False, sep='\\t')\n",
    "print('finished.')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}