results-append.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Cleaning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is the process followed to go from the generated runs to a clean dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import config as c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local_rwth/sw/python/3.7.3/x86_64/extra/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3049: DtypeWarning: Columns (3,50) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "data = pd.read_csv('data/runs/n_results.csv', index_col='run')\n",
    "drug_data = pd.read_csv('data/runs/drug_data.csv')\n",
    "#old_drug_data = pd.read_csv('data/runs/drug_data.csv')\n",
    "#new_drug_data = pd.read_csv('data/runs/new_drug.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We only use runs that were completed and for which we obtained a $ r^2 $ score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data[data['status']=='completed']\n",
    "data = data[data['r_2_mean'].notnull()]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We turn the time taken into a time delta and the start time into a datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['time'] = pd.to_timedelta(data['time'])\n",
    "data['started'] = pd.to_datetime(data['started'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We create a new column (`seconds`) that stores the seconds taken by a specific run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['seconds'] = data['time'].apply(lambda x:x.seconds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We create a new column (`datasets`) which contains the number of datasets used"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['datasets'] = data['gdsc']+data['ctrp']+data['ccle']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We drop runs that used no datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data[data['datasets']>0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "All runs before 2020-04-15 18:00:00 are corrupt so we drop them"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data[data['started'] > pd.to_datetime('2020-04-15 18:00:00')]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We drop all runs with IC quality below -0.95"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data[data['Quality']>-0.95]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Append"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This method allows us to attach the models created using `train.py` to the results dataframe. For this train should be ran using Guild AI. Here the directory to guild should be defined on the config file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from config import guild\n",
    "def append(data, run):\n",
    "    result = pd.DataFrame()\n",
    "    run = [i for i in run if i not in data['run'].values]\n",
    "    print(len(run))\n",
    "    for a in run:\n",
    "        f = subprocess.run(\"guild select \" +str(a), shell=True, capture_output=True)\n",
    "        d = pd.read_csv(guild+'runs/'+f.stdout.decode('utf-8')[:-1]+'/scores.csv').rename(columns={'Unnamed: 0':'Drug'})\n",
    "        d['run'] = [a for i in range(0, d.shape[0])]\n",
    "        result = pd.concat([result, d])\n",
    "        run.remove(a)\n",
    "        if len(run) % 20 == 0:\n",
    "            new = pd.concat([data, result])\n",
    "            print('Drug data length is {}'.format(len(new)))\n",
    "            print('{} runs left'.format(len(run)))\n",
    "            new.to_csv('full_drug_new.csv', index=False)\n",
    "    new = pd.concat([data, result])\n",
    "    print('Drug data length is {}'.format(len(new)))\n",
    "    new.to_csv('full_drug_new.csv', index=False)\n",
    "    return new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/rwthfs/rz/cluster/home/kz035322/Thesis/venv/lib/python3.7/site-packages/ipykernel_launcher.py:10: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n",
      "of pandas will change to not sort by default.\n",
      "\n",
      "To accept the future behavior, pass 'sort=False'.\n",
      "\n",
      "To retain the current behavior and silence the warning, pass 'sort=True'.\n",
      "\n",
      "  # Remove the CWD from sys.path while we load stuff.\n",
      "/rwthfs/rz/cluster/home/kz035322/Thesis/venv/lib/python3.7/site-packages/ipykernel_launcher.py:13: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n",
      "of pandas will change to not sort by default.\n",
      "\n",
      "To accept the future behavior, pass 'sort=False'.\n",
      "\n",
      "To retain the current behavior and silence the warning, pass 'sort=True'.\n",
      "\n",
      "  del sys.path[0]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Drug data length is 162600\n",
      "20 runs left\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/rwthfs/rz/cluster/home/kz035322/Thesis/venv/lib/python3.7/site-packages/ipykernel_launcher.py:17: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n",
      "of pandas will change to not sort by default.\n",
      "\n",
      "To accept the future behavior, pass 'sort=False'.\n",
      "\n",
      "To retain the current behavior and silence the warning, pass 'sort=True'.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Drug data length is 162687\n"
     ]
    }
   ],
   "source": [
    "k = append(data, ndata.index)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}