Create paired image dataset from simulation results.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import necessary libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-24T09:34:28.352543Z",
     "start_time": "2019-03-24T09:34:28.240658Z"
    }
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "import glob\n",
    "import PIL\n",
    "from PIL import Image\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Set input and output image folders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "archetypes_folder = 'insert/folder/with/input/heightmaps/here'\n",
    "daylightMaps_folder = 'insert/folder/with/input/daylight/maps/here'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-24T09:34:28.919622Z",
     "start_time": "2019-03-24T09:34:28.885657Z"
    }
   },
   "outputs": [],
   "source": [
    "archetypes = glob.glob(archetypes_folder + '/*.png', recursive=False)\n",
    "archetypes.sort(key=solar_chars) #lambda x: os.path.getmtime(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-24T09:34:28.919622Z",
     "start_time": "2019-03-24T09:34:28.885657Z"
    }
   },
   "outputs": [],
   "source": [
    "daylightMaps = glob.glob(daylightMaps_folder + '/*.png', recursive=False)\n",
    "daylightMaps.sort(key=solar_chars) #lambda x: os.path.getmtime(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-24T09:34:29.741999Z",
     "start_time": "2019-03-24T09:34:29.734007Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1374, 1401, 1401)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(archetypes), len(daylightMaps)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Make sure only images with valid pairs are included"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-24T09:34:31.605246Z",
     "start_time": "2019-03-24T09:34:31.598254Z"
    }
   },
   "outputs": [],
   "source": [
    "arch_ids = []\n",
    "for image in archetypes:\n",
    "    arch_ids.append(image.split('\\\\')[-1].split('.png')[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-24T09:34:31.844657Z",
     "start_time": "2019-03-24T09:34:31.837665Z"
    }
   },
   "outputs": [],
   "source": [
    "day_ids = []\n",
    "for image in daylightMaps:\n",
    "    day_ids.append(image.split('\\\\')[-1].split('.png')[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-24T09:34:34.105381Z",
     "start_time": "2019-03-24T09:34:33.990000Z"
    }
   },
   "outputs": [],
   "source": [
    "both = set(arch_ids).intersection(day_ids)\n",
    "\n",
    "indices_A = [arch_ids.index(x) for x in both]\n",
    "indices_B = [day_ids.index(x) for x in both]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-24T09:34:34.745394Z",
     "start_time": "2019-03-24T09:34:34.741399Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1374, 1374, 1374)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(indices_A), len(indices_B)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create folders to store your paired dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-24T09:34:35.536624Z",
     "start_time": "2019-03-24T09:34:35.533627Z"
    }
   },
   "outputs": [],
   "source": [
    "save_folder = 'your/location/here'\n",
    "\n",
    "paths = [folder + '/train', folder + '/val', folder + '/test']\n",
    "\n",
    "for path in paths:\n",
    "    try:\n",
    "        os.makedirs(path)\n",
    "    except OSError:\n",
    "        print (\"Creation of the directory %s failed\" % path)\n",
    "    else:\n",
    "        print (\"Successfully created the directory %s\" % path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get train, test, and validation split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-24T09:34:48.071297Z",
     "start_time": "2019-03-24T09:34:48.066302Z"
    }
   },
   "outputs": [],
   "source": [
    "# Split in training, test, and validation sets\n",
    "\n",
    "total_imgs = len(indices_A)\n",
    "test_pct = 0.1 # 10% for test set\n",
    "val_pct = 0.01 # 1% for valudation\n",
    "\n",
    "indices = np.arange(0, total_imgs, 1)\n",
    "rand_idx = np.random.permutation(indices)\n",
    "\n",
    "test_cut = int(test_pct*total_imgs)\n",
    "val_cut = int(total_imgs*val_pct)\n",
    "cut = test_cut + val_cut\n",
    "\n",
    "train_idx = np.setdiff1d(indices, rand_idx[:cut])\n",
    "test_idx = rand_idx[0:test_cut]\n",
    "val_idx = rand_idx[test_cut:cut]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-24T09:44:40.795835Z",
     "start_time": "2019-03-24T09:35:22.222207Z"
    }
   },
   "outputs": [],
   "source": [
    "for i in range(len(indices_A)):\n",
    "    im_list = [archetypes[indices_A[i]], daylightMaps[indices_B[i]]]\n",
    "    imgs = [ PIL.Image.open(i) for i in im_list ]\n",
    "    min_shape = sorted( [(np.sum(i.size), i.size ) for i in imgs])[0][1]\n",
    "    #horizontal stack\n",
    "    imgs_comb = np.hstack( (np.asarray( i.resize(min_shape) ) for i in imgs ) )\n",
    "    imgs_comb = PIL.Image.fromarray( imgs_comb)\n",
    "    if i in train_idx:\n",
    "        imgs_comb.save( '{}.png'.format(save_folder + 'train\\\\' + archetypes[i].split('\\\\')[-1][:-4]) )\n",
    "    elif i in test_idx:\n",
    "        imgs_comb.save( '{}.png'.format(save_folder + 'test\\\\' + archetypes[i].split('\\\\')[-1][:-4]) )\n",
    "    else:\n",
    "        imgs_comb.save( '{}.png'.format(save_folder + 'val\\\\' + archetypes[i].split('\\\\')[-1][:-4]) )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}