DataPreprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "CROHME_PATH = './data/crohme/'\n",
    "EMNIST_PATH = './data/emnist/'\n",
    "\n",
    "test_processed = EMNIST_PATH + 'processed_balanced_test.csv'\n",
    "test_raw = EMNIST_PATH + 'emnist-balanced-test.csv'\n",
    "\n",
    "train_processed = EMNIST_PATH + 'processed_balanced_train.csv'\n",
    "train_raw = EMNIST_PATH + 'emnist-balanced-train.csv'\n",
    "\n",
    "mapping_processed = EMNIST_PATH + 'processed-mapping.csv'\n",
    "mapping_raw = EMNIST_PATH + 'emnist-balanced-mapping.txt'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Remove duplicated images from CROHME dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import imageio\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "# source - https://www.kaggle.com/xainano/handwrittenmathsymbols/discussion/85277\n",
    "def deletefromfolder(path):\n",
    "    datadir = path\n",
    "    print('Directory:', datadir)\n",
    "\n",
    "    rmmap = dict()\n",
    "    total = 0\n",
    "    repeatcnt = 0\n",
    "    for root, directories, filenames in os.walk(datadir):\n",
    "        for filename in filenames:\n",
    "            total += 1\n",
    "            if filename.endswith('.jpg') and not filename.startswith('._'):\n",
    "                filei = os.path.join(root, filename)\n",
    "                imi = imageio.imread(filei)\n",
    "                npi = np.asarray(imi).reshape(1, -1).reshape((2025, ))\n",
    "                idf = npi.tolist()\n",
    "                for i in range(len(idf)):\n",
    "                    idf[i] = str(idf[i])\n",
    "                strlist = ''.join(idf)\n",
    "\n",
    "                if strlist in rmmap.keys():\n",
    "                    repeatcnt += 1\n",
    "                    rmmap[strlist].append(filename)\n",
    "                else:\n",
    "                    rmmap[strlist] = list()\n",
    "\n",
    "    #for key in rmmap:\n",
    "    #    print(rmmap[key])\n",
    "    print('Repeat/Total: {}/{}'.format(repeatcnt, total))\n",
    "    for key in rmmap:\n",
    "        for item in rmmap[key]:\n",
    "            os.remove(os.path.join(datadir, item))\n",
    "\n",
    "dirlist = os.listdir(CROHME_PATH)\n",
    "for item in dirlist:\n",
    "    deletefromfolder(os.path.join(images_path,item))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Mapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "extra_chars = ['(', ')', '+', '-', '=']\n",
    "df = pd.read_csv(mapping_raw, sep = ' ', header=None, names=[\"id\", \"code\"])\n",
    "chars = []\n",
    "for x in df.code:\n",
    "    chars.append(chr(x))\n",
    "\n",
    "df[\"char\"] = chars\n",
    "nextId = df.shape[0]\n",
    "for i in range(nextId, nextId + len(extra_chars)):\n",
    "    c = extra_chars[i - nextId]\n",
    "    df.loc[i] = [i, ord(c), c]\n",
    "    \n",
    "df.to_csv(mapping_processed, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## EMNIST data preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mirror(X):\n",
    "    res = np.zeros(X.shape)\n",
    "    n = 28\n",
    "    for r in range(n, n**2 + 1, n):\n",
    "        l = r - n\n",
    "        for k in range(l, r):\n",
    "            index = l + (r - k - 1)\n",
    "            res[:,k] = X[:,index]\n",
    "            \n",
    "    return res        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rotate_clockwise(X):\n",
    "    res = np.zeros(X.shape)\n",
    "    size = 28\n",
    "    k = 0\n",
    "    for i in reversed(range(size)):\n",
    "        j = i\n",
    "        while j < size**2:\n",
    "            res[:,k] = X[:,j]\n",
    "            k += 1\n",
    "            j += size\n",
    "\n",
    "    return res\n",
    "\n",
    "def rotate(X, times):\n",
    "    for i in range(times):\n",
    "        X = rotate_clockwise(X)\n",
    "    \n",
    "    return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: remove one confusing character from pairs such as \"0\" and \"O\", \"I\", \"l\" and \"1\"\n",
    "def process_data(file_from, file_to):\n",
    "    label = 'label'\n",
    "    names = [label] + [\"px\" + str(i) for i in range(784)]\n",
    "    data = pd.read_csv(file_from, header=None, names=names)\n",
    "    \n",
    "    Y_data = data[label]\n",
    "    X_data = data.drop(labels = [label], axis = 1)\n",
    "    \n",
    "    X_data = X_data / 255\n",
    "    X_data = np.where(X_data > 0.5, 1, 0)\n",
    "    X_data = rotate(X_data, times=3)\n",
    "    X_data = mirror(X_data)\n",
    "    \n",
    "    data = pd.DataFrame(X_data, columns=names[1:], dtype='int')    \n",
    "    data.insert(0, label, Y_data)        \n",
    "    data.to_csv(file_to, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "process_data(train_raw, train_processed)\n",
    "process_data(test_raw, test_processed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Image transformation to EMNIST format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from PIL import Image,ImageOps\n",
    "from random import sample "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(mapping_processed)\n",
    "char2code = {}\n",
    "for index, row in df.iterrows():\n",
    "    char2code[row['char']] = row['id']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "\n",
    "def img2emnist(filepath, char_code):\n",
    "    img = cv2.imread(filepath, 0)\n",
    "    kernel = np.ones((3,3), np.uint8)\n",
    "    dilation = cv2.erode(img, kernel, iterations = 1)\n",
    "    \n",
    "    img = Image.fromarray(dilation).resize((28, 28))\n",
    "    inv_img = ImageOps.invert(img)\n",
    "    \n",
    "    flatten = np.array(inv_img).flatten()\n",
    "    flatten = flatten / 255\n",
    "    flatten = np.where(flatten > 0.5, 1, 0)\n",
    "    \n",
    "    csv_img = ','.join([str(num) for num in flatten])\n",
    "    csv_str = '{},{}'.format(char_code, csv_img)\n",
    "    return csv_str"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_size = 2400\n",
    "test_size = 400"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing \"(\" character...\n",
      "Processing \")\" character...\n",
      "Processing \"+\" character...\n",
      "Processing \"-\" character...\n",
      "Processing \"=\" character...\n"
     ]
    }
   ],
   "source": [
    "f_test = open(test_processed, 'a')\n",
    "f_train = open(train_processed, 'a')\n",
    "\n",
    "for c in extra_chars:\n",
    "    print('Processing \"{}\" character...'.format(c))\n",
    "    current_dir = CROHME_PATH + c + '/'\n",
    "    files = [f for r, d, f in os.walk(current_dir)]\n",
    "    subset = sample(files[0], train_size + test_size)\n",
    "    train_subset = subset[0:train_size]\n",
    "    test_subset = subset[train_size:train_size + test_size]\n",
    "    \n",
    "    for filename in train_subset:\n",
    "        csv_str = img2emnist(current_dir + filename, char2code[c])\n",
    "        print(csv_str, file=f_train)\n",
    "    \n",
    "    for filename in test_subset:\n",
    "        csv_str = img2emnist(current_dir + filename, char2code[c])\n",
    "        print(csv_str, file=f_test)\n",
    "\n",
    "f_test.close()\n",
    "f_train.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}