data_process.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Make Parsing and Preprocessing Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "metadata": {}
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "DATA_DIR = os.path.join(\"english\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "metadata": {}
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('english\\\\Durian',\n",
       " [],\n",
       " ['aligned.swc',\n",
       "  'audio.ogg',\n",
       "  'audiometa.txt',\n",
       "  'info.json',\n",
       "  'wiki.html',\n",
       "  'wiki.txt',\n",
       "  'wiki.xml'])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "next(os.walk(os.path.join(DATA_DIR, 'Durian')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "metadata": {}
   },
   "outputs": [],
   "source": [
    "# audio = AudioSegment.from_ogg('english/Durian/audio.ogg')\n",
    "# print(audio.channels, audio.frame_rate)\n",
    "# swc = SWC('english/Durian/aligned.swc')\n",
    "# split_clip(audio, swc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "metadata": {}
   },
   "outputs": [],
   "source": [
    "# clips = load_utterances(None) # 50 minutes to load 100,000 clips\n",
    "# print(len(clips))\n",
    "# clips[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Specifically Audio Manipulation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "                    <audio controls>\n",
       "                        <source src=\"data:audio/mpeg;base64,SUQzBAAAAAAAI1RTU0UAAAAPAAADTGF2ZjU4LjI3LjEwMwAAAAAAAAAAAAAA//NYwAAAAAAAAAAAAEluZm8AAAAPAAAAOgAAGSwACw8PFBQYHBwgICUlKS0tMjI2Ojo+PkNDR0tLUFBUVFhcXGFhZWlpbm5ycnZ6en9/g4eHjIyQkJSZmZ2doaGlqqqurrK3t7u7v7/DyMjMzNDQ1dnZ3d3h5ubq6u7u8/f3+/v/AAAAAExhdmM1OC41MgAAAAAAAAAAAAAAACQDQAAAAAAAABksULlCpwAAAAAAAAAAAAAA//M4xAAS+gaNR0EYAAFgIBAAGY2uY3xjGMYxjRERC//d3P3d//dEL6/ufoidRERP9z/+uIRE9z/0L/d3d3OIBgYGBgYGLNAMDc/gg7AgIAgCAIA/EBz/oDC67bba6622WSRtxy2gBZlNDRlA//M4xBMZ6vMCX4koA5gjNJiaLV7RjRUYoicQd3HFiwoKuUcLuQWscSRSymqgpS7DkE0cVBUJIhJHq6qahlnqVLUa0yvO+xkSq0Q5yCzvZs9KvZdeZCI7y0lHt//XrTp4tj2OOyRx9KUqGQdB//M4xAoWuTraV8Z4AnySMs20aFJxWxZ3m00LUi5peJWBhmcvvMPd7Vlizq4mkave+BrWb4+6Yz/jFrY1ecQiw13NjXzlzkjgRB8Jg2NVE6kM9alpqaUeqBZ7//tcKG+764S6PViyEgqwNAwJ//M4xA4XeqrJjjIFQs8SmZQFBdFCmQEEUI1ilob8mEoLh4mWKAndkxWSCqgVgMtpGS79Vr8z2md17Hv3cpCGhnU9W4wpytI7PsqXZ3zO+azqpEXbaHgedf/vmZUxluuukeoFXjsLhZWTKqTL//M4xA8YenrGVkjFbGYp0nLMwm0v8mtaVo9N5SsJJZDEtYvVXR0uNNkKbJlWuHVFNRVj9ne0y9//L97SCQtiJEzM2ZMUcIegy0VTo8uVvskhLrd7CIZgss9uJsVZ/6csjsj76lCnRiMHtWND//M4xAwWqzrSNkBHltEUVHi7qaOpTSVRkSLyVNNV7MiFNGDFiztdRtAFq7HtFnubPHVtMSPbsrN5eJh6f37bZmdBDM7JEs8Ep6IRDNDmkpXuf8jEv1P//uZRxB1mmIXWR1gHcOgo6sRuMBqb//M4xBAU6OLO9kjOlGUeILpkHEIfXdFKpmQ5pMz0I3hvcC7oORiBRAtKDQdWTSIlxRDWgffdhocQYLyLBq2/DZqibDKA5FkFzOaSd4xtirR62//321tklwHuqlVDjmw3E1HubRUGadOKqPFd//M4xBsUyTcKXjDM5gBvDIqz6/SRzCxDyQibxEst+9RX+zTfTijTTlqLgYdWAiLXrKE4apAKQ5f81SQYaKg4DYuA3oVK1VL9o5ZShVA/ZO00LOHpGdWlYLh3zZvy8NOuc+IY8NnL2zY8Ugjm//M4xCYUqYrOVjJFRLpWrj8RXj8nqestsq7/s9yKprCnAs162vCB5gPoBovAK/vVLAU8RBpIrzw/cqv6VoS4IVQVNJOsYRzt7UddjdzV2Kvtt91LK+O5qeOt0JCe7bNLEKXtI1JGZKZ0eXOz//M4xDIU2gLFjlhHbhE3Nlmdu+JGzauPsjhTCBEKLJlFG2mQLOJ+0UXzGrRrddWy3/WyQoAvAbLKHYgo6EEAQYAKaqgkh8uMjxcKFNXcGtcbThhUEtRvQxUu6rux9yhUeTSJetQZidYvuNiw//M4xD0U0SrOXhpFQGDJABjUmxQG24Sa5rzLKnmPiZrkT2UVkd81caAoB5Jo2009Gjs0jjbIp+tEx0dIalik32p7FajOt6CtI73A4wClzAUx7WlPwaerxqLFFHmr6PaOTkAcAZ9Z5Pc7/v7f//M4xEgVSPLCTjDQxfj3LF7GPX55zvN/zv2qkV0iSRBg6Z7ya3L0yZBiNLeLCftG1yfplXuRRWpq+YQdGBjFSwQUxZxc0LpqdzfOfJbDyVTLZF5cd78tbi2RltlEJ2CEBvWvhCIxExwZCq/H//M4xFEUoeK+TDDM3DqaNYZW0qayBXDzNBI81YPBy305KdpYdXu2ozDDmvNi1HdZUnL/NShkS0s4erWw1zKJKawy+DMEB4SMRYUg6EAEE0OPrWGFXcCix5wiVPvb0nEXX/2rRYZ3aHZZGg3g//M4xF0VAXrK9jBHAC7RYULKRUmbboGn3qtOY5IwpzyNnonnNBa9DD0iNvyhwQfw0Z64c1HgjPoEWdAYbQcaj4DSuOEYYKw8w4TcpLlVVAci/pDDpbRVNTZ5lmZK0KwHhjxRd5hGyiN2cZQi//M4xGgUiQrXHhGGPBclI/pHaHDL7803pR3FJiTKP079BE3WNStcxRSEpN+eHsvP4ek1jEf+3fn0sWAQQDaiZAY/oqTQdsJuacy4DJxjovSqZFll6jVQBXjpnBy8OgwgLA7NZBUEY2DJP5RB//M4xHQWCdLPFjJGrAbMMr0t1Js522lPBkFQKgwMcQxmCcQnC3jkStEv/kxnq8tsJ8qRLChDCZ5Jo5zxSS+kuKybBEaY1rAVnXGf/roOaXpgZx2gYOPeQoseceTZiaksTS2U2pdXOmv8UUXj//M4xHoWocrK9jJGoBGTyBTNTuOwsTOtxIlYyikhFSh5PXerlXLowqwWp1cPD3EgT5CyQQLa4gBwXEELi1ISNhhAIgEKBQNraYXMtKqoudb6R41u7OmpABfHtAXZQHxCPKvPhIOPGZ+2lqLz//M4xH4ZQfq2JjBHQDHg7+HTxV4zUYhCNpoFN+YF2zfqhqr/ahw9blyV4jT7WHl5bpwdCVkAUxYMTBhDkCMo7srlXbMu+gRJ01+fKhcJC8zCE+n/4nc4fXWGZ9JJUhN4sRNEoc8MQTfwD8Jj//M4xHgY6qq6NkmGfA0Bqd401yzkb8GMnQcEQXTb8Y81cxmxYwpbyLkZKmGEzLPM8uTI8E5HxOZv6gjncazD0ieUYCAR1aXe9sBgb3oQQcSc/0MsalBEqQVliGeyR1oVePJ6SQo1AemDHzSS//M4xHMX4gLG9kmGHGRxyytIByYURRCNOaSwm06BFBVaGEJc7pyORiCm4ljvTQzpdiWRTvJ3st3ahRpMdTVZwxSpIwRCU0NwK1bovu/ljTh61t87klQBXiEgM94iSA/wRVZ6flZiCJlQ6m7Y//M4xHIVqebS9jBEzGZ3JysMUh7pfrIR1xjCCgqqcQNVndn5xzIvYkeG2CJ6A7EhoWCwhJLaSt+VjobUsY71rHzzL/8jFLDI9ECgCcdxiZmMbWPDKFmVruiiChpQooYhi7y7rxietzT003tM//M4xHoVAU7KVjDE1JuRfRC5ok0fhST++5n5Hvlzka586sizN/MMuLHWDgnULGoqHx5tTBO4Hz+Ud82KFVyh97tSMmIAGH6tFRRIGgVIOvEqIQ9tEGMwIYACZOpqLJ7T7GeYqc9d8maSqiV0//M4xIUX8dLCTkmGzHYz535IgPg5jcUaK79BRjceWhgGQ6yfsxUFhFIZBUq2shGvpeMbSbQTWml+rKD71l+o+cTVeSb6hA2ek4uagIMTRlep55InyxxDmJYnUWCMKBUCzgIdFWtmeAIQLKI4//M4xIQWScrKVjBFhFIjcjWTLDnr7anN+LAjspzx5k6ib3m0p9vf8dnxm837S7iVriHCijcKAI2kJsilqQcD86d98cisu6xnV365/FoRIBfUIikE5xmHDOaI+S1KM7j0zAeKxLs5YQV06HAt//M4xIkbQXLKLHsM7FLs11bxQEm6u7ECc2ElSZllAKXSWfTSsZxhamui0c6ofkqiggupaCpX2ZX1Y2tnlaefqafKXY9nM8nL2sbZLNO+v//7P5nc2palKUNxRLp2kAAkFLDaaKbjm2alm7xk//M4xHsaqyLKJnpE7fbSCEmnzK09OlWWspHoKiiTAVNwoCjEJB3D2x9Yd94SMDXZpM7mUOY7wyQrYvXkcqTu8p1YWFAQp5xdxF5TOLOVkViLO3ImQTVlTVXbV2kz/s3s516n21PiA0SDgiPU//M4xG8cIzq6LmDK+V26VaAQqEjJYTz0B7Rkdg4aLyuB9Xhm1eEtn1f5CVhAtwMxBEtdzHzS1x4Er10R3TJZ/M5+xWZ7hpATZzDsILvzKl1lpsMqBgygoPPH1xXKqCFBlbBVbmm0C7v/8Fct//M4xF0XOeK+AgsGGFAABsBdHxUxGU6D6tVtgwS7vIZKR1KwbJLsCJ1mzOsTuKp0KKDos9AOipAcU82U0JZPIviuGEEZF0zNpYsMy+mf/W4MUEENvrh9tWd2DsJNChGiZQj8//Of5Tt56EhO//M4xF8acoLCDnpGtFCHK8eNeGHib/5ZrAGFhNoqojExxcdlubJ7HSOPBICqrV6aqCMurzoA1AEN6WvwpKosyEcUT6GXr3lNZ+eXN9tadVfm8/PqLdYqaeY5IXFhqKTdRpAwyKC/kqnnGpd2//M4xFQWKgLFjgsGEDa1CjiC4AMDKNyFSShPwRKw+Q8rbBAggCBCBkZGQhkZmiEgfwPqb13dgMEAcnXUv3PO553inm/u8F1xb5vVMv+56UGWCQftWP+qysHtSG/SkuI+JoLJGmyix54Z4AJS//M4xFoVucbFZlmGjB9Qr6b27Xa8YbnlpH+5FTGXQi6iYle8lFkazjZOpa6L5uGpOPKEsgA/GQG3lpHUTQ2qQ/u+K75/qpp8fMckVIpZNl2s/f8ebiua0up7bWMOU6RUbFrsLJeqi/vQyAKC//M4xGIXuhbJiHrM9HWVBUAizB94BC8Ym5ZCiS/SVUSvLncKUYCoVGh3LiG30l44nciVSs2uU5mOfWb/m1I2gaicWAfHMQE1i4SCgdFjw5bl8J1UsFGqfvhgb+uEBxVNECgGg4a8AqANTqYS//M4xGIUgYLSVhGGGCwsMWbtmPYwtOuTaMrubUjGlzP/WoXEjLTI/LRGsvfs7y6/nCSFKKbueZ1EysS1ypab3tb0+XI07GRxR5pl5rJD9uuKR6xpMgVMPZhkwLdiM5RJImY1cI6YEFgKinV4//M4xG8UwoLGTkDE/JTRjSxmZs5ba3mffDOatCsrzGzn/TeR0MlUESwuaC4lBkCC444OBBBMW01irRdSwu/7UUa2qv+6mpZJrpZ/ylw4ECgbyqCKSA2kJDbMdQMBismKlwSkKpQxUEHYnl5g//M4xHsVIZrGVjDEtKmbEhIpuxl3F8kL9NfL7XfO/tBNjRqVhEVc1CDZW3wKNIMcTFgrTcTOxYvFv/0V221/1t8t7iooJ74RcH7vv91iRgyyyRL0guFvUjkiwrQx2y3yMkhn3inrtDOwzzky//M4xIUVGZruVhpGivt9USNGP4E4fz1VCVx6DcEIdTG2E0n1D7n6z7G3YRI//62K7dtxjIDaLikDvGKwYgswXSRrLG1RINyBks7Md8koYwxUp/wlKGYVgdOCSg0cWRkRy7sUk5U8MxTbFo97//M4xI8UcerWVhGGHLnScrQVLjFico4522OuS1kGn71GitH/1mtNrnXuhV41KnAIILHjI4gIJu5Bx6CcnDuadQ0QcUwxAXiZJfYTmyrxplc5WusM9mDFXo0FM5TQBDrpIRKQaCZha1u1pDpK//M4xJwVOdbGVhJGEPGLg+QIJ+AYyn/SVmy5GqigFwdqaUZT2KwhTkOXMm0zU9PQLQbGNK7JdTNMPbsgTWzNLREBhR3UW37n9OF8vnT2tpJQbwG0KfnAgWPA4MEs4BHZ+KvUhZAOtKBn9LtX//M4xKYUsZbKVhhFoP0Kcllkkf9JwCimZnqtDGzZZotuTUh4OhmWNI7pWpRHA+TMkVcQxKQd3rDh1m8UFrfZlhnHHW/7HdodLsgZPL7wl2QIXDT2DjTU+ZEINnaYHA93wBdVcbdkbUIBUCxp//M4xLIVQb6+TkBHQJM2dN2fcFDEHejTSotWNBUEzHgT8zZsSN4YGiZGOUO3umzovfevv0JFIjhfqOZQWBYhAp3v+P/9f9v9XpRtzZnvxchure3NKn7XfZGsBV4+HOSWmhmQvvqG0KJspUSt//M4xLwUydbmVgpGHsXuJyUnMPI6SwrSPyMmJDU40Rj7U5ElL8+ft3KzwSGBkVoFQ6VKEgTJAUCOFiRBL5oL3Umnr6TZ+vT/Yqq5//+x9oVePNkEkJbMosRzyKzMuudQrcRUjypATh3MnS4U//M4xMcUCH7GVgmGFbnUL/N0ZrC6In6GXuGUzc86TfO1ctaX9E1fYmxynZw5l6Q4qL5JDYgJn1+UG43/9j7aOSORQADOOLwqdK6t9r+MmC0l3ZtFFvBpqVlLRPRugd6L135bFmGnseUW8IZR//M4xNUU0ZLKVjDE9PuTO/XZz9cjWkall8vKRqoals6nvfpAhKNaENTdYkNbFks8p/kWsFZ5ln4YIMfeHoQQ1rRQOD7uE3gobrbLrXH/KUwgJBKKAXQaYZZVim2RoF0I0JX0gtV5vojuzNC///M4xOAUsfbOVkjE9JKRySnXYPqzBkIS7kaTOmQh5gKH9SVhRo/z7kGZcMDzJIy3cl9+HLCxJ/QTZYOv/idkBj5GhWWaa7on+PSZUSH0LBd+Jdyu4llv3T4Hpoce2WHM6SQWUkq1J1c3HSxR//M4xOwacnq+VmGGfNlBWZHSqWqQRDVFLsDL5H5mxnd6blzJzt90yCAzjhml6JLRS3tajf7bjprO1b/tSsIV76HX22xRl7Hb+a2WfQVeGlBI8QATlYQtuu0784Xkbrts/jGs4xXsCYEVLY7S//M4xOEVsTLmVhJGGjQDh820JENPZmTvdBMaGCcUAoxCWqsiBIAIqIhdoRKvP+QQpjoAMv8P//8juVLI4y0QK8RwMEJESQOhJRkctGN0kicJ6CjJbhwSBCzEKzKRsSvFzU+sMZCsJGI2oNYc//M4xOkZOdrK9kmG6Wj4d6RfCaGJNRRSQi3QkweD6IkCsxha9CwvjyNqRee4UsjulsZQqIbKWpQwxqkBD8ey6n97nfh9VYtkeEaVtTAVuHCROKTQOIFHBu1qSyaQuFIQns5IAJk0Tb5p0/4E//M4xOMUcUrOVjDLJCFERvXcoRwYiXtuU1nf4WefO3L6e05ERgUNCUgD4mWZYwb0OQugUAJkaL5YOkStW3j9VarPWStNUBuHIyAe6EnOh+RKMnFJO6gWyu5+uYzbRZ30Y2bEaCIw/lWgwPKs//M4xPAbAqq6ThGGFUfFE8lUq0po6sSr6G/CVcUEHFAmFe84pkwao4zmoYUQgcROPH200oMGwB8ubUEDBcOps79rHLpJuyJtyAFAMgy45IaGOQIQLBOMKcChqnRDgyBQAdXpsoMSEEqxFNQb//M4xOMWCc7K9jGGGFEW2E6A00LwElqYDrDYCYmaJ1khmmqMISDiSlhg1UOjAvB3smAyjZf5w0oB6n/ubf1Q1Uo3PvqQp0e3KUQCAGUgVaWigsJyoIfcPIVtzfVKdscZlsELPCzo2FOxsV/J//M4xOkYYea+TkmGWAtPOfTpfN7qh7Qtof7kdm+XCK0Db1p9JQGyC47kUYPf9bfL+f0Kl2iy6hhVBooF0LKE9yt/6xzxwGqTPWpM8BmDlcoHcgOzTqUcCFe3AwZV1/rz/9PN35EZm/cz4+7e//M4xOYWwfq+VhhHhekLnZaS1NSONhW45cPntpJEAeKtYlGV3mlM7oUcqAQ6aIrV00s/1c9nZg5w+MGw9Hxfp/ZCrkZ3h9ZoniW4+JBWyYnCiEcAgkYYIIjDTXUr6WecfJeageyJkFFnmzWX//M4xOoZYnbONjBHwm2XvemydJONyeLJkbubmv+dOoLgLWo/lvlnTMy2hwgYhAoMVyLnCzgNgJE8Jtc6lnOmmzhSqzuii4rV8ttlbTSBTiyYC2HHrctZViGY7Zwk1azQlYlKyoqgfqmg73bi//M4xOMWwnrCTjBFhPZKbvkCqLQpemuumdKLMF3/rGOUlNZ/973VSVS2xIQL9forE4BWeQP8cpLmt/9KB6qQWy62yWx1utMppNuMAjIlkmZH5gGmJOeWcl8QwQ0DLqxHDTENQ1BDM9YS0XRo//M4xOcYyirK9kmGXDiQA9zgaoU9emEYaAt54uaNdMzNVTN070dcdiZFtJvtvGVDm/OmKFerOxG86YkLNuz3VYTCw///HhUxBale4yN7FBgtTuWIkVN8f5z/1GqGx9DvaLLHzdRR21lQ2k7d//M4xOIVCerGV0kYAJibYf86tj//9Gs2GSPqsDe/rX0f12GSNlamSrPxWgBGwJBFFQjtzZHdy6CohgUd//iC5DAo6Kb0V+XhXivx+F5sX//+grAzI74uIr//8nxfBQXI4KOBRUUVCO4/EVC///M4xOwqosbKX494Avzbf///iuHQjvFdFUxBTUUzLjEwMFVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV//M4xKATOAHh58EYAVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV\" type=\"audio/mpeg\"/>\n",
       "                        Your browser does not support the audio element.\n",
       "                    </audio>\n",
       "                  "
      ],
      "text/plain": [
       "<pydub.audio_segment.AudioSegment at 0x1fda7f9dbe0>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pydub import AudioSegment\n",
    "\n",
    "from data_processing import gen_clip, overlay_noise\n",
    "\n",
    "clip = gen_clip(AudioSegment.from_file('parsed_files/raw_clips/government_9302.wav'))\n",
    "noise = AudioSegment.from_file('audio/outside_rural/snipped119_start_240_end_250.wav')\n",
    "overlay_noise(2.0, clip, noise)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Count word frequencies to consider which data points we will keep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "metadata": {}
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'whiskers': [<matplotlib.lines.Line2D at 0x17d81714260>,\n",
       "  <matplotlib.lines.Line2D at 0x17d81717ec0>],\n",
       " 'caps': [<matplotlib.lines.Line2D at 0x17d81717c80>,\n",
       "  <matplotlib.lines.Line2D at 0x17d81717980>],\n",
       " 'boxes': [<matplotlib.lines.Line2D at 0x17d81714560>],\n",
       " 'medians': [<matplotlib.lines.Line2D at 0x17d817176b0>],\n",
       " 'fliers': [<matplotlib.lines.Line2D at 0x17d81717440>],\n",
       " 'means': []}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAGdCAYAAAAMm0nCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAs2UlEQVR4nO3df3RU9Z3/8ddMfgnJBOIPJgEFg0GpBukmYBLrGCFSZI2VWV3FetTualsspx7PguxGd/eI31OTLWdxdyt6TherrSK7blpC1x8gUBNSmqWCRcOqrJMA9YTJRBJCQhLJj/l8/7AZM5LSIJPMvTfPxznvc2bu553MJzmH3Bf33s+9LklGAAAANuKO9wQAAADOFgEGAADYDgEGAADYDgEGAADYDgEGAADYDgEGAADYDgEGAADYDgEGAADYTmK8JzCapk6dqs7OznhPAwAAnAWPx6OjR4+escexAWbq1KlqamqK9zQAAMCXMG3atDOGGMcGmMEjL9OmTeMoDAAANuHxeNTU1PQn992ODTCDOjs7CTAAADjMWV/E6/P59Mtf/lJNTU0yxujWW289rWfNmjU6evSouru7tX37duXk5ESNZ2Rk6KWXXtKJEyd0/PhxbdiwQampqVE9c+bM0a5du9TT06Pf//73euSRR852qgAAwKHOOsCkpqbq3Xff1YoVK4YdX716tR566CEtX75cBQUF6urq0rZt25SSkhLp2bhxo6666iotWrRIpaWluv766/XjH/84Mu7xePTmm2/qyJEjys/P1yOPPKLHH39c3/72t7/EjwgAAJzIfNkyxphbb701atvRo0fNypUrI+/T09NNT0+PufPOO40kM3v2bGOMMfn5+ZGexYsXm4GBAZOVlWUkmeXLl5vW1laTlJQU6SkvLzcffPDBiOfm8XiMMcZ4PJ4v/fNRFEVRFDW2NdL9d0zvA5Odna2srCzt2LEjsq2jo0N79uxRUVGRJKmoqEjHjx/Xvn37Ij07duxQOBxWQUFBpGfXrl3q6+uL9Gzbtk2zZ8/W5MmTh/3s5ORkeTyeqAIAAM4U0wCTmZkpSQqFQlHbQ6FQZCwzM1MtLS1R4wMDA2pra4vqGe57DP2MLyorK1NHR0ekWEINAIBzOeZOvOXl5UpPT4/UtGnT4j0lAAAwSmIaYJqbmyVJXq83arvX642MNTc3a8qUKVHjCQkJOv/886N6hvseQz/ji3p7eyNLplk6DQCAs8U0wBw6dEjBYFAlJSWRbR6PRwUFBaqrq5Mk1dXVKSMjQ3l5eZGehQsXyu12a8+ePZGe66+/XomJn9+mZtGiRfrwww/V3t4eyykDAACbOqurg1NTU83cuXPN3LlzjTHGPPzww2bu3LnmkksuMZLM6tWrTVtbm7nllltMbm6u2bx5s2loaDApKSmR7/H666+bffv2mfnz55trr73WHDx40GzcuDEynp6eboLBoPnpT39qrrzySnPHHXeYkydPmm9/+9sxv4qZoih7ldvtNsXFxWbZsmWmuLjYuN3uuM+JoqjY1Vnsv8/uGxcXF5vhPP/885GeNWvWmGAwaHp6esz27dvNrFmzor5HRkaG2bhxo+no6DDt7e3mueeeM6mpqVE9c+bMMbt27TI9PT3m448/NqtXrx6tXwBFUTYpv99vGhsbo/72NDY2Gr/fH/e5URQVmxq1AGOXIsBQlLPK7/ebgYEBs2XLFlNQUGBSU1NNQUGB2bJlixkYGCDEUJRDaqT7b9cfXjiOx+NRR0eH0tPTuaAXsDm3261AIKD6+notXbpUxnz+Z8vlcqmqqkq5ubmaNWuWwuFwHGcK4FyNdP/tmGXUAJzL5/MpOztbTz75ZFR4kSRjjMrLyzVz5kz5fL44zRDAWCPAALC8rKwsSdKBAweGHR/cPtgHwPkIMAAsLxgMSpJyc3OHHR/cPtgHwPkIMAAsr7a2VocOHdKjjz4ql8sVNeZyuVRWVqbGxkbV1tbGaYYAxhoBBoDlhcNhrVy5UqWlpaqqqlJhYaHS0tJUWFioqqoqlZaWatWqVVzAC4wzcV8yNRrFMmqKcl4Ndx+YhoYGllBTlIOKZdQsowYcye12y+fzKSsrS8FgULW1tRx5ARxkpPvvxD86AgAWFA6HVVNTE+9pAIgzroEBAAC2Q4ABAAC2Q4ABAAC2Q4ABAAC2Q4ABAAC2Q4ABAAC2Q4ABAAC2Q4ABAAC2Q4ABAAC2Q4ABAAC2Q4ABAAC2Q4ABAAC2Q4ABAAC2Q4ABAAC2Q4ABAAC2Q4ABAAC2Q4ABAAC2Q4ABAAC2Q4ABAAC2Q4ABAAC2Q4ABAAC2Q4ABAAC2Q4ABAAC2Q4ABAAC2Q4ABAAC2Q4ABAAC2Q4ABAAC2Q4ABAAC2Q4ABAAC2Q4ABAAC2Q4ABAAC2Q4ABAAC2kxjvCQDA2XC73fL5fMrKylIwGFRtba3C4XC8pwVgjHEEBoBt+P1+BQIBVVdXa9OmTaqurlYgEJDf74/31ACMMQIMAFvw+/2qrKxUfX29CgsLlZaWpsLCQtXX16uyspIQA4wzLkkm3pMYDR6PRx0dHUpPT1dnZ2e8pwPgHLjdbgUCAdXX12vp0qUy5vM/Wy6XS1VVVcrNzdWsWbM4nQTY3Ej33xyBAWB5Pp9P2dnZevLJJ6PCiyQZY1ReXq6ZM2fK5/PFaYYAxhoBBoDlZWVlSZIOHDgw7Pjg9sE+AM5HgAFgecFgUJKUm5s77Pjg9sE+AM5HgAFgebW1tTp06JAeffRRuVyuqDGXy6WysjI1NjaqtrY2TjMEMNYIMAAsLxwOa+XKlSotLVVVVVXUKqSqqiqVlpZq1apVXMALjDPGieXxeIwxxng8nrjPhaKo2JTf7zeNjY1mqIaGBuP3++M+N4qiYlMj3X+zjBqArXAnXsDZRrr/5lECAGwlHA6rpqYm3tMAEGdcAwMAAGyHAAMAAGyHAAMAAGyHAAMAAGyHAAMAAGwn5gHG7XbriSeeUGNjo7q7uxUIBPT3f//3p/WtWbNGR48eVXd3t7Zv366cnJyo8YyMDL300ks6ceKEjh8/rg0bNig1NTXW0wUAADYV0xvQlJWVmU8++cT8+Z//uZkxY4a57bbbTEdHh/n+978f6Vm9erU5fvy4+cY3vmHmzJljqqqqTENDg0lJSYn0vP766+Z3v/udueaaa8zXvvY183//939m48aNMb8RDkVRFEVR1qmz2H/H9oP/+7//22zYsCFqW2VlpXnxxRcj748ePWpWrlwZeZ+enm56enrMnXfeaSSZ2bNnG2OMyc/Pj/QsXrzYDAwMmKysrFj/AiiKoiiKskiNdP8d81NIv/nNb1RSUqJZs2ZJkq6++mpdd911euONNyRJ2dnZysrK0o4dOyJf09HRoT179qioqEiSVFRUpOPHj2vfvn2Rnh07digcDqugoGDYz01OTpbH44kqAADgTDG/E29FRYXS09P14YcfamBgQAkJCXrsscf08ssvS5IyMzMlSaFQKOrrQqFQZCwzM1MtLS1R4wMDA2pra4v0fFFZWZkef/zxGP80AADAimJ+BOaOO+7Q3XffrW9+85vKy8vTfffdp1WrVunee++N9UdFKS8vV3p6eqSmTZs2qp8HAADiJ+ZHYNauXauKigr953/+pyTpwIEDmjFjhsrKyvSzn/1Mzc3NkiSv1xt5Pfh+//79kqTm5mZNmTIl6vsmJCTo/PPPj/qaoXp7e9Xb2xvrHwcAAFhQzI/ATJw48bQnww4MDMjt/uyjDh06pGAwqJKSksi4x+NRQUGB6urqJEl1dXXKyMhQXl5epGfhwoVyu93as2dPrKcMAABsKKZXDz///PPm448/jiyjXrp0qWlpaTEVFRWRntWrV5u2tjZzyy23mNzcXLN58+Zhl1Hv27fPzJ8/31x77bXm4MGDLKOmKIqiKIdX3JZRp6WlmaeeesocPnzYdHd3m0AgYP7f//t/JikpKapvzZo1JhgMmp6eHrN9+3Yza9asqPGMjAyzceNG09HRYdrb281zzz1nUlNTR+MXQFEURVGURWqk+2/XH144jsfjUUdHh9LT09XZ2Rnv6QCIEbfbLZ/Pp6ysLAWDQdXW1p522hqAfY10/82zkADYht/vVyAQUHV1tTZt2qTq6moFAgH5/f54Tw3AGCPAALAFv9+vyspK1dfXq7CwUGlpaSosLFR9fb0qKysJMcA4wykkAJbndrsVCARUX1+vpUuXypjP/2y5XC5VVVUpNzdXs2bN4nQSYHOcQgLgGD6fT9nZ2XryySejwoskGWNUXl6umTNnyufzxWmGAMYaAQaA5WVlZUn67MaYwxncPtgHwPkIMAAsLxgMSpJyc3OHHR/cPtgHwPkIMAAsr7a2VocOHdKjjz4ql8sVNeZyuVRWVqbGxkbV1tbGaYYAxhoBBoDlhcNhrVy5UqWlpaqqqopahVRVVaXS0lKtWrWKC3iBcSbud90bjeJOvBTlvPL7/aaxsdEM1dDQYPx+f9znRlFUbIo78bKMGnAk7sQLONtI99+JYzgnADhn4XBYNTU18Z4GgDjjGhgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7BBgAAGA7oxJgpk6dqhdffFHHjh1Td3e33nvvPeXn50f1rFmzRkePHlV3d7e2b9+unJycqPGMjAy99NJLOnHihI4fP64NGzYoNTV1NKYLAABsJuYBZvLkydq9e7f6+vq0ZMkSXXnllVq5cqWOHz8e6Vm9erUeeughLV++XAUFBerq6tK2bduUkpIS6dm4caOuuuoqLVq0SKWlpbr++uv14x//ONbTBQAANmViWeXl5WbXrl1n7Dl69KhZuXJl5H16errp6ekxd955p5FkZs+ebYwxJj8/P9KzePFiMzAwYLKyskY0D4/HY4wxxuPxxPTnoyiKoihq9Gqk+++YH4H5xje+ob179+qVV15RKBTSO++8owceeCAynp2draysLO3YsSOyraOjQ3v27FFRUZEkqaioSMePH9e+ffsiPTt27FA4HFZBQcGwn5ucnCyPxxNVAADAmWIeYGbOnKkHH3xQH330kRYvXqxnn31W//Zv/6Z7771XkpSZmSlJCoVCUV8XCoUiY5mZmWppaYkaHxgYUFtbW6Tni8rKytTR0RGppqamWP9oAADAIhJj/Q3dbrf27t2rxx57TJK0f/9+5ebmavny5frZz34W64+LKC8v17p16yLvPR4PIQZwILfbLZ/Pp6ysLAWDQdXW1iocDsd7WgDGWMyPwASDQb3//vtR2z744ANNnz5dktTc3CxJ8nq9UT1erzcy1tzcrClTpkSNJyQk6Pzzz4/0fFFvb686OzujCoCz+P1+BQIBVVdXa9OmTaqurlYgEJDf74/31ACMsZgHmN27d+uKK66I2nb55ZfryJEjkqRDhw4pGAyqpKQkMu7xeFRQUKC6ujpJUl1dnTIyMpSXlxfpWbhwodxut/bs2RPrKQOwAb/fr8rKStXX16uwsFBpaWkqLCxUfX29KisrCTHAOBTTq4fnzZtnent7TVlZmbnsssvMXXfdZU6ePGm++c1vRnpWr15t2trazC233GJyc3PN5s2bTUNDg0lJSYn0vP7662bfvn1m/vz55tprrzUHDx40GzdujPlVzBRFWb/cbrdpbGw0W7ZsMS6XK2rM5XKZLVu2mIaGBuN2u+M+V4qizq3OYv8d+w+/+eabzXvvvWd6enrM+++/bx544IHTetasWWOCwaDp6ekx27dvN7NmzYoaz8jIMBs3bjQdHR2mvb3dPPfccyY1NXU0fgEURVm8iouLjTHGFBQUDDteWFhojDGmuLg47nOlKOrcKq4BxgpFgKEo59SyZcuMMeaP/icmLS3NGGPMsmXL4j5XiqLOreJ2HxgAiLVgMChJys3NHXZ8cPtgHwDnI8AAsLza2lodOnRIjz76qFwuV9SYy+VSWVmZGhsbVVtbG6cZAhhrBBgAlhcOh7Vy5UqVlpaqqqoqahVSVVWVSktLtWrVKu4HA4wzcT/fNRrFNTAU5bzy+/2msbHRDNXQ0GD8fn/c50ZRVGxqpPtv1x9eOI7H41FHR4fS09O5qR3gIMnJyfrnf/5n5eTkKBAIaOXKlert7Y33tADEyNnsv+OetkajOAJDUc6riooK09vbG3UEpre311RUVMR9bhRFxaZYhQTAUSoqKrR69Wq1trbq/vvvl9fr1f3336/W1latXr1aFRUV8Z4igDHEKSQAlpeYmKju7m61trZq2rRpURfrut1uNTU16YILLtDEiRPV398fx5kCOFcj3X9zBAaA5a1YsUJJSUl67LHHTltpFA6H9Q//8A9KSkrSihUr4jRDAGONAAPA8i677DJJ0quvvjrs+OD2wT4AzkeAAWB5DQ0NkqTS0tJhxwe3D/YBcD6ugQFgeVwDA4wfXAMDwDH6+/u1bt06eb1eNTU16YEHHlBmZqYeeOABNTU1yev1at26dYQXYJyJ+5rv0SjuA0NRzivuA0NRzi/uxMspJMCREhMTtWLFCl122WVqaGjQ+vXrOfICOMhI998EGAAAYBlcAwMAAByLAAMAAGyHAAMAAGyHAAMAAGyHAAMAAGyHAAMAAGyHAAMAAGyHAAMAAGyHAAMAAGyHAAMAAGyHAAMAAGyHAAMAAGyHAAMAAGyHAAMAAGyHAAMAAGyHAAMAAGyHAAMAAGyHAAMAAGyHAAMAAGyHAAMAAGwnMd4TAICz4Xa75fP5lJWVpWAwqNraWoXD4XhPC8AY4wgMANvw+/0KBAKqrq7Wpk2bVF1drUAgIL/fH++pARhjHIEBYAt+v1+VlZV67bXXVFVVpQkTJqinp0c5OTmqrKzU7bffrs2bN8d7mgDGiEuSifckRoPH41FHR4fS09PV2dkZ7+kAOAdut1uBQED9/f269NJLlZSUFBnr6+vT4cOHlZCQoFmzZnE6CbC5ke6/OYUEwPJ8Pp+ys7OVk5Oj1tZW3X///fJ6vbr//vvV2tqqnJwczZw5Uz6fL95TBTBGOIUEwPIuvvhiSVJLS4umTZsWOcryk5/8RC+88IKOHj0qr9cb6QPgfByBAWB5hYWFkqQNGzacdoooHA7rJz/5SVQfAOcjwACwjXnz5snlckVtc7lcysvLi9OMAMQLAQaA5QUCAUnSokWLVFVVpcLCQqWlpamwsFBVVVVatGhRVB8A52MVEgDLS0xMVHd3t06ePKn29nZlZ2dHxhobG5WRkaG0tDRNnDhR/f39cZwpgHPFKiQAjtHf369169Zp8uTJmjBhgtauXasHH3xQa9eu1cSJEzV58mStW7eO8AKMM8aJ5fF4jDHGeDyeuM+FoqjYVEVFhent7TVD9fb2moqKirjPjaKo2NRI99+cQgJgK4mJiVqxYoUuu+wyNTQ0aP369Rx5ARxkpPtv7gMDwFbC4bD279+vUCikYDDInXeBcYprYADYBg9zBDCIAAPAFgYf5jhlypSo7VOmTFFlZSUhBhhnCDAALM/tduvZZ5+Vy+XSzp07o+4Ds3PnTrlcLj3zzDNyu/mTBowncb/ieDSKVUgU5ZxasGCBMcaYXbt2GZfLFTXmcrnMrl27jDHGLFiwIO5zpSjq3Gqk+2/+uwLA8m644QZJ0j/+4z/KGBM1ZozR448/HtUHwPkIMAAAwHYIMAAsr7q6WpL0xBNPDPswx8EjMIN9AJyPAAPA8mpqahQKhXTdddcN+zDH6667Ts3NzaqpqYn3VAGMoVG9GOdv//ZvjTHGPPXUU5FtKSkp5umnnzbHjh0znZ2dprKy0kyZMiXq6y655BLz6quvmq6uLhMKhcwPf/hDk5CQEPOLgCiKskf5/X4zMDBgTp48GfUogZMnT5qBgQHj9/vjPkeKos69zmL/PXqTmDdvnmlsbDT79++PCjDPPPOMOXLkiFmwYIHJy8szv/nNb8yvf/3ryLjb7TbvvfeeefPNN83cuXPNTTfdZFpaWswPfvCD0fgFUBRlk/L7/aaxsTEqwDQ0NBBeKMpBFfcAk5qaag4ePGhKSkrMW2+9FQkw6enp5tSpU+a2226L9F5xxRXGGGMKCgqMJHPTTTeZ/v7+qKMy3/3ud017e7tJSkqK9S+AoigbldvtNsXFxWbZsmWmuLjYuN3uuM+JoqjYVdyXUa9fv16vvfaadu7cGbU9Pz9fycnJ2rFjR2TbwYMHdeTIERUVFUmSioqKVF9fr5aWlkjPtm3bNGnSJF111VXDfl5ycrI8Hk9UAXCecDismpoa/cd//Idqamp4FhIwTo3KwxzvvPNO5eXlaf78+aeNZWZm6tSpUzpx4kTU9lAopMzMzEhPKBQ6bXxwbDhlZWWRlQgAAMDZYn4E5uKLL9a//uu/6u6779apU6di/e3/qPLycqWnp0dq2rRpY/bZAABgbMU8wOTn58vr9eqdd95RX1+f+vr6dMMNN+ihhx5SX1+fQqGQUlJSNGnSpKiv83q9am5uliQ1NzfL6/WeNj44Npze3l51dnZGFQAAcKaYB5idO3cqNzdXX/3qVyP19ttva+PGjfrqV7+qvXv3qre3VyUlJZGvufzyyzVjxgzV1dVJkurq6jRnzhxddNFFkZ5FixbpxIkTev/992M9ZQA24na7VVxcrGXLlqm4uJgHOALj2KhfUTx0FZL02TLqw4cPmxtuuMHk5eWZ3bt3m927d39+ZfEfllFv3brVXH311ebrX/+6CYVCLKOmqHFewy2jbmxsZBk1RTmo4r6Memh9McAM3siutbXVnDx50vz85z83Xq836mumT59uXnvtNdPV1WVaWlrM2rVruZEdRY3jGryR3ZYtW0xBQYFJTU01BQUFZsuWLdzIjqIcVCPdf7v+8MJxPB6POjo6lJ6ezvUwgM253W4FAgHV19dr6dKlUU+kdrlcqqqqUm5urmbNmsWyasDmRrr/5uQxAMvz+XzKzs7Wk08+GRVeJMkYo/Lycs2cOVM+ny9OMwQw1ggwACwvKytLknTgwIFhxwe3D/YBcD4CDADLCwaDkqTc3Nxhxwe3D/YBcD4CDADLq62t1aFDh/Too4/K5XJFjblcLpWVlamxsVG1tbVxmiGAsUaAAWB54XBYK1euVGlpqaqqqlRYWKi0tDQVFhaqqqpKpaWlWrVqFRfwAuNM3JdMjUaxjJqinFfD3QemoaGBJdQU5aBiGTXLqAFHcrvd8vl8ysrKUjAYVG1tLUdeAAcZ6f57VJ5GDQCjJRwOq6amJt7TABBnXAMDAABshwADAABsh1NIAGyFa2AASByBAWAjfr9fgUBA1dXV2rRpk6qrqxUIBOT3++M9NQBjjCMwAGzB7/ersrJSr776qtauXauenh5NmDBBN910kyorK3X77bdr8+bN8Z4mgDHCMmoAljf4NOpjx47pwgsvVHZ2dmTs0KFDOnbsmC644AKeRg04AE+jBuAYg0+jzs/PV319fdSdeOvr65Wfn8/TqIFxhgADwPKmTp0qSXrjjTe0dOlS7dmzR11dXdqzZ4+WLl2qrVu3RvUBcD4CDADLmzJliiTpF7/4hYyJPuttjIlc+zLYB8D5CDAALO+TTz6RJP3FX/zFsE+jXrp0aVQfAOdjFRIAy2tqapIkLVmyRFu2bNEbb7wRWYW0ZMkSLVmyJKoPgPOxCgmA5Q2uQurv79ell16qpKSkyFhfX58OHz6shIQEViEBDsAqJACOEQ6H9corrygnJ0etra1au3atHnzwQa1du1atra3KycnRf/3XfxFegHGEIzAALO9M94FpbGxUa2sr94EBHGKk+2+ugQFgeYP3gbnrrrv09ttvn/YspGuuuUZ1dXXy+XyqqamJ93QBjAECDADLy8rKkiQdOHBA4XD4tJBy4MCBqD4Azsc1MAAsLxgMSpJyc3PldrtVXFysZcuWqbi4WG63W7m5uVF9AJyPa2AAWB7PQgLGD1YhAXCMwVVI8+bN0yWXXBI1dskll2jevHmsQgLGGQIMAMtzu9361re+FXn9xTFJuu+++04bA+Bc/GsHYHnFxcXyer2SpJ6enqixwfeZmZkqLi4e87kBiA8CDADLW7BgQeT1zp07VVhYqLS0NBUWFmrnzp3D9gFwNpZRA7C8GTNmSJLq6+u1dOnSyBOp9+zZo6VLl2r//v26+uqrI30AnI8jMAAsb/AJ1IPB5U/1AXA+AgwAyzt8+LAk6eqrr1ZVVVXUKaSqqirNmTMnqg+A8xFgAFjeW2+9FXldUlKiuro6dXZ2qq6uTiUlJcP2AXA2AgwAy6upqVEoFJL0x08TNTc38xwkYBwhwACwvHA4rAcffFDGGE2YMCFqbOLEiTLG6Hvf+x43sgPGEQIMAFsoKCiQy+XSwMBA1PaBgQG5XC4VFBTEaWYA4oFnIQGwvMTERHV3d+vkyZNqb28/7VlIkydPVlpamiZOnKj+/v44zhTAueJZSAAcY8WKFUpKStKkSZM0ZcqUqLEpU6Zo0qRJSkpK0ooVK+I0QwBjjQADwPJycnIkfXYB73B34h28sHewD4DzEWAAWN5gQPnoo490++23q7CwUOXl5SosLNTtt9+uQCAQ1QfA+XiUAADLa29vlyRdfPHF6urqUlJSUmRs7dq16uvri+oD4HwEGACWN7jy6ItLqKXPLvBNTEyM6gPgfJxCAmB5tbW1kddffB7S0PdD+wA4GwEGgOUNPutIkrZu3apAIKBjx44pEAho69atw/YBcDYCDADLu+666yKvlyxZopycHF144YXKycnRkiVLhu0D4GwEGACWd/LkyZj2AbA/LuIFYHmbNm3SvffeK0l69dVX9emnn2ry5Mlqb2/Xeeedp9LS0kgfgPGBAAPA8r7zne9EXpeWlkbd72XoRbzf+c53oq6JAeBcnEICYHkjvTiXi3iB8YMAA8Dyjh8/Hnnd3d0dNTb0/dA+AM7GKSQAlrdv3z7Nnz9fknTBBReosLBQWVlZCgaD+p//+R/19PRE+gCMDxyBAWB5Q59A3dXVpbKyMk2fPl1lZWXq6uoatg+As3EEBoDldXZ2Svrsgl23263Fixdr8eLFkW3GGLlcrkgfAOfjCAwAy3vxxRclffa06eEeJTC4KmmwD4DzEWAAWF5NTc2ffFDjwMCAampqxmhGAOKNAAPA8nw+nxISEiQp6h4wQ98nJCTI5/ON+dwAxAcBBoDlLViwQJLU1tY27Pjg9sE+AM5HgAFgedOnT5cknX/++cOOD24f7APgfDEPMH/3d3+n3/72t+ro6FAoFNLmzZt1+eWXR/WkpKTo6aef1rFjx9TZ2anKysrTlj9ecsklevXVV9XV1aVQKKQf/vCHkUPIAMaXpqamyOsvXgsz9P3QPgDOFvMAU1xcrPXr16uwsFCLFi1SUlKS3nzzTU2cODHS89RTT+mWW27RX/7lX6q4uFhTp07VL37xi88n5XbrtddeU3Jysq699lrdd999+ta3vqUnnngi1tMFYAMZGRmR18eOHdOmTZu0YcMGbdq0SceOHRu2D4DzmdGsCy+80BhjjM/nM5JMenq6OXXqlLntttsiPVdccYUxxpiCggIjydx0002mv7/fTJkyJdLz3e9+17S3t5ukpKQRfa7H4zHGGOPxeEb156MoavSrurraGGNMOBw24XDYDDV0W3V1ddznSlHUudVI99+jfg3MpEmTJH1+kV1+fr6Sk5O1Y8eOSM/Bgwd15MgRFRUVSZKKiopUX1+vlpaWSM+2bds0adIkXXXVVcN+TnJysjweT1QBcIbJkydHXg93H5jh+gA426gGGJfLpX/5l3/Rr3/9a/3v//6vJCkzM1OnTp3SiRMnonpDoZAyMzMjPaFQ6LTxwbHhlJWVqaOjI1KcCwec48CBA5I+Cyu///3vo8aOHDkSCTGDfQCcb1QfJbB+/Xrl5ubquuuuG82PkSSVl5dr3bp1kfcej4cQAzjE4H94XC6XzjvvPK1du1aNjY2aOXOm7rnnnsi9YL74HyMAzjVqAeZHP/qRSktLdf3110cFiebmZqWkpGjSpElRf2y8Xq+am5sjPddcc03U9/N6vZGx4fT29qq3tzfWPwYACxi60sjr9eqRRx6JvB96CulP3a0XgHOMyimkH/3oR/L7/Vq4cKEOHz4cNbZv3z719vaqpKQksu3yyy/XjBkzVFdXJ0mqq6vTnDlzdNFFF0V6Fi1apBMnTuj9998fjSkDsLCGhobI6zNdAzO0D4DzxfTq4fXr15vjx4+b66+/3ni93kidd955kZ5nnnnGHD582Nxwww0mLy/P7N692+zevfvzK4vdbvPee++ZrVu3mquvvtp8/etfN6FQyPzgBz+I+VXMFEVZvxITE01vb69pa2szhw4dilqF1NjYaNra2kxvb69JTEyM+1wpijq3Oov9d2w/+I+57777Ij0pKSnm6aefNq2trebkyZPm5z//ufF6vVHfZ/r06ea1114zXV1dpqWlxaxdu9YkJCSMxi+AoigbVEVFhQmHw6avry/qb0tfX58Jh8OmoqIi7nOkKOrcK24BxipFgKEoZ5Xf7z/jfWD8fn/c50hR1LmXZe4DAwDnyu1269lnn5Uk9fT0RI0Nvn/mmWfkdvMnDRgvRnUZNQDEQnFxsbxer4wx+tWvfqWenh5NnjxZ7e3tmjBhgm6++WZlZmaquLhYb731VrynC2AMEGAAWN6CBQskfXZH75tvvjly3xdJMsaora1NF1xwgRYsWECAAcYJjrcCsLzp06dLks4///xhxwe3D/YBcD4CDADLCwaDkddnug/M0D4AzsYpJACW95WvfCXyuqWlRW+99Za6urqUmpqqBQsWRO7UPbQPgLMRYABY3ty5cyOvvV6v7rrrrsj7oUdghvYBcDZOIQGwvPPOOy+mfQDsjwADwPIOHDgQeX2ma2CG9gFwNgIMAMsb+pDG3t5e7d27V7t27dLevXujnkLPwxyB8YNrYABYXkFBQeR1SkqK5s2bF3k/9AjM0D4AzsYRGACWl5KSEtM+APZHgAFgeTU1NZHXZ7oGZmgfAGcjwACwvEAgENM+APZHgAFgeV/72tcir4c+B+mL74f2AXA2AgwAy5s4cWJM+wDYHwEGgOV98sknMe0DYH8EGACWN/iso1j1AbA/AgwAy7vyyitj2gfA/ggwACwvOTk5pn0A7I8AA8Dyenp6YtoHwP4IMAAsr6+vL6Z9AOyPAAPA8pKSkmLaB8D+CDAALK+7uzumfQDsjwADwPI++uijmPYBsD8CDADLu/HGG2PaB8D+CDAALC8lJSWmfQDsjwADwPKMMTHtA2B/BBgAltfb2xvTPgD2R4ABYHmJiYkx7QNgfwQYAJbndo/sT9VI+wDYH//aAQCA7RBgAACA7RBgAFiey+WKaR8A+yPAAAAA2yHAAAAA2yHAAAAA2yHAAAAA2yHAAAAA2yHAAAAA2yHAAAAA2yHAAAAA2yHAAAAA2yHAAAAA2yHAAAAA2yHAAAAA2yHAAAAA2yHAAAAA20mM9wQAjA8TJkzQ7NmzR/1z/uzP/uysv+bDDz9UT0/PKMwGwGghwAAYE7Nnz9Y777xzTt/D5XKdcZvb7f5Sn5GXl6ff/e535zQ3AGOLAANgTHz44YfKy8v7Ul/79ttvKyEhQcYYSdGhZei2gYEBzZ8//0vNDYC9EGAAjImenp4vfZTjxhtv1K9+9asz9hhjdOONN3IkBRgnXJJMvCcxGjwejzo6OpSenq7Ozs54TwfAOerv75fb/dm6g+GOwITDYSUm8n8ywO5Guv9mFRIAW0hMTFQ4HB52jPACjD8EGAC2kZiYqIULF2pgYECSNDAwoIULFxJegHGIf/UA/qScnBx5PJ54T0OSdOLECd1zzz16+eWXdc899+jEiRNfaun0aOjs7FQgEIj3NIBxgQAD4IxycnL00UcfxXsaw3r55ZfjPYXTzJo1ixADjAECDIAzGjzycvfdd+uDDz6I82w+c9555+nSSy/V4cOH9emnn8Z7OpKkr3zlK9q4caNljlQBTmfpAPO9731PjzzyiDIzM/Xuu+/q+9//vt5+++14TwsYd4w+lTHtkjriPRVJ0qefdujDD1viPY0oxrTLyBphChgPLBtg7rjjDq1bt07Lly/Xnj179PDDD2vbtm264oor9Mknn8R7esC4MXHiRElHtPHlB+M9FRs4Eu8JAOOGZQPM3/zN3+jf//3f9cILL0iSli9frptvvll//dd/rX/6p3+K7+SAceSz5xfNkJQZ76nYQAr3nQLGiCUDTFJSkvLz81VeXh7ZZozRjh07VFRUFMeZAeNPVVWVpM9ut9/d3f2lv8/gdStWFKtraViFBIwdSwaYCy+8UImJiQqFQlHbQ6HQH32abXJyslJSUiLvuZAOiI3W1lY999xzMfledXV1Mfk+AOCYG9mVlZWpo6MjUk1NTfGeEgAAGCWWDDDHjh1Tf3+/vF5v1Hav16vm5uZhv6a8vFzp6emRmjZt2lhMFQAAxIElA0xfX5/27dunkpKSyDaXy6WSkpI/egi6t7dXnZ2dUQUAAJzJktfASNK6dev005/+VHv37tVvf/tbPfzww0pNTdXzzz8f76kBAIA4s2yAeeWVV3TRRRfpiSeeUGZmpvbv36+bbrpJLS3WunkVAAAYey5JJt6TGA0ej0cdHR1KT0/ndBIAADYx0v23Ja+BAQAAOBMCDAAAsB0CDAAAsB0CDAAAsB0CDAAAsB0CDAAAsB0CDAAAsB3L3sguVngqNQAA9jHS/bZjA8zgL4CnUgMAYD8ej+eMN7Jz7J14JWnq1KnchRdwII/Ho6amJk2bNo1/44ADeTweHT169Iw9jg4wAJyJR4UA4CJeAABgOwQYAABgOwQYALZz6tQpPf744zp16lS8pwIgTrgGBgAA2A5HYAAAgO0QYAAAgO0QYAAAgO0QYAAAgO0QYADYhs/n0y9/+Us1NTXJGKNbb7013lMCECcEGAC2kZqaqnfffVcrVqyI91QAxJljH+YIwHm2bt2qrVu3xnsaACyAIzAAAMB2CDAAAMB2CDAAAMB2CDAAAMB2CDAAAMB2WIUEwDZSU1OVk5MTeZ+dna25c+eqra1NH3/8cRxnBmCs8TRqALZRXFys6urq07a/8MIL+qu/+quxnxCAuCHAAAAA2+EaGAAAYDsEGAAAYDsEGAAAYDsEGAAAYDsEGAAAYDsEGAAAYDsEGAAAYDsEGAAAYDsEGAAAYDsEGAAAYDsEGAAAYDsEGAAAYDv/H3jDpfiOm9IDAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "freq_dict = {}\n",
    "# for pc in clips:\n",
    "#     freq_dict[pc.word] = freq_dict.get(pc.word, 0) + 1\n",
    "\n",
    "freqs = list(filter(lambda x: 5 < x < 200000, freq_dict.values()))\n",
    "\n",
    "plt.boxplot(freqs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'freq_dict' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[14], line 3\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtyping\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Dict\n\u001b[1;32m----> 3\u001b[0m wf \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mdict\u001b[39m(\u001b[38;5;28mfilter\u001b[39m(\u001b[38;5;28;01mlambda\u001b[39;00m x: \u001b[38;5;241m50\u001b[39m \u001b[38;5;241m<\u001b[39m x[\u001b[38;5;241m1\u001b[39m] \u001b[38;5;241m<\u001b[39m \u001b[38;5;241m300\u001b[39m, \u001b[43mfreq_dict\u001b[49m\u001b[38;5;241m.\u001b[39mitems()))\n\u001b[0;32m      4\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlen\u001b[39m(wf)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m words remain after filtering\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m      6\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mopen\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mvocab.txt\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mw\u001b[39m\u001b[38;5;124m'\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m f:\n",
      "\u001b[1;31mNameError\u001b[0m: name 'freq_dict' is not defined"
     ]
    }
   ],
   "source": [
    "wf = dict(filter(lambda x: 50 < x[1] < 300, freq_dict.items()))\n",
    "print(f'{len(wf)} words remain after filtering')\n",
    "\n",
    "with open('vocab.txt', 'w') as f:\n",
    "    f.write(\"\\n\".join(wf.keys()))\n",
    "    print('wrote vocabulary file')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save Audio Files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[WinError 3] The system cannot find the path specified: 'audio\\\\music'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[3], line 5\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mos\u001b[39;00m\n\u001b[0;32m      3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mdata_processing\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m make_noise_playlist\n\u001b[1;32m----> 5\u001b[0m music_clips \u001b[38;5;241m=\u001b[39m \u001b[43mmake_noise_playlist\u001b[49m\u001b[43m(\u001b[49m\u001b[43mos\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpath\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mjoin\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43maudio\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mmusic\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m200\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[0;32m      6\u001b[0m outdoor_clips \u001b[38;5;241m=\u001b[39m make_noise_playlist(os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124maudio\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124moutside_rural\u001b[39m\u001b[38;5;124m'\u001b[39m))\n\u001b[0;32m      7\u001b[0m babble_clips \u001b[38;5;241m=\u001b[39m make_noise_playlist(os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124maudio\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mbabble\u001b[39m\u001b[38;5;124m'\u001b[39m))\n",
      "File \u001b[1;32mc:\\Users\\nwong\\Documents\\nick-repo\\data_processing.py:131\u001b[0m, in \u001b[0;36mmake_noise_playlist\u001b[1;34m(path, limit)\u001b[0m\n\u001b[0;32m    129\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mmake_noise_playlist\u001b[39m(path: os\u001b[38;5;241m.\u001b[39mPathLike, limit: \u001b[38;5;28mint\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m List[AudioSegment]:\n\u001b[0;32m    130\u001b[0m     clips \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m--> 131\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m \u001b[43mos\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlistdir\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m)\u001b[49m:\n\u001b[0;32m    132\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m f\u001b[38;5;241m.\u001b[39mendswith(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mwav\u001b[39m\u001b[38;5;124m'\u001b[39m):\n\u001b[0;32m    133\u001b[0m             \u001b[38;5;28;01mcontinue\u001b[39;00m\n",
      "\u001b[1;31mFileNotFoundError\u001b[0m: [WinError 3] The system cannot find the path specified: 'audio\\\\music'"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "from data_processing import make_noise_playlist\n",
    "\n",
    "music_clips = make_noise_playlist(os.path.join('audio', 'music'), 200)\n",
    "outdoor_clips = make_noise_playlist(os.path.join('audio', 'outside_rural'))\n",
    "babble_clips = make_noise_playlist(os.path.join('audio', 'babble'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "m_path = os.path.join('parsed_files', 'music')\n",
    "o_path = os.path.join('parsed_files', 'outdoor')\n",
    "b_path = os.path.join('parsed_files', 'babble')\n",
    "\n",
    "os.makedirs(m_path, exist_ok = True)\n",
    "os.makedirs(o_path, exist_ok = True)\n",
    "os.makedirs(b_path, exist_ok = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded vocabulary vocab.txt with 1 words.\n",
      "Processing file #50, took 31.222121000289917s\n",
      "Processing file #100, took 75.3955762386322s\n",
      "Processing file #150, took 118.96602511405945s\n",
      "Processing file #200, took 172.90856909751892s\n",
      "Processing file #250, took 216.94991445541382s\n",
      "Processing file #300, took 260.6452143192291s\n",
      "Processing file #350, took 342.6633789539337s\n",
      "Processing file #400, took 446.7177982330322s\n",
      "Failed to parse file: Fuck_(film) (Decoding failed. ffmpeg returned error code: 1\n",
      "\n",
      "Output from ffmpeg/avlib:\n",
      "\n",
      "ffmpeg version N-93913-g1a74b04737 Copyright (c) 2000-2019 the FFmpeg developers\n",
      "  built with gcc 8.3.1 (GCC) 20190414\n",
      "  configuration: --enable-gpl --enable-version3 --enable-sdl2 --enable-fontconfig --enable-gnutls --enable-iconv --enable-libass --enable-libdav1d --enable-libbluray --enable-libfreetype --enable-libmp3lame --enable-libopencore-amrnb --enable-libopencore-amrwb --enable-libopenjpeg --enable-libopus --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libtheora --enable-libtwolame --enable-libvpx --enable-libwavpack --enable-libwebp --enable-libx264 --enable-libx265 --enable-libxml2 --enable-libzimg --enable-lzma --enable-zlib --enable-gmp --enable-libvidstab --enable-libvorbis --enable-libvo-amrwbenc --enable-libmysofa --enable-libspeex --enable-libxvid --enable-libaom --enable-libmfx --enable-amf --enable-ffnvcodec --enable-cuvid --enable-d3d11va --enable-nvenc --enable-nvdec --enable-dxva2 --enable-avisynth --enable-libopenmpt\n",
      "  libavutil      56. 28.100 / 56. 28.100\n",
      "  libavcodec     58. 52.102 / 58. 52.102\n",
      "  libavformat    58. 27.103 / 58. 27.103\n",
      "  libavdevice    58.  7.100 / 58.  7.100\n",
      "  libavfilter     7. 54.101 /  7. 54.101\n",
      "  libswscale      5.  4.101 /  5.  4.101\n",
      "  libswresample   3.  4.100 /  3.  4.100\n",
      "  libpostproc    55.  4.100 / 55.  4.100\n",
      "[ogg @ 000001bee7f4a440] cannot find sync word\n",
      "english\\Fuck_(film)\\audio.ogg: Invalid data found when processing input\n",
      ")\n",
      "Processing file #450, took 522.3014514446259s\n",
      "Processing file #500, took 585.1550860404968s\n",
      "Processing file #550, took 659.3704717159271s\n",
      "Processing file #600, took 752.2772107124329s\n",
      "Processing file #650, took 800.9042580127716s\n",
      "Processing file #700, took 858.1163063049316s\n",
      "Failed to parse file: Caesium (Unable to process >4GB files)\n"
     ]
    }
   ],
   "source": [
    "from data_processing import load_utterances, load_vocab\n",
    "\n",
    "vocab = load_vocab()\n",
    "load_utterances(700, vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\nwong\\Documents\\nick-repo\\.conda\\Lib\\site-packages\\pycochleagram-0.1-py3.11.egg\\pycochleagram\\erbfilter.py:8: RuntimeWarning: pycochleagram using non-interactive Agg matplotlib backend\n",
      "  from pycochleagram import utils\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "20003"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from data_processing import load_clips\n",
    "\n",
    "clips = load_clips()\n",
    "len(clips)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'os' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[2], line 18\u001b[0m\n\u001b[0;32m     12\u001b[0m \u001b[38;5;66;03m# music_snr = np.random.normal(-6, 2, N)\u001b[39;00m\n\u001b[0;32m     13\u001b[0m \u001b[38;5;66;03m# outdoor_snr = np.random.normal(-3, 2, N)\u001b[39;00m\n\u001b[0;32m     14\u001b[0m \u001b[38;5;66;03m# babble_snr = np.random.normal(-3, 2, N)\u001b[39;00m\n\u001b[0;32m     16\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtime\u001b[39;00m\n\u001b[1;32m---> 18\u001b[0m m_path \u001b[38;5;241m=\u001b[39m \u001b[43mos\u001b[49m\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mparsed_files_snrs\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmusic\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m     19\u001b[0m o_path \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mparsed_files_snrs\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124moutdoor\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m     20\u001b[0m b_path \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mparsed_files_snrs\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mbabble\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
      "\u001b[1;31mNameError\u001b[0m: name 'os' is not defined"
     ]
    }
   ],
   "source": [
    "import random\n",
    "from concurrent.futures import ThreadPoolExecutor\n",
    "\n",
    "# import numpy as np\n",
    "\n",
    "from data_processing import overlay_noise_clips\n",
    "\n",
    "# music is -6dB mean, other noise is -3dB mean. all with 2dB variance\n",
    "# np.random.seed(123)\n",
    "# N = len(clips)\n",
    "N = 6000\n",
    "# music_snr = np.random.normal(-6, 2, N)\n",
    "# outdoor_snr = np.random.normal(-3, 2, N)\n",
    "# babble_snr = np.random.normal(-3, 2, N)\n",
    "\n",
    "import time\n",
    "\n",
    "m_path = os.path.join('parsed_files_snrs', 'music')\n",
    "o_path = os.path.join('parsed_files_snrs', 'outdoor')\n",
    "b_path = os.path.join('parsed_files_snrs', 'babble')\n",
    "\n",
    "os.makedirs(m_path, exist_ok = True)\n",
    "os.makedirs(o_path, exist_ok = True)\n",
    "os.makedirs(b_path, exist_ok = True)\n",
    "\n",
    "random.seed(123)\n",
    "starttime = time.time()\n",
    "with ThreadPoolExecutor() as executor:\n",
    "    futures = [executor.submit(\n",
    "        overlay_noise_clips,\n",
    "        clip,\n",
    "        [\n",
    "            random.choice(music_clips),\n",
    "            random.choice(outdoor_clips),\n",
    "            random.choice(babble_clips)\n",
    "        ],\n",
    "        [-12.0, -6.0, -3.0, 0.0, 2.0, 6.0, 12.0],\n",
    "        [m_path, o_path, b_path],\n",
    "        i, starttime\n",
    "    ) for i, clip in enumerate(clips[N])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "from concurrent.futures import ThreadPoolExecutor\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "from data_processing import overlay_noise_clips\n",
    "\n",
    "# music is -6dB mean, other noise is -3dB mean. all with 2dB variance\n",
    "np.random.seed(123)\n",
    "N = len(clips)\n",
    "\n",
    "random.seed(123)\n",
    "starttime = time.time()\n",
    "with ThreadPoolExecutor() as executor:\n",
    "    futures = [executor.submit(\n",
    "        overlay_noise_clips,\n",
    "        clip,\n",
    "        [\n",
    "            random.choice(music_clips),\n",
    "            random.choice(outdoor_clips),\n",
    "            random.choice(babble_clips)\n",
    "        ],\n",
    "        [music_snr[i], outdoor_snr[i], babble_snr[i]],\n",
    "        [m_path, o_path, b_path],\n",
    "        i, starttime\n",
    "    ) for i, clip in enumerate(clips)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Data\n",
    "Load audio files functions, vocabulary, word-to-index dictionary, cochleagrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded vocabulary vocab.txt with 237 words.\n"
     ]
    }
   ],
   "source": [
    "from data_processing import load_clips\n",
    "\n",
    "clips = load_clips()\n",
    "vocab = load_vocab()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "237 words from 20003 clips\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'union': 0,\n",
       " 'since': 1,\n",
       " 'town': 2,\n",
       " 'public': 3,\n",
       " 'many': 4,\n",
       " 'business': 5,\n",
       " 'season': 6,\n",
       " 'include': 7,\n",
       " 'became': 8,\n",
       " 'family': 9,\n",
       " 'million': 10,\n",
       " 'countries': 11,\n",
       " 'modern': 12,\n",
       " 'long': 13,\n",
       " 'played': 14,\n",
       " 'king': 15,\n",
       " 'held': 16,\n",
       " 'fifteen': 17,\n",
       " 'early': 18,\n",
       " 'forty': 19,\n",
       " 'social': 20,\n",
       " 'part': 21,\n",
       " 'group': 22,\n",
       " 'role': 23,\n",
       " 'eleven': 24,\n",
       " 'television': 25,\n",
       " 'century': 26,\n",
       " 'created': 27,\n",
       " 'during': 28,\n",
       " 'english': 29,\n",
       " 'ninety': 30,\n",
       " 'seventy': 31,\n",
       " 'zero': 32,\n",
       " 'economic': 33,\n",
       " 'audio': 34,\n",
       " 'world': 35,\n",
       " 'largest': 36,\n",
       " 'produced': 37,\n",
       " 'made': 38,\n",
       " 'show': 39,\n",
       " 'state': 40,\n",
       " 'church': 41,\n",
       " 'another': 42,\n",
       " 'central': 43,\n",
       " 'between': 44,\n",
       " 'america': 45,\n",
       " 'best': 46,\n",
       " 'then': 47,\n",
       " 'have': 48,\n",
       " 'american': 49,\n",
       " 'born': 50,\n",
       " 'period': 51,\n",
       " 'still': 52,\n",
       " 'their': 53,\n",
       " 'them': 54,\n",
       " 'several': 55,\n",
       " 'science': 56,\n",
       " 'these': 57,\n",
       " 'term': 58,\n",
       " 'than': 59,\n",
       " 'life': 60,\n",
       " 'into': 61,\n",
       " 'only': 62,\n",
       " 'usually': 63,\n",
       " 'original': 64,\n",
       " 'death': 65,\n",
       " 'released': 66,\n",
       " 'place': 67,\n",
       " 'will': 68,\n",
       " 'thirteen': 69,\n",
       " 'similar': 70,\n",
       " 'development': 71,\n",
       " 'history': 72,\n",
       " 'become': 73,\n",
       " 'large': 74,\n",
       " 'when': 75,\n",
       " 'served': 76,\n",
       " 'episode': 77,\n",
       " 'through': 78,\n",
       " 'would': 79,\n",
       " 'president': 80,\n",
       " 'point': 81,\n",
       " 'such': 82,\n",
       " 'number': 83,\n",
       " 'person': 84,\n",
       " 'europe': 85,\n",
       " 'station': 86,\n",
       " 'different': 87,\n",
       " 'following': 88,\n",
       " 'using': 89,\n",
       " 'same': 90,\n",
       " 'united': 91,\n",
       " 'album': 92,\n",
       " 'high': 93,\n",
       " 'thirty': 94,\n",
       " 'australia': 95,\n",
       " 'canadian': 96,\n",
       " 'eighteen': 97,\n",
       " 'south': 98,\n",
       " 'september': 99,\n",
       " 'international': 100,\n",
       " 'built': 101,\n",
       " 'london': 102,\n",
       " 'years': 103,\n",
       " 'york': 104,\n",
       " 'species': 105,\n",
       " 'works': 106,\n",
       " 'located': 107,\n",
       " 'about': 108,\n",
       " 'times': 109,\n",
       " 'written': 110,\n",
       " 'title': 111,\n",
       " 'book': 112,\n",
       " 'like': 113,\n",
       " 'generally': 114,\n",
       " 'could': 115,\n",
       " 'been': 116,\n",
       " 'each': 117,\n",
       " 'lake': 118,\n",
       " 'common': 119,\n",
       " 'sometimes': 120,\n",
       " 'human': 121,\n",
       " 'both': 122,\n",
       " 'died': 123,\n",
       " 'among': 124,\n",
       " 'british': 125,\n",
       " 'because': 126,\n",
       " 'game': 127,\n",
       " 'video': 128,\n",
       " 'series': 129,\n",
       " 'east': 130,\n",
       " 'used': 131,\n",
       " 'states': 132,\n",
       " 'very': 133,\n",
       " 'work': 134,\n",
       " 'general': 135,\n",
       " 'player': 136,\n",
       " 'film': 137,\n",
       " 'later': 138,\n",
       " 'north': 139,\n",
       " 'eighty': 140,\n",
       " 'area': 141,\n",
       " 'under': 142,\n",
       " 'canada': 143,\n",
       " 'william': 144,\n",
       " 'various': 145,\n",
       " 'house': 146,\n",
       " 'some': 147,\n",
       " 'although': 148,\n",
       " 'were': 149,\n",
       " 'year': 150,\n",
       " 'based': 151,\n",
       " 'even': 152,\n",
       " 'jesus': 153,\n",
       " 'more': 154,\n",
       " 'political': 155,\n",
       " 'hundred': 156,\n",
       " 'including': 157,\n",
       " 'sixty': 158,\n",
       " 'example': 159,\n",
       " 'known': 160,\n",
       " 'named': 161,\n",
       " 'park': 162,\n",
       " 'song': 163,\n",
       " 'january': 164,\n",
       " 'name': 165,\n",
       " 'before': 166,\n",
       " 'currently': 167,\n",
       " 'population': 168,\n",
       " 'small': 169,\n",
       " 'often': 170,\n",
       " 'former': 171,\n",
       " 'eight': 172,\n",
       " 'over': 173,\n",
       " 'university': 174,\n",
       " 'city': 175,\n",
       " 'after': 176,\n",
       " 'music': 177,\n",
       " 'club': 178,\n",
       " 'those': 179,\n",
       " 'around': 180,\n",
       " 'saint': 181,\n",
       " 'time': 182,\n",
       " 'fiction': 183,\n",
       " 'november': 184,\n",
       " 'major': 185,\n",
       " 'england': 186,\n",
       " 'considered': 187,\n",
       " 'system': 188,\n",
       " 'there': 189,\n",
       " 'sixteen': 190,\n",
       " 'well': 191,\n",
       " 'fifty': 192,\n",
       " 'second': 193,\n",
       " 'government': 194,\n",
       " 'football': 195,\n",
       " 'found': 196,\n",
       " 'people': 197,\n",
       " 'seventeen': 198,\n",
       " 'while': 199,\n",
       " 'form': 200,\n",
       " 'widely': 201,\n",
       " 'without': 202,\n",
       " 'national': 203,\n",
       " 'originally': 204,\n",
       " 'being': 205,\n",
       " 'home': 206,\n",
       " 'called': 207,\n",
       " 'language': 208,\n",
       " 'published': 209,\n",
       " 'within': 210,\n",
       " 'others': 211,\n",
       " 'games': 212,\n",
       " 'company': 213,\n",
       " 'developed': 214,\n",
       " 'throughout': 215,\n",
       " 'twelve': 216,\n",
       " 'cent': 217,\n",
       " 'until': 218,\n",
       " 'began': 219,\n",
       " 'line': 220,\n",
       " 'popular': 221,\n",
       " 'school': 222,\n",
       " 'they': 223,\n",
       " 'much': 224,\n",
       " 'military': 225,\n",
       " 'members': 226,\n",
       " 'northern': 227,\n",
       " 'computer': 228,\n",
       " 'where': 229,\n",
       " 'team': 230,\n",
       " 'what': 231,\n",
       " 'october': 232,\n",
       " 'this': 233,\n",
       " 'against': 234,\n",
       " 'nine': 235,\n",
       " 'however': 236}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from data_processing import make_word2ind\n",
    "\n",
    "word2ind = make_word2ind(clips)\n",
    "print(f'{len(word2ind)} words from {len(clips)} clips')\n",
    "word2ind"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded vocabulary vocab.txt with 251 words.\n"
     ]
    }
   ],
   "source": [
    "from data_processing import load_vocab, vocab_word2ind\n",
    "\n",
    "vocab = load_vocab()\n",
    "word2ind = vocab_word2ind(vocab)\n",
    "# load_cochleagrams('parsed_files/babble', word2ind)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train/Test Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "from data_processing import load_cochleagrams\n",
    "\n",
    "X = np.zeros([0,256,256])\n",
    "T = np.zeros([0])\n",
    "\n",
    "x, t = load_cochleagrams('parsed_files/babble', word2ind, limit=12_000)\n",
    "X = np.append(X, x, axis=0)\n",
    "T = np.append(T, t, axis=0)\n",
    "x, t = load_cochleagrams('parsed_files/music', word2ind, limit=12_000)\n",
    "X = np.append(X, x, axis=0)\n",
    "T = np.append(T, t, axis=0)\n",
    "x, t = load_cochleagrams('parsed_files/outdoor', word2ind, limit=12_000)\n",
    "X = np.append(X, x, axis=0)\n",
    "T = np.append(T, t, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((36000, 256, 256), (36000,))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape, T.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, T, test_size=0.2,\n",
    "                                                    random_state=1234)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.makedirs('model_train/train', exist_ok=True)\n",
    "os.makedirs('model_train/test', exist_ok=True)\n",
    "\n",
    "np.save('model_train/train/inputs.npy', X_train)\n",
    "np.save('model_train/train/targets.npy', y_train)\n",
    "np.save('model_train/test/inputs.npy', X_test)\n",
    "np.save('model_train/test/targets.npy', y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load data with given SNR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "from data_processing import load_cochleagrams_with_snr\n",
    "\n",
    "SNRs = [-12, -6, -3, 0, 2, 6, 12]\n",
    "\n",
    "for i, snr in enumerate(SNRs):\n",
    "    X = np.zeros([0,256,256])\n",
    "    T = np.zeros([0])\n",
    "    x, t = load_cochleagrams_with_snr('parsed_files_snrs/babble', word2ind, snr, limit=2000)\n",
    "    np.save(f'model_train/test{snr}dB/inputs_babble.npy', x)\n",
    "    np.save(f'model_train/test{snr}dB/targets_babble.npy', t)\n",
    "    X = np.append(X, x, axis=0)\n",
    "    T = np.append(T, t, axis=0)\n",
    "    x, t = load_cochleagrams_with_snr('parsed_files_snrs/music', word2ind, snr, limit=2000)\n",
    "    np.save(f'model_train/test{snr}dB/inputs_music.npy', x)\n",
    "    np.save(f'model_train/test{snr}dB/targets_music.npy', t)\n",
    "    X = np.append(X, x, axis=0)\n",
    "    T = np.append(T, t, axis=0)\n",
    "    x, t = load_cochleagrams_with_snr('parsed_files_snrs/outdoor', word2ind, snr, limit=2000)\n",
    "    np.save(f'model_train/test{snr}dB/inputs_outdoor.npy', x)\n",
    "    np.save(f'model_train/test{snr}dB/targets_outdoor.npy', t)\n",
    "    X = np.append(X, x, axis=0)\n",
    "    T = np.append(T, t, axis=0)\n",
    "    np.save(f'model_train/test{snr}dB/inputs.npy', X)\n",
    "    np.save(f'model_train/test{snr}dB/targets.npy', T)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}