analyze.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import librosa\n",
    "import soundfile as sf\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/internal/internal audio/metadata_v2.csv', sep='|', header=None, names=['file_path', 'text'], dtype=object)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['file_path'] = df['file_path'].apply(lambda x: 'data/internal/internal audio/' + str(x) + '.wav')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/bahnar/metadata.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c22cf2f660cf4a35a4afad046dd01792",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2290 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/khuongnd6/tungtk2/lib/python3.6/site-packages/librosa/core/audio.py:165: UserWarning: PySoundFile failed. Trying audioread instead.\n",
      "  warnings.warn(\"PySoundFile failed. Trying audioread instead.\")\n"
     ]
    }
   ],
   "source": [
    "from tqdm.auto import tqdm\n",
    "tqdm.pandas()\n",
    "\n",
    "def process(path):\n",
    "    waveform, sample_rate = librosa.load(path)\n",
    "    if waveform.ndim > 1:\n",
    "        waveform = waveform[:, 0]\n",
    "    return waveform.shape[0]/sample_rate\n",
    "\n",
    "df['length'] = df['file_path'].progress_apply(process)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('data/bahnar/metadata.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'pd' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-988ddf5e5c43>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'data/internal/internal audio/metadata_v2.csv'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'pd' is not defined"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv('data/internal/internal audio/metadata_v2.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_path</th>\n",
       "      <th>text</th>\n",
       "      <th>length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>data/bahnar/TTS/Kinh tế-Xã hội/02/B-Dữ ...</td>\n",
       "      <td>naêr 317 ubmttqvn huyeân vinh thanh pôloêk hoâ...</td>\n",
       "      <td>13.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>data/bahnar/TTS/Kinh tế-Xã hội/02/B-Dữ ...</td>\n",
       "      <td>lôêm 06 kheêi kôl sônaêm mttqvn rim kôêp lôêm ...</td>\n",
       "      <td>11.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>data/bahnar/TTS/Kinh tế-Xã hội/02/B-Dữ ...</td>\n",
       "      <td>rim trong glong tôjroh ameêm teh ñak rim maêng...</td>\n",
       "      <td>22.417642</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>data/bahnar/TTS/Kinh tế-Xã hội/02/B-Dữ ...</td>\n",
       "      <td>kon tôring dach ñoan keât tôjroh pôm choh jang...</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>data/bahnar/TTS/Kinh tế-Xã hội/02/B-Dữ ...</td>\n",
       "      <td>höông öêng maêng krao akoêm ñiê ñaêng naê ma ñ...</td>\n",
       "      <td>23.992381</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           file_path  \\\n",
       "0  data/bahnar/TTS/Kinh tế-Xã hội/02/B-Dữ ...   \n",
       "1  data/bahnar/TTS/Kinh tế-Xã hội/02/B-Dữ ...   \n",
       "2  data/bahnar/TTS/Kinh tế-Xã hội/02/B-Dữ ...   \n",
       "3  data/bahnar/TTS/Kinh tế-Xã hội/02/B-Dữ ...   \n",
       "4  data/bahnar/TTS/Kinh tế-Xã hội/02/B-Dữ ...   \n",
       "\n",
       "                                                text     length  \n",
       "0  naêr 317 ubmttqvn huyeân vinh thanh pôloêk hoâ...  13.000000  \n",
       "1  lôêm 06 kheêi kôl sônaêm mttqvn rim kôêp lôêm ...  11.500000  \n",
       "2  rim trong glong tôjroh ameêm teh ñak rim maêng...  22.417642  \n",
       "3  kon tôring dach ñoan keât tôjroh pôm choh jang...   8.000000  \n",
       "4  höông öêng maêng krao akoêm ñiê ñaêng naê ma ñ...  23.992381  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_df =  df[(3 <= df['length']) & (df['length'] <= 10)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_df = filtered_df[filtered_df['file_path'].notna()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_df['text'] = filtered_df['text'].apply(lambda x: re.sub(r'[^\\w\\s]', '', x).lower())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_df.to_csv('data/internal/internal audio/metadata_v3.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_df = pd.read_csv('data/internal/internal audio/metadata_v3.csv', index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_path</th>\n",
       "      <th>text</th>\n",
       "      <th>length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>data/bahnar/TTS/Tuyên truyền - Chính sách...</td>\n",
       "      <td>ksô naêr 113 khul toâ chök huyeân uy vinh than...</td>\n",
       "      <td>11.016916</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>data/bahnar/TTS/Tuyên truyền - Chính sách...</td>\n",
       "      <td>truh ahrei huyeân vinh thanh jôh tôplih oeng a...</td>\n",
       "      <td>7.289252</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>data/bahnar/TTS/Kinh tế-Xã hội/20/B-Dữ ...</td>\n",
       "      <td>pôgôl oeng pôchi naê ma dech naêm ngo hnhei tô...</td>\n",
       "      <td>7.318549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>data/bahnar/TTS/Kinh tế-Xã hội/17/B-Dữ ...</td>\n",
       "      <td>lôêm sônaêm 2019 ubnd xa vinh hieâp jôh pôgôl ...</td>\n",
       "      <td>8.544853</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>data/bahnar/TTS/Kinh tế-Xã hội/23/B-Dữ ...</td>\n",
       "      <td>thu yoêk jôh thu yoêk 2 phao ñoâ cheâ lôêm tôr...</td>\n",
       "      <td>4.800000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2285</th>\n",
       "      <td>data/bahnar/TTS/Tuyên truyền - Chính sách...</td>\n",
       "      <td>tôjung pôm glong pônhroh pôlok ñai hoâi rim ch...</td>\n",
       "      <td>29.773832</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2286</th>\n",
       "      <td>data/bahnar/TTS/Kinh tế-Xã hội/22/B-Dữ ...</td>\n",
       "      <td>naê ma pôgloh höch hanh jônang pôtaêm keh kong...</td>\n",
       "      <td>14.080045</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2287</th>\n",
       "      <td>data/bahnar/TTS/Tuyên truyền - Chính sách...</td>\n",
       "      <td>sônaêm 2019 ñang uy quaân sö huyeân vinh thanh...</td>\n",
       "      <td>10.151791</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2288</th>\n",
       "      <td>data/bahnar/TTS/Tuyên truyền - Chính sách...</td>\n",
       "      <td>poâk xuoân thu yok</td>\n",
       "      <td>1.153605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2289</th>\n",
       "      <td>data/bahnar/TTS/Kinh tế-Xã hội/31/B-Dữ ...</td>\n",
       "      <td>thoi qua rim maêng pôtho khan jôh tôgum aên na...</td>\n",
       "      <td>32.375692</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2290 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              file_path  \\\n",
       "0     data/bahnar/TTS/Tuyên truyền - Chính sách...   \n",
       "1     data/bahnar/TTS/Tuyên truyền - Chính sách...   \n",
       "2     data/bahnar/TTS/Kinh tế-Xã hội/20/B-Dữ ...   \n",
       "3     data/bahnar/TTS/Kinh tế-Xã hội/17/B-Dữ ...   \n",
       "4     data/bahnar/TTS/Kinh tế-Xã hội/23/B-Dữ ...   \n",
       "...                                                 ...   \n",
       "2285  data/bahnar/TTS/Tuyên truyền - Chính sách...   \n",
       "2286  data/bahnar/TTS/Kinh tế-Xã hội/22/B-Dữ ...   \n",
       "2287  data/bahnar/TTS/Tuyên truyền - Chính sách...   \n",
       "2288  data/bahnar/TTS/Tuyên truyền - Chính sách...   \n",
       "2289  data/bahnar/TTS/Kinh tế-Xã hội/31/B-Dữ ...   \n",
       "\n",
       "                                                   text     length  \n",
       "0     ksô naêr 113 khul toâ chök huyeân uy vinh than...  11.016916  \n",
       "1     truh ahrei huyeân vinh thanh jôh tôplih oeng a...   7.289252  \n",
       "2     pôgôl oeng pôchi naê ma dech naêm ngo hnhei tô...   7.318549  \n",
       "3     lôêm sônaêm 2019 ubnd xa vinh hieâp jôh pôgôl ...   8.544853  \n",
       "4     thu yoêk jôh thu yoêk 2 phao ñoâ cheâ lôêm tôr...   4.800000  \n",
       "...                                                 ...        ...  \n",
       "2285  tôjung pôm glong pônhroh pôlok ñai hoâi rim ch...  29.773832  \n",
       "2286  naê ma pôgloh höch hanh jônang pôtaêm keh kong...  14.080045  \n",
       "2287  sônaêm 2019 ñang uy quaân sö huyeân vinh thanh...  10.151791  \n",
       "2288                                 poâk xuoân thu yok   1.153605  \n",
       "2289  thoi qua rim maêng pôtho khan jôh tôgum aên na...  32.375692  \n",
       "\n",
       "[2290 rows x 3 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_df = df.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2290, 3)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_df = filtered_df.sample(frac=1).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_df[:2000][['file_path', 'text']].to_csv('resources/filelists/bahnar/train.txt', header=False, index=False, sep='|')\n",
    "filtered_df[2000:2200][['file_path', 'text']].to_csv('resources/filelists/bahnar/test.txt', header=False, index=False, sep='|')\n",
    "filtered_df[2200:][['file_path', 'text']].to_csv('resources/filelists/bahnar/valid.txt', header=False, index=False, sep='|')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/lib/python3/dist-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "filtered_df['file_path'] = filtered_df['file_path'].apply(lambda x: 'data/infore/' + x[x.rfind('/') + 1:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'filtered_df' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-5c597da1017a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mfiltered_df\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m14000\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'file_path'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'text'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'resources/filelists/infore/train.txt'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheader\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msep\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'|'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mfiltered_df\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m14000\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m14500\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'file_path'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'text'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'resources/filelists/infore/test.txt'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheader\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msep\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'|'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mfiltered_df\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m14500\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'file_path'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'text'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'resources/filelists/infore/valid.txt'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheader\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msep\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'|'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'filtered_df' is not defined"
     ]
    }
   ],
   "source": [
    "filtered_df[:14000][['file_path', 'text']].to_csv('resources/filelists/infore/train.txt', header=False, index=False, sep='|')\n",
    "filtered_df[14000:14500][['file_path', 'text']].to_csv('resources/filelists/infore/test.txt', header=False, index=False, sep='|')\n",
    "filtered_df[14500:][['file_path', 'text']].to_csv('resources/filelists/infore/valid.txt', header=False, index=False, sep='|')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45%|████▌     | 1041/2290 [12:14<10:37,  1.96it/s]/home/khuongnd6/tungtk2/lib/python3.6/site-packages/librosa/core/audio.py:165: UserWarning: PySoundFile failed. Trying audioread instead.\n",
      "  warnings.warn(\"PySoundFile failed. Trying audioread instead.\")\n",
      "100%|██████████| 2290/2290 [26:57<00:00,  1.42it/s]\n"
     ]
    }
   ],
   "source": [
    "for file_path in tqdm(filtered_df['file_path']):\n",
    "    y, sr = librosa.load(file_path, sr=22050, mono=True)\n",
    "    sf.write(file_path, y, sr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_path</th>\n",
       "      <th>text</th>\n",
       "      <th>length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>data/infore/00000.wav</td>\n",
       "      <td>hiện nay vị trí của bàn thờ thường được đặt tr...</td>\n",
       "      <td>5.201270</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>data/infore/00001.wav</td>\n",
       "      <td>để cậu nhỏ hướng lên hay hướng xuống khi mặc q...</td>\n",
       "      <td>3.157914</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>data/infore/00002.wav</td>\n",
       "      <td>cả phố thắp đèn lồng lung linh và cảm giác ở n...</td>\n",
       "      <td>5.387029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>data/infore/00003.wav</td>\n",
       "      <td>đây là mức chi phí có lí nhất dựa trên kỹ thuậ...</td>\n",
       "      <td>6.687347</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>data/infore/00004.wav</td>\n",
       "      <td>ca sĩ thái thùy linh chia sẻ con chị không mắc...</td>\n",
       "      <td>5.479909</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14930</th>\n",
       "      <td>data/infore/14930.wav</td>\n",
       "      <td>với kiểu dáng đơn giản chiếc quần ống chạm đến...</td>\n",
       "      <td>6.873107</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14931</th>\n",
       "      <td>data/infore/14931.wav</td>\n",
       "      <td>quả thật hiếm người phụ nữ nào lại không xiêu ...</td>\n",
       "      <td>5.387029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14932</th>\n",
       "      <td>data/infore/14932.wav</td>\n",
       "      <td>dạt né khôn gà miệng lông tốt từ ngoài vo tron...</td>\n",
       "      <td>6.594467</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14933</th>\n",
       "      <td>data/infore/14933.wav</td>\n",
       "      <td>lỡ lạc chân vào phòng ngủ của vợ cũ hồ quang h...</td>\n",
       "      <td>6.501587</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14934</th>\n",
       "      <td>data/infore/14934.wav</td>\n",
       "      <td>theo đó nam sinh duy nhất sẽ hóa thành cô dâu ...</td>\n",
       "      <td>5.572789</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14651 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                   file_path  \\\n",
       "0      data/infore/00000.wav   \n",
       "1      data/infore/00001.wav   \n",
       "2      data/infore/00002.wav   \n",
       "3      data/infore/00003.wav   \n",
       "4      data/infore/00004.wav   \n",
       "...                      ...   \n",
       "14930  data/infore/14930.wav   \n",
       "14931  data/infore/14931.wav   \n",
       "14932  data/infore/14932.wav   \n",
       "14933  data/infore/14933.wav   \n",
       "14934  data/infore/14934.wav   \n",
       "\n",
       "                                                    text    length  \n",
       "0      hiện nay vị trí của bàn thờ thường được đặt tr...  5.201270  \n",
       "1      để cậu nhỏ hướng lên hay hướng xuống khi mặc q...  3.157914  \n",
       "2      cả phố thắp đèn lồng lung linh và cảm giác ở n...  5.387029  \n",
       "3      đây là mức chi phí có lí nhất dựa trên kỹ thuậ...  6.687347  \n",
       "4      ca sĩ thái thùy linh chia sẻ con chị không mắc...  5.479909  \n",
       "...                                                  ...       ...  \n",
       "14930  với kiểu dáng đơn giản chiếc quần ống chạm đến...  6.873107  \n",
       "14931  quả thật hiếm người phụ nữ nào lại không xiêu ...  5.387029  \n",
       "14932  dạt né khôn gà miệng lông tốt từ ngoài vo tron...  6.594467  \n",
       "14933  lỡ lạc chân vào phòng ngủ của vợ cũ hồ quang h...  6.501587  \n",
       "14934  theo đó nam sinh duy nhất sẽ hóa thành cô dâu ...  5.572789  \n",
       "\n",
       "[14651 rows x 3 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = 'data/bahnar/TTS/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_excel(path + 'Quản lý chung âm thanh.xlsx', sheet_name=None, engine='openpyxl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df['Giọng Nam']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Chính sách-Tuyên truyền</th>\n",
       "      <th>Unnamed: 1</th>\n",
       "      <th>Unnamed: 2</th>\n",
       "      <th>Khoa học-Công nghệ</th>\n",
       "      <th>Unnamed: 4</th>\n",
       "      <th>Kinh tế-Xã hội</th>\n",
       "      <th>Unnamed: 6</th>\n",
       "      <th>Unnamed: 7</th>\n",
       "      <th>Văn hóa-Thể thao</th>\n",
       "      <th>Unnamed: 9</th>\n",
       "      <th>Unnamed: 10</th>\n",
       "      <th>Tổng giờ âm thanh</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>VB ID</th>\n",
       "      <td>Trạng Thái</td>\n",
       "      <td>SL Phút</td>\n",
       "      <td>VB ID</td>\n",
       "      <td>Trạng Thái</td>\n",
       "      <td>VB ID</td>\n",
       "      <td>Trạng Thái</td>\n",
       "      <td>SL Phút</td>\n",
       "      <td>VB ID</td>\n",
       "      <td>Trạng Thái</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11.213833</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>01</th>\n",
       "      <td>Hoàn thành</td>\n",
       "      <td>18.3</td>\n",
       "      <td>01</td>\n",
       "      <td>?</td>\n",
       "      <td>01</td>\n",
       "      <td>Đang xử lý bugs</td>\n",
       "      <td>NaN</td>\n",
       "      <td>01</td>\n",
       "      <td>?</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>02</th>\n",
       "      <td>Hoàn thành</td>\n",
       "      <td>15.45</td>\n",
       "      <td>02</td>\n",
       "      <td>?</td>\n",
       "      <td>02</td>\n",
       "      <td>Hoàn thành</td>\n",
       "      <td>21.54</td>\n",
       "      <td>02</td>\n",
       "      <td>?</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>03</th>\n",
       "      <td>Hoàn thành</td>\n",
       "      <td>12.3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>03</td>\n",
       "      <td>Hoàn thành</td>\n",
       "      <td>11.16</td>\n",
       "      <td>03</td>\n",
       "      <td>?</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>04</th>\n",
       "      <td>Hoàn thành</td>\n",
       "      <td>19.07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>04</td>\n",
       "      <td>Hoàn thành</td>\n",
       "      <td>9.42</td>\n",
       "      <td>04</td>\n",
       "      <td>?</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Chính sách-Tuyên truyền Unnamed: 1 Unnamed: 2 Khoa học-Công nghệ  \\\n",
       "VB ID               Trạng Thái    SL Phút      VB ID         Trạng Thái   \n",
       "01                  Hoàn thành       18.3         01                    ?   \n",
       "02                  Hoàn thành      15.45         02                    ?   \n",
       "03                  Hoàn thành       12.3        NaN                  NaN   \n",
       "04                  Hoàn thành      19.07        NaN                  NaN   \n",
       "\n",
       "      Unnamed: 4 Kinh tế-Xã hội Unnamed: 6 Unnamed: 7 Văn hóa-Thể thao  \\\n",
       "VB ID      VB ID      Trạng Thái    SL Phút      VB ID       Trạng Thái   \n",
       "01            01   Đang xử lý bugs        NaN         01                  ?   \n",
       "02            02      Hoàn thành      21.54         02                  ?   \n",
       "03            03      Hoàn thành      11.16         03                  ?   \n",
       "04            04      Hoàn thành       9.42         04                  ?   \n",
       "\n",
       "       Unnamed: 9  Unnamed: 10  Tổng giờ âm thanh  \n",
       "VB ID         NaN    11.213833                NaN  \n",
       "01            NaN          NaN                NaN  \n",
       "02            NaN          NaN                NaN  \n",
       "03            NaN          NaN                NaN  \n",
       "04            NaN          NaN                NaN  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "f = []\n",
    "for (dirpath, dirnames, filenames) in os.walk(path):\n",
    "    # print(dirpath, filenames)\n",
    "    for filename in filenames:\n",
    "        if filename.endswith('.xlsx'):\n",
    "            f.append(os.path.join(dirpath, filename))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data/bahnar/TTS/Kinh tế-Xã hội/02/B-Dữ liệu đã xữ lý/0002_03-8-2019.xlsx\n",
      "data/bahnar/TTS/Kinh tế-Xã hội/03/B-Dữ liệu đã xữ lý/03.xlsx\n",
      "data/bahnar/TTS/Kinh tế-Xã hội/04/B-Dữ liệu đã xữ lý/0004_07-9-2019.xlsx\n",
      "data/bahnar/TTS/Kinh tế-Xã hội/05/B-Dữ liệu đã xữ lý/05.xlsx\n",
      "data/bahnar/TTS/Kinh tế-Xã hội/06/B-Dữ liệu đã xữ lý/0006_21-9-2019.xlsx\n",
      "data/bahnar/TTS/Kinh tế-Xã hội/07/B-Dữ liệu đã xữ lý/0007_28-9-2019.xlsx\n",
      "data/bahnar/TTS/Kinh tế-Xã hội/08/B-Dữ liệu đã xữ lý/08.xlsx\n",
      "data/bahnar/TTS/Kinh tế-Xã hội/09/B-Dữ liệu đã xữ lý/0009_12-10-2019.xlsx\n",
      "data/bahnar/TTS/Kinh tế-Xã hội/10/B-Dữ liệu đã xữ lý/0010_26-10-2019.xlsx\n",
      "data/bahnar/TTS/Kinh tế-Xã hội/11/B-Dữ liệu đã xữ lý/0011_02-11-2019.xlsx\n",
      "data/bahnar/TTS/Kinh tế-Xã hội/12/B-Dữ liệu đã xữ lý/0012_23-11-2019.xlsx\n",
      "data/bahnar/TTS/Kinh tế-Xã hội/17/B-Dữ liệu đã xữ lý/0017_04-01-2020.xlsx\n",
      "data/bahnar/TTS/Kinh tế-Xã hội/18/B-Dữ liệu đã xữ lý/0018_11-01-2020.xlsx\n",
      "data/bahnar/TTS/Kinh tế-Xã hội/20/B-Dữ liệu đã xữ lý/0020_KT-XH.xlsx\n",
      "data/bahnar/TTS/Kinh tế-Xã hội/21/B-Dữ liệu đã xữ lý/0021_KT-XH.xlsx\n",
      "data/bahnar/TTS/Kinh tế-Xã hội/22/B-Dữ liệu đã xữ lý/0022_KT-XH.xlsx\n",
      "data/bahnar/TTS/Kinh tế-Xã hội/23/B-Dữ liệu đã xữ lý/0023_KT-XH.xlsx\n",
      "data/bahnar/TTS/Kinh tế-Xã hội/24/B-Dữ liệu đã xữ lý/0024_KT-XH.xlsx\n",
      "data/bahnar/TTS/Kinh tế-Xã hội/26/B-Dữ liệu đã xữ lý/0026_KT-XH.xlsx\n",
      "data/bahnar/TTS/Kinh tế-Xã hội/27/B-Dữ liệu đã xữ lý/0027_KT-XH.xlsx\n",
      "data/bahnar/TTS/Kinh tế-Xã hội/28/B-Dữ liệu đã xữ lý/0028_KT-XH.xlsx\n",
      "data/bahnar/TTS/Kinh tế-Xã hội/29/B-Dữ liệu đã xữ lý/0029_KT-XH.xlsx\n",
      "data/bahnar/TTS/Kinh tế-Xã hội/30/B-Dữ liệu đã xữ lý/0030_KT-XH.xlsx\n",
      "data/bahnar/TTS/Kinh tế-Xã hội/31/B-Dữ liệu đã xữ lý/0031_KT-XH.xlsx\n",
      "data/bahnar/TTS/Kinh tế-Xã hội/32/B-Dữ liệu đã xữ lý/0032_KT_XH.xlsx\n",
      "data/bahnar/TTS/Tuyên truyền - Chính sách/01/B-Dữ liệu đã xử lý/01.xlsx\n",
      "data/bahnar/TTS/Tuyên truyền - Chính sách/02/B-Dữ liệu đã xử lý/02.xlsx\n",
      "data/bahnar/TTS/Tuyên truyền - Chính sách/03/B-Dữ liệu đã xử lý/03.xlsx\n",
      "data/bahnar/TTS/Tuyên truyền - Chính sách/04/B-Dữ liệu đã xữ lý/04.xlsx\n",
      "data/bahnar/TTS/Tuyên truyền - Chính sách/05/B-Dữ liệu đã xữ lý/05.xlsx\n",
      "data/bahnar/TTS/Tuyên truyền - Chính sách/06/B-Dữ liệu đã xữ lý/0006_TTCS.xlsx\n",
      "data/bahnar/TTS/Tuyên truyền - Chính sách/07/B-Dữ liệu đã xữ lý/0007_TTCS.xlsx\n",
      "data/bahnar/TTS/Tuyên truyền - Chính sách/08/B-Dữ liệu đã xữ lý/0008_TTCS.xlsx\n",
      "data/bahnar/TTS/Tuyên truyền - Chính sách/09/B-Dữ liệu đã xữ lý/0009_TTCS.xlsx\n",
      "data/bahnar/TTS/Tuyên truyền - Chính sách/10/B-Dữ liệu đã xữ lý/0010_TTCS.xlsx\n",
      "data/bahnar/TTS/Tuyên truyền - Chính sách/11/B-Dữ liệu đã xữ lý/0011_TTCS.xlsx\n",
      "data/bahnar/TTS/Tuyên truyền - Chính sách/12/B-Dữ liệu đã xữ lý/0012_TTCS.xlsx\n",
      "data/bahnar/TTS/Tuyên truyền - Chính sách/13/B-Dữ liệu đã xữ lý/0013_TTCS.xlsx\n",
      "data/bahnar/TTS/Tuyên truyền - Chính sách/14/B-Dữ liệu đã xữ lý/0014_TTCS.xlsx\n",
      "data/bahnar/TTS/Tuyên truyền - Chính sách/15/B-Dữ liệu đã xữ lý/0015_TTCS.xlsx\n",
      "data/bahnar/TTS/Tuyên truyền - Chính sách/16/B-Dữ liệu đã xữ lý/0016_TTCS.xlsx\n",
      "data/bahnar/TTS/Tuyên truyền - Chính sách/17/B-Dữ liệu đã xữ lý/17.xlsx\n",
      "data/bahnar/TTS/Tuyên truyền - Chính sách/18/B-Dữ liệu đã xữ lý/0018_TTCS.xlsx\n"
     ]
    }
   ],
   "source": [
    "df_full = pd.DataFrame()\n",
    "df_split = pd.DataFrame()\n",
    "\n",
    "def read_and_clean(xls, sheet_name) -> pd.DataFrame:\n",
    "    df = pd.read_excel(xls, sheet_name, header=None, skiprows=1)\n",
    "    df = df.dropna(how='all').dropna(axis=1, how='all')\n",
    "\n",
    "    return df\n",
    "\n",
    "for filename in f[2:]:\n",
    "    print(filename)\n",
    "    # df_transcript = df_transcript.append(pd.read_excel(filename, engine='openpyxl'), ignore_index=True)\n",
    "    xls = pd.ExcelFile(filename, engine='openpyxl')\n",
    "\n",
    "    #NOTE: inconsistency in the original data \n",
    "    try:\n",
    "        df_1 = read_and_clean(xls,'Câu ID')\n",
    "    except:\n",
    "        df_1 = read_and_clean(xls,'CÂU ID')\n",
    "\n",
    "    try:\n",
    "        df_2 = read_and_clean(xls, 'Cụm ID')\n",
    "    except:\n",
    "        df_2 = read_and_clean(xls, 'CỤM ID')\n",
    "\n",
    "    df_1[0] = df_1[0].apply(lambda id: os.path.join(filename[:filename.rfind('/')], 'Âm thanh Nam', f'{int(id):04d}.wav'))\n",
    "    df_2[1] = df_2[1].apply(lambda id: os.path.join(filename[:filename.rfind('/')], 'Âm thanh Nam', f'{id:06.1f}.wav'))\n",
    "\n",
    "\n",
    "    df_full = df_full.append(df_1)\n",
    "    df_split = df_split.append(df_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_full = df_full.rename({0:'ID', 2:'Bana'}, axis='columns')\n",
    "df_split = df_split.rename({0:'GroupID', 1:'ID', 3:'Bana'}, axis='columns')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_full['Bana'] = df_full['Bana'].apply(lambda x: x.lower())\n",
    "df_split['Bana'] = df_split['Bana'].apply(lambda x: x.lower())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/host/ubuntu/data/tungtk2/Speech-Backbones/Grad-TTS/data/bahnar/TTS/Kinh tế-Xã hội/07/B-Dữ liệu đã xữ lý/Âm thanh Nam/0054.1.wav\n",
      "/host/ubuntu/data/tungtk2/Speech-Backbones/Grad-TTS/data/bahnar/TTS/Kinh tế-Xã hội/10/B-Dữ liệu đã xữ lý/Âm thanh Nam/0014.1.wav\n",
      "/host/ubuntu/data/tungtk2/Speech-Backbones/Grad-TTS/data/bahnar/TTS/Kinh tế-Xã hội/10/B-Dữ liệu đã xữ lý/Âm thanh Nam/0014.2.wav\n",
      "/host/ubuntu/data/tungtk2/Speech-Backbones/Grad-TTS/data/bahnar/TTS/Kinh tế-Xã hội/21/B-Dữ liệu đã xữ lý/Âm thanh Nam/0231.0.wav\n",
      "/host/ubuntu/data/tungtk2/Speech-Backbones/Grad-TTS/data/bahnar/TTS/Kinh tế-Xã hội/23/B-Dữ liệu đã xữ lý/Âm thanh Nam/0006.1.wav\n",
      "/host/ubuntu/data/tungtk2/Speech-Backbones/Grad-TTS/data/bahnar/TTS/Kinh tế-Xã hội/23/B-Dữ liệu đã xữ lý/Âm thanh Nam/0008.1.wav\n",
      "/host/ubuntu/data/tungtk2/Speech-Backbones/Grad-TTS/data/bahnar/TTS/Kinh tế-Xã hội/23/B-Dữ liệu đã xữ lý/Âm thanh Nam/0009.1.wav\n",
      "/host/ubuntu/data/tungtk2/Speech-Backbones/Grad-TTS/data/bahnar/TTS/Kinh tế-Xã hội/23/B-Dữ liệu đã xữ lý/Âm thanh Nam/0012.1.wav\n",
      "/host/ubuntu/data/tungtk2/Speech-Backbones/Grad-TTS/data/bahnar/TTS/Kinh tế-Xã hội/23/B-Dữ liệu đã xữ lý/Âm thanh Nam/0016.1.wav\n",
      "/host/ubuntu/data/tungtk2/Speech-Backbones/Grad-TTS/data/bahnar/TTS/Kinh tế-Xã hội/23/B-Dữ liệu đã xữ lý/Âm thanh Nam/0024.1.wav\n",
      "/host/ubuntu/data/tungtk2/Speech-Backbones/Grad-TTS/data/bahnar/TTS/Kinh tế-Xã hội/23/B-Dữ liệu đã xữ lý/Âm thanh Nam/0025.1.wav\n",
      "/host/ubuntu/data/tungtk2/Speech-Backbones/Grad-TTS/data/bahnar/TTS/Kinh tế-Xã hội/23/B-Dữ liệu đã xữ lý/Âm thanh Nam/0062.1.wav\n",
      "/host/ubuntu/data/tungtk2/Speech-Backbones/Grad-TTS/data/bahnar/TTS/Kinh tế-Xã hội/30/B-Dữ liệu đã xữ lý/Âm thanh Nam/0004.2.wav\n",
      "/host/ubuntu/data/tungtk2/Speech-Backbones/Grad-TTS/data/bahnar/TTS/Tuyên truyền - Chính sách/04/B-Dữ liệu đã xữ lý/Âm thanh Nam/0029.1.wav\n",
      "/host/ubuntu/data/tungtk2/Speech-Backbones/Grad-TTS/data/bahnar/TTS/Tuyên truyền - Chính sách/08/B-Dữ liệu đã xữ lý/Âm thanh Nam/0032.1.wav\n",
      "/host/ubuntu/data/tungtk2/Speech-Backbones/Grad-TTS/data/bahnar/TTS/Tuyên truyền - Chính sách/08/B-Dữ liệu đã xữ lý/Âm thanh Nam/0032.2.wav\n",
      "/host/ubuntu/data/tungtk2/Speech-Backbones/Grad-TTS/data/bahnar/TTS/Tuyên truyền - Chính sách/08/B-Dữ liệu đã xữ lý/Âm thanh Nam/0032.3.wav\n"
     ]
    }
   ],
   "source": [
    "for path in df_split['ID']:\n",
    "    path = os.path.join(os.getcwd(), path)\n",
    "    if not os.path.isfile(path):\n",
    "        print(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_full['Bana'] = df_full['Bana'].apply(lambda x: re.sub(r'[^\\w\\s]', '', x))\n",
    "df_split['Bana'] = df_split['Bana'].apply(lambda x: re.sub(r'[^\\w\\s]', '', x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GroupID</th>\n",
       "      <th>ID</th>\n",
       "      <th>2</th>\n",
       "      <th>Bana</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.0</td>\n",
       "      <td>data/bahnar/TTS/Kinh tế-Xã hội/02/B-Dữ ...</td>\n",
       "      <td>đa dạng hóa các hình thức tập hợp</td>\n",
       "      <td>ña dang hoa rim tôdrong pônhoâm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>data/bahnar/TTS/Kinh tế-Xã hội/02/B-Dữ ...</td>\n",
       "      <td>xây dựng và củng cố khối đại đoàn kết toàn dân</td>\n",
       "      <td>tôjung pôm oeêng cung coêâ khoêâi ñai ñoan keât</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3.0</td>\n",
       "      <td>data/bahnar/TTS/Kinh tế-Xã hội/02/B-Dữ ...</td>\n",
       "      <td>Các phong trào thi đua yêu nước</td>\n",
       "      <td>rim trong glong tôjroh ameêm teh ñak</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.0</td>\n",
       "      <td>data/bahnar/TTS/Kinh tế-Xã hội/02/B-Dữ ...</td>\n",
       "      <td>đoàn kết thi đua lao động sản xuất</td>\n",
       "      <td>ñoan keât tôjroh pôm choh jang</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>data/bahnar/TTS/Kinh tế-Xã hội/02/B-Dữ ...</td>\n",
       "      <td>vươn lên làm giàu</td>\n",
       "      <td>kjung pôm bôdrong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>34.0</td>\n",
       "      <td>data/bahnar/TTS/Tuyên truyền - Chính sách...</td>\n",
       "      <td>Chương trình Tháng Nhân đạo năm 2020</td>\n",
       "      <td>chöông trinh khei nhaân ñao sônaêm 2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>45.0</td>\n",
       "      <td>data/bahnar/TTS/Tuyên truyền - Chính sách...</td>\n",
       "      <td>Chương trình cải cách hành chính Nhà nước giai...</td>\n",
       "      <td>tôdrong cai cach hanh chinh nha nöôcgiai ñoan ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>46.0</td>\n",
       "      <td>data/bahnar/TTS/Tuyên truyền - Chính sách...</td>\n",
       "      <td>Cơ chế “một cửa”, “một cửa liên thông”</td>\n",
       "      <td>tôdrong moât cöa moât cöa lieân thoâng</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>57.0</td>\n",
       "      <td>data/bahnar/TTS/Tuyên truyền - Chính sách...</td>\n",
       "      <td>Phần mềm một cửa điện tử VNPT-iGate</td>\n",
       "      <td>phaân meâm moât cöa ñieân tö vnptigate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>NaN</td>\n",
       "      <td>data/bahnar/TTS/Tuyên truyền - Chính sách...</td>\n",
       "      <td>Tập đoàn VNPT</td>\n",
       "      <td>taâp ñoan vnpt</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>612 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    GroupID                                                 ID  \\\n",
       "0       2.0  data/bahnar/TTS/Kinh tế-Xã hội/02/B-Dữ ...   \n",
       "1       NaN  data/bahnar/TTS/Kinh tế-Xã hội/02/B-Dữ ...   \n",
       "2       3.0  data/bahnar/TTS/Kinh tế-Xã hội/02/B-Dữ ...   \n",
       "3       4.0  data/bahnar/TTS/Kinh tế-Xã hội/02/B-Dữ ...   \n",
       "4       NaN  data/bahnar/TTS/Kinh tế-Xã hội/02/B-Dữ ...   \n",
       "..      ...                                                ...   \n",
       "12     34.0  data/bahnar/TTS/Tuyên truyền - Chính sách...   \n",
       "13     45.0  data/bahnar/TTS/Tuyên truyền - Chính sách...   \n",
       "14     46.0  data/bahnar/TTS/Tuyên truyền - Chính sách...   \n",
       "15     57.0  data/bahnar/TTS/Tuyên truyền - Chính sách...   \n",
       "16      NaN  data/bahnar/TTS/Tuyên truyền - Chính sách...   \n",
       "\n",
       "                                                    2  \\\n",
       "0                   đa dạng hóa các hình thức tập hợp   \n",
       "1      xây dựng và củng cố khối đại đoàn kết toàn dân   \n",
       "2                     Các phong trào thi đua yêu nước   \n",
       "3                  đoàn kết thi đua lao động sản xuất   \n",
       "4                                   vươn lên làm giàu   \n",
       "..                                                ...   \n",
       "12               Chương trình Tháng Nhân đạo năm 2020   \n",
       "13  Chương trình cải cách hành chính Nhà nước giai...   \n",
       "14            Cơ chế “một cửa”, “một cửa liên thông”    \n",
       "15               Phần mềm một cửa điện tử VNPT-iGate    \n",
       "16                                      Tập đoàn VNPT   \n",
       "\n",
       "                                                 Bana  \n",
       "0                     ña dang hoa rim tôdrong pônhoâm  \n",
       "1     tôjung pôm oeêng cung coêâ khoêâi ñai ñoan keât  \n",
       "2                rim trong glong tôjroh ameêm teh ñak  \n",
       "3                      ñoan keât tôjroh pôm choh jang  \n",
       "4                                   kjung pôm bôdrong  \n",
       "..                                                ...  \n",
       "12            chöông trinh khei nhaân ñao sônaêm 2020  \n",
       "13  tôdrong cai cach hanh chinh nha nöôcgiai ñoan ...  \n",
       "14             tôdrong moât cöa moât cöa lieân thoâng  \n",
       "15             phaân meâm moât cöa ñieân tö vnptigate  \n",
       "16                                     taâp ñoan vnpt  \n",
       "\n",
       "[612 rows x 4 columns]"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df_full[['ID', 'Bana']].append(df_split[['ID', 'Bana']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Bana</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>data/bahnar/TTS/Kinh tế-Xã hội/02/B-Dữ ...</td>\n",
       "      <td>naêr 317 ubmttqvn huyeân vinh thanh pôloêk hoâ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>data/bahnar/TTS/Kinh tế-Xã hội/02/B-Dữ ...</td>\n",
       "      <td>lôêm 06 kheêi kôl sônaêm mttqvn rim kôêp lôêm ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>data/bahnar/TTS/Kinh tế-Xã hội/02/B-Dữ ...</td>\n",
       "      <td>rim trong glong tôjroh ameêm teh ñak rim maêng...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>data/bahnar/TTS/Kinh tế-Xã hội/02/B-Dữ ...</td>\n",
       "      <td>kon tôring dach ñoan keât tôjroh pôm choh jang...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>data/bahnar/TTS/Kinh tế-Xã hội/02/B-Dữ ...</td>\n",
       "      <td>höông öêng maêng krao akoêm ñiê ñaêng naê ma ñ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>data/bahnar/TTS/Tuyên truyền - Chính sách...</td>\n",
       "      <td>chöông trinh khei nhaân ñao sônaêm 2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>data/bahnar/TTS/Tuyên truyền - Chính sách...</td>\n",
       "      <td>tôdrong cai cach hanh chinh nha nöôcgiai ñoan ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>data/bahnar/TTS/Tuyên truyền - Chính sách...</td>\n",
       "      <td>tôdrong moât cöa moât cöa lieân thoâng</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>data/bahnar/TTS/Tuyên truyền - Chính sách...</td>\n",
       "      <td>phaân meâm moât cöa ñieân tö vnptigate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>data/bahnar/TTS/Tuyên truyền - Chính sách...</td>\n",
       "      <td>taâp ñoan vnpt</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2310 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   ID  \\\n",
       "0   data/bahnar/TTS/Kinh tế-Xã hội/02/B-Dữ ...   \n",
       "1   data/bahnar/TTS/Kinh tế-Xã hội/02/B-Dữ ...   \n",
       "2   data/bahnar/TTS/Kinh tế-Xã hội/02/B-Dữ ...   \n",
       "3   data/bahnar/TTS/Kinh tế-Xã hội/02/B-Dữ ...   \n",
       "4   data/bahnar/TTS/Kinh tế-Xã hội/02/B-Dữ ...   \n",
       "..                                                ...   \n",
       "12  data/bahnar/TTS/Tuyên truyền - Chính sách...   \n",
       "13  data/bahnar/TTS/Tuyên truyền - Chính sách...   \n",
       "14  data/bahnar/TTS/Tuyên truyền - Chính sách...   \n",
       "15  data/bahnar/TTS/Tuyên truyền - Chính sách...   \n",
       "16  data/bahnar/TTS/Tuyên truyền - Chính sách...   \n",
       "\n",
       "                                                 Bana  \n",
       "0   naêr 317 ubmttqvn huyeân vinh thanh pôloêk hoâ...  \n",
       "1   lôêm 06 kheêi kôl sônaêm mttqvn rim kôêp lôêm ...  \n",
       "2   rim trong glong tôjroh ameêm teh ñak rim maêng...  \n",
       "3   kon tôring dach ñoan keât tôjroh pôm choh jang...  \n",
       "4   höông öêng maêng krao akoêm ñiê ñaêng naê ma ñ...  \n",
       "..                                                ...  \n",
       "12            chöông trinh khei nhaân ñao sônaêm 2020  \n",
       "13  tôdrong cai cach hanh chinh nha nöôcgiai ñoan ...  \n",
       "14             tôdrong moât cöa moât cöa lieân thoâng  \n",
       "15             phaân meâm moât cöa ñieân tö vnptigate  \n",
       "16                                     taâp ñoan vnpt  \n",
       "\n",
       "[2310 rows x 2 columns]"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.rename({'ID':'file_path', 'Bana':'text'}, axis='columns')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('data/bahnar/metadata.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CREATE LEXICON FOR BAHNARIC LANGUAGE\n",
    "Based on: https://moet.gov.vn/content/vanban/Lists/VBPQ/Attachments/1372/CT%20Tieng%20Bahnar.pdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       None\n",
       "1       None\n",
       "2       None\n",
       "3       None\n",
       "4       None\n",
       "        ... \n",
       "2285    None\n",
       "2286    None\n",
       "2287    None\n",
       "2288    None\n",
       "2289    None\n",
       "Name: text, Length: 2290, dtype: object"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chars = set()\n",
    "\n",
    "df['text'].apply(lambda s: chars.update([x for x in s]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'\\n',\n",
       " ' ',\n",
       " '0',\n",
       " '1',\n",
       " '2',\n",
       " '3',\n",
       " '4',\n",
       " '5',\n",
       " '6',\n",
       " '7',\n",
       " '8',\n",
       " '9',\n",
       " 'a',\n",
       " 'b',\n",
       " 'c',\n",
       " 'd',\n",
       " 'e',\n",
       " 'f',\n",
       " 'g',\n",
       " 'h',\n",
       " 'i',\n",
       " 'j',\n",
       " 'k',\n",
       " 'l',\n",
       " 'm',\n",
       " 'n',\n",
       " 'o',\n",
       " 'p',\n",
       " 'q',\n",
       " 'r',\n",
       " 's',\n",
       " 't',\n",
       " 'u',\n",
       " 'v',\n",
       " 'w',\n",
       " 'x',\n",
       " 'y',\n",
       " 'z',\n",
       " 'à',\n",
       " 'á',\n",
       " 'â',\n",
       " 'ã',\n",
       " 'ä',\n",
       " 'å',\n",
       " 'è',\n",
       " 'ê',\n",
       " 'ë',\n",
       " 'ì',\n",
       " 'í',\n",
       " 'ï',\n",
       " 'ñ',\n",
       " 'ò',\n",
       " 'ó',\n",
       " 'ô',\n",
       " 'õ',\n",
       " 'ö',\n",
       " 'ø',\n",
       " 'ù',\n",
       " 'ú',\n",
       " 'û',\n",
       " 'ý',\n",
       " 'ă',\n",
       " 'đ',\n",
       " 'ĩ',\n",
       " 'ũ',\n",
       " 'ơ',\n",
       " 'ư',\n",
       " 'ạ',\n",
       " 'ả',\n",
       " 'ấ',\n",
       " 'ầ',\n",
       " 'ẫ',\n",
       " 'ậ',\n",
       " 'ằ',\n",
       " 'ẵ',\n",
       " 'ẻ',\n",
       " 'ế',\n",
       " 'ề',\n",
       " 'ể',\n",
       " 'ễ',\n",
       " 'ệ',\n",
       " 'ỉ',\n",
       " 'ị',\n",
       " 'ọ',\n",
       " 'ố',\n",
       " 'ồ',\n",
       " 'ổ',\n",
       " 'ỗ',\n",
       " 'ộ',\n",
       " 'ớ',\n",
       " 'ờ',\n",
       " 'ở',\n",
       " 'ỡ',\n",
       " 'ợ',\n",
       " 'ụ',\n",
       " 'ủ',\n",
       " 'ứ',\n",
       " 'ừ',\n",
       " 'ữ',\n",
       " 'ự'}"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "characters = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'à', 'á', 'â', 'ã', 'ä', 'å', 'è', 'ê', 'ë', 'ì', 'í', 'ï', 'ñ', 'ò', 'ó', 'ô', 'õ', 'ö', 'ø', 'ù', 'ú', 'û', 'ý', 'ă', 'đ', 'ĩ', 'ũ', 'ơ', 'ư', 'ạ', 'ả', 'ấ', 'ầ', 'ẫ', 'ậ', 'ằ', 'ẵ', 'ẻ', 'ế', 'ề', 'ể', 'ễ', 'ệ', 'ỉ', 'ị', 'ọ', 'ố', 'ồ', 'ổ', 'ỗ', 'ộ', 'ớ', 'ờ', 'ở', 'ỡ', 'ợ', 'ụ', 'ủ', 'ứ', 'ừ', 'ữ', 'ự']\n",
    "double_vowels = ['ia', 'iă', 'ie', 'ië', 'iô', 'iö', 'ua', 'uă', 'ue', 'uë', 'uê']\n",
    "double_consonants = ['bl', 'br', 'by', 'ch', 'dj', 'dr', 'gl', 'gr', 'gy', 'hl', 'hm', 'hn', 'hñ', 'hr', 'hy', 'jr', 'kh', 'kl', 'kr', 'ky', 'ly', 'ml', 'mr', 'ny', 'my', 'ñr', 'ng', 'ph', 'pl', 'pr', 'py', 'sr', 'th', 'tr', 'ty']\n",
    "triple_consonants = ['hng', 'ngl', 'nhr']\n",
    "\n",
    "def create_entry(word):\n",
    "    res = word+'\\t'\n",
    "    while len(word) > 0:\n",
    "        for consonant in triple_consonants:\n",
    "            if consonant == word[:3]:\n",
    "                res += consonant + ' '\n",
    "                word = word[3:]\n",
    "                continue\n",
    "        for consonant in double_consonants:\n",
    "            if consonant == word[:2]:\n",
    "                res += consonant + ' '\n",
    "                word = word[2:]\n",
    "                continue\n",
    "        for vowel in double_vowels:\n",
    "            if vowel == word[:2]:\n",
    "                res += vowel + ' '\n",
    "                word = word[2:]\n",
    "                continue\n",
    "        for char in characters:\n",
    "            if char == word[:1]:\n",
    "                res += char + ' '\n",
    "                word = word[1:]\n",
    "                continue\n",
    "\n",
    "    res = res.rstrip()\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       None\n",
       "1       None\n",
       "2       None\n",
       "3       None\n",
       "4       None\n",
       "        ... \n",
       "2285    None\n",
       "2286    None\n",
       "2287    None\n",
       "2288    None\n",
       "2289    None\n",
       "Name: text, Length: 2290, dtype: object"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words = set()\n",
    "\n",
    "df.text.apply(lambda s: words.update(s.split()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "lexicon = list(map(create_entry, words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data/bahnar_lexicon.txt', 'w') as f:\n",
    "    for lex in lexicon:\n",
    "        f.write(f'{lex}\\n')"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "41c35d1750e41558d34c36c93b0d536dc278c953e293787191497be58dae9c0f"
  },
  "kernelspec": {
   "display_name": "Python 3.6.9 64-bit ('tungtk2': venv)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}