Unit 5-Handson.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "36a24468-fdc9-4788-a68d-6873c022b85c",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Hands-on exercise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fac593ce-0ddc-41a3-b57b-7a8bf47f57cc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting accelerate\n",
      "  Downloading accelerate-0.21.0-py3-none-any.whl (244 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m244.2/244.2 kB\u001b[0m \u001b[31m12.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from accelerate) (1.23.4)\n",
      "Requirement already satisfied: torch>=1.10.0 in /usr/local/lib/python3.9/dist-packages (from accelerate) (1.12.1+cu116)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.9/dist-packages (from accelerate) (23.0)\n",
      "Requirement already satisfied: psutil in /usr/local/lib/python3.9/dist-packages (from accelerate) (5.9.4)\n",
      "Requirement already satisfied: pyyaml in /usr/local/lib/python3.9/dist-packages (from accelerate) (5.4.1)\n",
      "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.9/dist-packages (from torch>=1.10.0->accelerate) (4.4.0)\n",
      "Installing collected packages: accelerate\n",
      "Successfully installed accelerate-0.21.0\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install accelerate -U"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "83c2fcec-64f4-48d6-a0bb-8e6b5ab0af37",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: wandb in /usr/local/lib/python3.9/dist-packages (0.13.4)\n",
      "Collecting wandb\n",
      "  Downloading wandb-0.15.5-py3-none-any.whl (2.1 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m73.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: requests<3,>=2.0.0 in /usr/local/lib/python3.9/dist-packages (from wandb) (2.28.2)\n",
      "Requirement already satisfied: docker-pycreds>=0.4.0 in /usr/local/lib/python3.9/dist-packages (from wandb) (0.4.0)\n",
      "Collecting appdirs>=1.4.3\n",
      "  Downloading appdirs-1.4.4-py2.py3-none-any.whl (9.6 kB)\n",
      "Requirement already satisfied: protobuf!=4.21.0,<5,>=3.15.0 in /usr/local/lib/python3.9/dist-packages (from wandb) (3.19.6)\n",
      "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.9/dist-packages (from wandb) (4.4.0)\n",
      "Requirement already satisfied: GitPython!=3.1.29,>=1.0.0 in /usr/local/lib/python3.9/dist-packages (from wandb) (3.1.30)\n",
      "Requirement already satisfied: Click!=8.0.0,>=7.1 in /usr/local/lib/python3.9/dist-packages (from wandb) (8.1.3)\n",
      "Requirement already satisfied: psutil>=5.0.0 in /usr/local/lib/python3.9/dist-packages (from wandb) (5.9.4)\n",
      "Requirement already satisfied: sentry-sdk>=1.0.0 in /usr/local/lib/python3.9/dist-packages (from wandb) (1.14.0)\n",
      "Requirement already satisfied: PyYAML in /usr/local/lib/python3.9/dist-packages (from wandb) (5.4.1)\n",
      "Requirement already satisfied: setproctitle in /usr/local/lib/python3.9/dist-packages (from wandb) (1.3.2)\n",
      "Requirement already satisfied: setuptools in /usr/local/lib/python3.9/dist-packages (from wandb) (66.1.1)\n",
      "Requirement already satisfied: pathtools in /usr/local/lib/python3.9/dist-packages (from wandb) (0.1.2)\n",
      "Requirement already satisfied: six>=1.4.0 in /usr/lib/python3/dist-packages (from docker-pycreds>=0.4.0->wandb) (1.14.0)\n",
      "Requirement already satisfied: gitdb<5,>=4.0.1 in /usr/local/lib/python3.9/dist-packages (from GitPython!=3.1.29,>=1.0.0->wandb) (4.0.10)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/lib/python3/dist-packages (from requests<3,>=2.0.0->wandb) (2019.11.28)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests<3,>=2.0.0->wandb) (1.26.14)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.9/dist-packages (from requests<3,>=2.0.0->wandb) (2.1.1)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests<3,>=2.0.0->wandb) (2.8)\n",
      "Requirement already satisfied: smmap<6,>=3.0.1 in /usr/local/lib/python3.9/dist-packages (from gitdb<5,>=4.0.1->GitPython!=3.1.29,>=1.0.0->wandb) (5.0.0)\n",
      "Installing collected packages: appdirs, wandb\n",
      "  Attempting uninstall: wandb\n",
      "    Found existing installation: wandb 0.13.4\n",
      "    Uninstalling wandb-0.13.4:\n",
      "      Successfully uninstalled wandb-0.13.4\n",
      "Successfully installed appdirs-1.4.4 wandb-0.15.5\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install wandb --upgrade"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "09f8c5d7-da5b-43ce-8a17-bea904fdc037",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: datasets in /usr/local/lib/python3.9/dist-packages (2.4.0)\n",
      "Requirement already satisfied: xxhash in /usr/local/lib/python3.9/dist-packages (from datasets) (3.2.0)\n",
      "Requirement already satisfied: pyarrow>=6.0.0 in /usr/local/lib/python3.9/dist-packages (from datasets) (10.0.1)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.9/dist-packages (from datasets) (23.0)\n",
      "Requirement already satisfied: dill<0.3.6 in /usr/local/lib/python3.9/dist-packages (from datasets) (0.3.5.1)\n",
      "Requirement already satisfied: huggingface-hub<1.0.0,>=0.1.0 in /usr/local/lib/python3.9/dist-packages (from datasets) (0.12.0)\n",
      "Requirement already satisfied: pandas in /usr/local/lib/python3.9/dist-packages (from datasets) (1.5.0)\n",
      "Requirement already satisfied: responses<0.19 in /usr/local/lib/python3.9/dist-packages (from datasets) (0.18.0)\n",
      "Requirement already satisfied: aiohttp in /usr/local/lib/python3.9/dist-packages (from datasets) (3.8.3)\n",
      "Requirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.9/dist-packages (from datasets) (2023.1.0)\n",
      "Requirement already satisfied: multiprocess in /usr/local/lib/python3.9/dist-packages (from datasets) (0.70.13)\n",
      "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from datasets) (1.23.4)\n",
      "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.9/dist-packages (from datasets) (4.64.1)\n",
      "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.9/dist-packages (from datasets) (2.28.2)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (18.2.0)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (1.3.1)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (6.0.4)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (1.3.3)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (1.8.2)\n",
      "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (2.1.1)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (4.0.2)\n",
      "Requirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (3.9.0)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (5.4.1)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (4.4.0)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/lib/python3/dist-packages (from requests>=2.19.0->datasets) (2019.11.28)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests>=2.19.0->datasets) (1.26.14)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests>=2.19.0->datasets) (2.8)\n",
      "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.9/dist-packages (from pandas->datasets) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.9/dist-packages (from pandas->datasets) (2022.7.1)\n",
      "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.14.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0mCollecting soundfile\n",
      "  Downloading soundfile-0.12.1-py2.py3-none-manylinux_2_31_x86_64.whl (1.2 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m6.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: cffi>=1.0 in /usr/local/lib/python3.9/dist-packages (from soundfile) (1.15.1)\n",
      "Requirement already satisfied: pycparser in /usr/local/lib/python3.9/dist-packages (from cffi>=1.0->soundfile) (2.21)\n",
      "Installing collected packages: soundfile\n",
      "Successfully installed soundfile-0.12.1\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0mCollecting librosa\n",
      "  Downloading librosa-0.10.0.post2-py3-none-any.whl (253 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m253.0/253.0 kB\u001b[0m \u001b[31m35.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: scipy>=1.2.0 in /usr/local/lib/python3.9/dist-packages (from librosa) (1.9.2)\n",
      "Collecting lazy-loader>=0.1\n",
      "  Downloading lazy_loader-0.3-py3-none-any.whl (9.1 kB)\n",
      "Requirement already satisfied: numpy!=1.22.0,!=1.22.1,!=1.22.2,>=1.20.3 in /usr/local/lib/python3.9/dist-packages (from librosa) (1.23.4)\n",
      "Requirement already satisfied: decorator>=4.3.0 in /usr/local/lib/python3.9/dist-packages (from librosa) (5.1.1)\n",
      "Requirement already satisfied: soundfile>=0.12.1 in /usr/local/lib/python3.9/dist-packages (from librosa) (0.12.1)\n",
      "Collecting audioread>=2.1.9\n",
      "  Downloading audioread-3.0.0.tar.gz (377 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m377.0/377.0 kB\u001b[0m \u001b[31m83.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting pooch<1.7,>=1.0\n",
      "  Downloading pooch-1.6.0-py3-none-any.whl (56 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.3/56.3 kB\u001b[0m \u001b[31m20.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: msgpack>=1.0 in /usr/local/lib/python3.9/dist-packages (from librosa) (1.0.4)\n",
      "Requirement already satisfied: scikit-learn>=0.20.0 in /usr/local/lib/python3.9/dist-packages (from librosa) (1.1.2)\n",
      "Requirement already satisfied: joblib>=0.14 in /usr/local/lib/python3.9/dist-packages (from librosa) (1.2.0)\n",
      "Requirement already satisfied: typing-extensions>=4.1.1 in /usr/local/lib/python3.9/dist-packages (from librosa) (4.4.0)\n",
      "Collecting numba>=0.51.0\n",
      "  Downloading numba-0.57.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (3.6 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.6/3.6 MB\u001b[0m \u001b[31m94.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n",
      "\u001b[?25hCollecting soxr>=0.3.2\n",
      "  Downloading soxr-0.3.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m108.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting llvmlite<0.41,>=0.40.0dev0\n",
      "  Downloading llvmlite-0.40.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (42.1 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.1/42.1 MB\u001b[0m \u001b[31m42.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.9/dist-packages (from pooch<1.7,>=1.0->librosa) (2.28.2)\n",
      "Requirement already satisfied: appdirs>=1.3.0 in /usr/local/lib/python3.9/dist-packages (from pooch<1.7,>=1.0->librosa) (1.4.4)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.9/dist-packages (from pooch<1.7,>=1.0->librosa) (23.0)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.9/dist-packages (from scikit-learn>=0.20.0->librosa) (3.1.0)\n",
      "Requirement already satisfied: cffi>=1.0 in /usr/local/lib/python3.9/dist-packages (from soundfile>=0.12.1->librosa) (1.15.1)\n",
      "Requirement already satisfied: pycparser in /usr/local/lib/python3.9/dist-packages (from cffi>=1.0->soundfile>=0.12.1->librosa) (2.21)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests>=2.19.0->pooch<1.7,>=1.0->librosa) (1.26.14)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests>=2.19.0->pooch<1.7,>=1.0->librosa) (2.8)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.9/dist-packages (from requests>=2.19.0->pooch<1.7,>=1.0->librosa) (2.1.1)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/lib/python3/dist-packages (from requests>=2.19.0->pooch<1.7,>=1.0->librosa) (2019.11.28)\n",
      "Building wheels for collected packages: audioread\n",
      "  Building wheel for audioread (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for audioread: filename=audioread-3.0.0-py3-none-any.whl size=23702 sha256=0a597f2d2fa4ab7ec0eae9b75ee6a0659b279a9bc1430f9df94693f5a81f9c8c\n",
      "  Stored in directory: /root/.cache/pip/wheels/7a/6a/b0/92760a6d6bc2bff5464970af910c0b0b921390993f3199cdf7\n",
      "Successfully built audioread\n",
      "Installing collected packages: soxr, llvmlite, lazy-loader, audioread, pooch, numba, librosa\n",
      "Successfully installed audioread-3.0.0 lazy-loader-0.3 librosa-0.10.0.post2 llvmlite-0.40.1 numba-0.57.1 pooch-1.6.0 soxr-0.3.5\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install datasets\n",
    "!pip install soundfile\n",
    "!pip install librosa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "29691f33-e0f5-49e4-8d44-9b6017662da7",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting git+https://github.com/huggingface/transformers\n",
      "  Cloning https://github.com/huggingface/transformers to /tmp/pip-req-build-5qtbngdz\n",
      "  Running command git clone --filter=blob:none --quiet https://github.com/huggingface/transformers /tmp/pip-req-build-5qtbngdz\n",
      "  Resolved https://github.com/huggingface/transformers to commit 5bb4430edc7df9f9950d412d98bbe505cc4d328b\n",
      "  Installing build dependencies ... \u001b[?25ldone\n",
      "\u001b[?25h  Getting requirements to build wheel ... \u001b[?25ldone\n",
      "\u001b[?25h  Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n",
      "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from transformers==4.31.0.dev0) (3.9.0)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.9/dist-packages (from transformers==4.31.0.dev0) (2022.10.31)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.9/dist-packages (from transformers==4.31.0.dev0) (23.0)\n",
      "Collecting huggingface-hub<1.0,>=0.14.1\n",
      "  Downloading huggingface_hub-0.16.4-py3-none-any.whl (268 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m268.8/268.8 kB\u001b[0m \u001b[31m38.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from transformers==4.31.0.dev0) (1.23.4)\n",
      "Collecting safetensors>=0.3.1\n",
      "  Downloading safetensors-0.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m100.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.9/dist-packages (from transformers==4.31.0.dev0) (4.64.1)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from transformers==4.31.0.dev0) (5.4.1)\n",
      "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.9/dist-packages (from transformers==4.31.0.dev0) (0.12.1)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.9/dist-packages (from transformers==4.31.0.dev0) (2.28.2)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers==4.31.0.dev0) (4.4.0)\n",
      "Requirement already satisfied: fsspec in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers==4.31.0.dev0) (2023.1.0)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/lib/python3/dist-packages (from requests->transformers==4.31.0.dev0) (2019.11.28)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.9/dist-packages (from requests->transformers==4.31.0.dev0) (2.1.1)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests->transformers==4.31.0.dev0) (2.8)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests->transformers==4.31.0.dev0) (1.26.14)\n",
      "Building wheels for collected packages: transformers\n",
      "  Building wheel for transformers (pyproject.toml) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for transformers: filename=transformers-4.31.0.dev0-py3-none-any.whl size=7352645 sha256=874338ad5a5e7174cbfddb29981da4d6d63f3fce3edc6b42bb599a3a984a6f78\n",
      "  Stored in directory: /tmp/pip-ephem-wheel-cache-_j0lvzdj/wheels/14/a0/7b/8f6b25ba4110aa215fcb8d6aedd6cd4f9b9b6619190999ac2b\n",
      "Successfully built transformers\n",
      "Installing collected packages: safetensors, huggingface-hub, transformers\n",
      "  Attempting uninstall: huggingface-hub\n",
      "    Found existing installation: huggingface-hub 0.12.0\n",
      "    Uninstalling huggingface-hub-0.12.0:\n",
      "      Successfully uninstalled huggingface-hub-0.12.0\n",
      "  Attempting uninstall: transformers\n",
      "    Found existing installation: transformers 4.21.3\n",
      "    Uninstalling transformers-4.21.3:\n",
      "      Successfully uninstalled transformers-4.21.3\n",
      "Successfully installed huggingface-hub-0.16.4 safetensors-0.3.1 transformers-4.31.0.dev0\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install git+https://github.com/huggingface/transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d8bcae10-8104-439c-a62b-90b5f3bd3847",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: huggingface_hub in /usr/local/lib/python3.9/dist-packages (0.16.4)\n",
      "Requirement already satisfied: fsspec in /usr/local/lib/python3.9/dist-packages (from huggingface_hub) (2023.1.0)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from huggingface_hub) (5.4.1)\n",
      "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.9/dist-packages (from huggingface_hub) (4.64.1)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.9/dist-packages (from huggingface_hub) (4.4.0)\n",
      "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.9/dist-packages (from huggingface_hub) (23.0)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.9/dist-packages (from huggingface_hub) (2.28.2)\n",
      "Requirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from huggingface_hub) (3.9.0)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests->huggingface_hub) (1.26.14)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/lib/python3/dist-packages (from requests->huggingface_hub) (2019.11.28)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.9/dist-packages (from requests->huggingface_hub) (2.1.1)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests->huggingface_hub) (2.8)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install -U huggingface_hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "95c12d2c-0122-424e-a6c8-d0ffe99f3565",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting evaluate\n",
      "  Downloading evaluate-0.4.0-py3-none-any.whl (81 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m81.4/81.4 kB\u001b[0m \u001b[31m21.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: huggingface-hub>=0.7.0 in /usr/local/lib/python3.9/dist-packages (from evaluate) (0.16.4)\n",
      "Requirement already satisfied: pandas in /usr/local/lib/python3.9/dist-packages (from evaluate) (1.5.0)\n",
      "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.9/dist-packages (from evaluate) (4.64.1)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.9/dist-packages (from evaluate) (23.0)\n",
      "Requirement already satisfied: responses<0.19 in /usr/local/lib/python3.9/dist-packages (from evaluate) (0.18.0)\n",
      "Requirement already satisfied: dill in /usr/local/lib/python3.9/dist-packages (from evaluate) (0.3.5.1)\n",
      "Requirement already satisfied: multiprocess in /usr/local/lib/python3.9/dist-packages (from evaluate) (0.70.13)\n",
      "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.9/dist-packages (from evaluate) (2.28.2)\n",
      "Requirement already satisfied: fsspec[http]>=2021.05.0 in /usr/local/lib/python3.9/dist-packages (from evaluate) (2023.1.0)\n",
      "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from evaluate) (1.23.4)\n",
      "Requirement already satisfied: xxhash in /usr/local/lib/python3.9/dist-packages (from evaluate) (3.2.0)\n",
      "Requirement already satisfied: datasets>=2.0.0 in /usr/local/lib/python3.9/dist-packages (from evaluate) (2.4.0)\n",
      "Requirement already satisfied: aiohttp in /usr/local/lib/python3.9/dist-packages (from datasets>=2.0.0->evaluate) (3.8.3)\n",
      "Requirement already satisfied: pyarrow>=6.0.0 in /usr/local/lib/python3.9/dist-packages (from datasets>=2.0.0->evaluate) (10.0.1)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from huggingface-hub>=0.7.0->evaluate) (5.4.1)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.9/dist-packages (from huggingface-hub>=0.7.0->evaluate) (4.4.0)\n",
      "Requirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from huggingface-hub>=0.7.0->evaluate) (3.9.0)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/lib/python3/dist-packages (from requests>=2.19.0->evaluate) (2019.11.28)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.9/dist-packages (from requests>=2.19.0->evaluate) (2.1.1)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests>=2.19.0->evaluate) (2.8)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests>=2.19.0->evaluate) (1.26.14)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.9/dist-packages (from pandas->evaluate) (2022.7.1)\n",
      "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.9/dist-packages (from pandas->evaluate) (2.8.2)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (6.0.4)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.3.3)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (4.0.2)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.3.1)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.8.2)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (18.2.0)\n",
      "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.8.1->pandas->evaluate) (1.14.0)\n",
      "Installing collected packages: evaluate\n",
      "Successfully installed evaluate-0.4.0\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0mCollecting jiwer\n",
      "  Downloading jiwer-3.0.2-py3-none-any.whl (21 kB)\n",
      "Requirement already satisfied: click<9.0.0,>=8.1.3 in /usr/local/lib/python3.9/dist-packages (from jiwer) (8.1.3)\n",
      "Collecting rapidfuzz==2.13.7\n",
      "  Downloading rapidfuzz-2.13.7-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.2 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.2/2.2 MB\u001b[0m \u001b[31m76.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: rapidfuzz, jiwer\n",
      "Successfully installed jiwer-3.0.2 rapidfuzz-2.13.7\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install evaluate\n",
    "!pip install jiwer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "010b7baf-0a0c-4f42-af39-22850d0a9e20",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading package lists... Done\n",
      "Building dependency tree       \n",
      "Reading state information... Done\n",
      "The following NEW packages will be installed:\n",
      "  git-lfs\n",
      "0 upgraded, 1 newly installed, 0 to remove and 3 not upgraded.\n",
      "Need to get 3316 kB of archives.\n",
      "After this operation, 11.1 MB of additional disk space will be used.\n",
      "Get:1 http://archive.ubuntu.com/ubuntu focal/universe amd64 git-lfs amd64 2.9.2-1 [3316 kB]\n",
      "Fetched 3316 kB in 1s (3644 kB/s) \n",
      "Selecting previously unselected package git-lfs.\n",
      "(Reading database ... 69943 files and directories currently installed.)\n",
      "Preparing to unpack .../git-lfs_2.9.2-1_amd64.deb ...\n",
      "Unpacking git-lfs (2.9.2-1) ...\n",
      "Setting up git-lfs (2.9.2-1) ...\n",
      "Processing triggers for man-db (2.9.1-1) ...\n"
     ]
    }
   ],
   "source": [
    "!apt-get install git-lfs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "75040f31-6bdf-43df-a916-243c59dd04bd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updated git hooks.\n",
      "Git LFS initialized.\n"
     ]
    }
   ],
   "source": [
    "!git lfs install"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "27fdea5e-9d8f-4bf6-ac8a-a94cb757faf7",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "69fc23b6d918498287d7aa3aea3328f1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from huggingface_hub import notebook_login\n",
    "notebook_login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4d4cae4e-0802-4da8-a6f6-971eb7276c42",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Logging into wandb.ai. (Learn how to deploy a W&B server locally: https://wandb.me/wandb-server)\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: You can find your API key in your browser here: https://wandb.ai/authorize\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Paste an API key from your profile and hit enter, or press ctrl+c to quit:"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "  ········\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import wandb\n",
    "wandb.login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "9cd1e4ec-7ed9-48b9-a711-3c0e46f10b8e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mptah23\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.15.5"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/notebooks/wandb/run-20230715_154118-uh6x2qpj</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Resuming run <strong><a href='https://wandb.ai/ptah23/huggingface-audio-course-unit5-handson-af/runs/uh6x2qpj' target=\"_blank\">royal-glitter-1</a></strong> to <a href='https://wandb.ai/ptah23/huggingface-audio-course-unit5-handson-af' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View project at <a href='https://wandb.ai/ptah23/huggingface-audio-course-unit5-handson-af' target=\"_blank\">https://wandb.ai/ptah23/huggingface-audio-course-unit5-handson-af</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run at <a href='https://wandb.ai/ptah23/huggingface-audio-course-unit5-handson-af/runs/uh6x2qpj' target=\"_blank\">https://wandb.ai/ptah23/huggingface-audio-course-unit5-handson-af/runs/uh6x2qpj</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<button onClick=\"this.nextSibling.style.display='block';this.style.display='none';\">Display W&B run</button><iframe src='https://wandb.ai/ptah23/huggingface-audio-course-unit5-handson-af/runs/uh6x2qpj?jupyter=true' style='border:none;width:100%;height:420px;display:none;'></iframe>"
      ],
      "text/plain": [
       "<wandb.sdk.wandb_run.Run at 0x7f9d251b74f0>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wandb.init(project=\"huggingface-audio-course-unit5-handson-af\", resume=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ad1cb43-4fdd-41da-9729-ffd24063f1c6",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Prepare Environment"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ea93abfd-5126-4302-b933-45be8cf01b95",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Load Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "b9b13c5d-766a-4dd9-aeb8-bc5b661f7d4b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "51f189c1588d40129c75440afcae4e8c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading builder script:   0%|          | 0.00/12.6k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset fleurs/af_za to /root/.cache/huggingface/datasets/google___fleurs/af_za/2.0.0/af82dbec419a815084fa63ebd5d5a9f24a6e9acdf9887b9e3b8c6bbd64e0b7ac...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "76ce87701cb54047b7a3211767b18e47",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data files:   0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a2ca5b4830384f3dae2cc4dea5915206",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/647M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "86ec4449aff548b7a214f2737e4d0a62",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/111M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bbd64e9e54e941a0932697c1886b6645",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/156M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ce6bda3e2696449f9d1d95c5cafea688",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Extracting data files:   0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6e2958f76cce4bb0a0d8f89bb61276b0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data files:   0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5dd9151b0dba4bafb6bb5121ce9c5299",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/595k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "627686f8a7da46639aec2ad101850704",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/108k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0b910f51da0242c1a5f569c6819e0db1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data:   0%|          | 0.00/150k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset fleurs downloaded and prepared to /root/.cache/huggingface/datasets/google___fleurs/af_za/2.0.0/af82dbec419a815084fa63ebd5d5a9f24a6e9acdf9887b9e3b8c6bbd64e0b7ac. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['id', 'num_samples', 'path', 'audio', 'transcription', 'raw_transcription', 'gender', 'lang_id', 'language', 'lang_group_id'],\n",
       "    num_rows: 1230\n",
       "})"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from datasets import load_dataset, DatasetDict, Audio\n",
    "fleurs = load_dataset(\"google/fleurs\", name=\"af_za\", split=\"train+validation\")\n",
    "fleurs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "7d416827-5724-474c-b4cd-136c50cc2ef4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "#from datasets import load_dataset, DatasetDict\n",
    "#minds_dataset = minds.train_test_split(len(minds) - 450)\n",
    "#minds_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f63f68cd-e5a8-48ea-ba7d-75b9cd4d26cb",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['audio', 'transcription'],\n",
       "    num_rows: 1230\n",
       "})"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fleurs_dataset = fleurs.remove_columns(['id', 'num_samples', 'path', 'raw_transcription', 'gender', 'lang_id', 'language', 'lang_group_id'])\n",
    "fleurs_dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b3e4b199-c68e-45c0-ba9e-21aab966480c",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Feature Extractor, Tokenizer and Processor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "9cb1fcb7-ad03-420d-af68-ff68b43a5524",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'english': 'en',\n",
       " 'chinese': 'zh',\n",
       " 'german': 'de',\n",
       " 'spanish': 'es',\n",
       " 'russian': 'ru',\n",
       " 'korean': 'ko',\n",
       " 'french': 'fr',\n",
       " 'japanese': 'ja',\n",
       " 'portuguese': 'pt',\n",
       " 'turkish': 'tr',\n",
       " 'polish': 'pl',\n",
       " 'catalan': 'ca',\n",
       " 'dutch': 'nl',\n",
       " 'arabic': 'ar',\n",
       " 'swedish': 'sv',\n",
       " 'italian': 'it',\n",
       " 'indonesian': 'id',\n",
       " 'hindi': 'hi',\n",
       " 'finnish': 'fi',\n",
       " 'vietnamese': 'vi',\n",
       " 'hebrew': 'he',\n",
       " 'ukrainian': 'uk',\n",
       " 'greek': 'el',\n",
       " 'malay': 'ms',\n",
       " 'czech': 'cs',\n",
       " 'romanian': 'ro',\n",
       " 'danish': 'da',\n",
       " 'hungarian': 'hu',\n",
       " 'tamil': 'ta',\n",
       " 'norwegian': 'no',\n",
       " 'thai': 'th',\n",
       " 'urdu': 'ur',\n",
       " 'croatian': 'hr',\n",
       " 'bulgarian': 'bg',\n",
       " 'lithuanian': 'lt',\n",
       " 'latin': 'la',\n",
       " 'maori': 'mi',\n",
       " 'malayalam': 'ml',\n",
       " 'welsh': 'cy',\n",
       " 'slovak': 'sk',\n",
       " 'telugu': 'te',\n",
       " 'persian': 'fa',\n",
       " 'latvian': 'lv',\n",
       " 'bengali': 'bn',\n",
       " 'serbian': 'sr',\n",
       " 'azerbaijani': 'az',\n",
       " 'slovenian': 'sl',\n",
       " 'kannada': 'kn',\n",
       " 'estonian': 'et',\n",
       " 'macedonian': 'mk',\n",
       " 'breton': 'br',\n",
       " 'basque': 'eu',\n",
       " 'icelandic': 'is',\n",
       " 'armenian': 'hy',\n",
       " 'nepali': 'ne',\n",
       " 'mongolian': 'mn',\n",
       " 'bosnian': 'bs',\n",
       " 'kazakh': 'kk',\n",
       " 'albanian': 'sq',\n",
       " 'swahili': 'sw',\n",
       " 'galician': 'gl',\n",
       " 'marathi': 'mr',\n",
       " 'punjabi': 'pa',\n",
       " 'sinhala': 'si',\n",
       " 'khmer': 'km',\n",
       " 'shona': 'sn',\n",
       " 'yoruba': 'yo',\n",
       " 'somali': 'so',\n",
       " 'afrikaans': 'af',\n",
       " 'occitan': 'oc',\n",
       " 'georgian': 'ka',\n",
       " 'belarusian': 'be',\n",
       " 'tajik': 'tg',\n",
       " 'sindhi': 'sd',\n",
       " 'gujarati': 'gu',\n",
       " 'amharic': 'am',\n",
       " 'yiddish': 'yi',\n",
       " 'lao': 'lo',\n",
       " 'uzbek': 'uz',\n",
       " 'faroese': 'fo',\n",
       " 'haitian creole': 'ht',\n",
       " 'pashto': 'ps',\n",
       " 'turkmen': 'tk',\n",
       " 'nynorsk': 'nn',\n",
       " 'maltese': 'mt',\n",
       " 'sanskrit': 'sa',\n",
       " 'luxembourgish': 'lb',\n",
       " 'myanmar': 'my',\n",
       " 'tibetan': 'bo',\n",
       " 'tagalog': 'tl',\n",
       " 'malagasy': 'mg',\n",
       " 'assamese': 'as',\n",
       " 'tatar': 'tt',\n",
       " 'hawaiian': 'haw',\n",
       " 'lingala': 'ln',\n",
       " 'hausa': 'ha',\n",
       " 'bashkir': 'ba',\n",
       " 'javanese': 'jw',\n",
       " 'sundanese': 'su',\n",
       " 'burmese': 'my',\n",
       " 'valencian': 'ca',\n",
       " 'flemish': 'nl',\n",
       " 'haitian': 'ht',\n",
       " 'letzeburgesch': 'lb',\n",
       " 'pushto': 'ps',\n",
       " 'panjabi': 'pa',\n",
       " 'moldavian': 'ro',\n",
       " 'moldovan': 'ro',\n",
       " 'sinhalese': 'si',\n",
       " 'castilian': 'es'}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers.models.whisper.tokenization_whisper import TO_LANGUAGE_CODE\n",
    "TO_LANGUAGE_CODE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "54ac1726-ab77-4114-b27b-f31931ff6061",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f7265a76dc214af883f54a1c0a8ae7d9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)rocessor_config.json:   0%|          | 0.00/185k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b07e4e35503b432f89ae75f23af31be6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)okenizer_config.json:   0%|          | 0.00/842 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b6f77e83555548ba9a0c920806bd73de",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)olve/main/vocab.json:   0%|          | 0.00/1.04M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f9f381f6156345bd8c2da9f88825078c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)/main/tokenizer.json:   0%|          | 0.00/2.20M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "557b6cf7d4464e988969021344231ff8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)olve/main/merges.txt:   0%|          | 0.00/494k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "723f31c32e3e43669cda51deed33ab92",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)main/normalizer.json:   0%|          | 0.00/52.7k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ef12406c184148009ad77c0b07688678",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)in/added_tokens.json:   0%|          | 0.00/2.08k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7a7d711dfa644bae9340b77122603ed0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)cial_tokens_map.json:   0%|          | 0.00/2.08k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# in case of from scratch training run this\n",
    "# NL is closest\n",
    "\n",
    "from transformers import WhisperProcessor\n",
    "\n",
    "processor = WhisperProcessor.from_pretrained(\"openai/whisper-small\", language=\"dutch\", task=\"transcribe\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "ba8ffc65-41eb-4f17-8058-b9d70d5424b3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6626d3b1769f4ab7a22470274a3f77a1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)rocessor_config.json:   0%|          | 0.00/339 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1b087e6d45914ebd83a32cf618984bbc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)okenizer_config.json:   0%|          | 0.00/805 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3fe855c5e993404dbedacfcad3850f70",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)olve/main/vocab.json:   0%|          | 0.00/1.04M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4d95c235dd5b42c295f2db9daf8d24a9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)olve/main/merges.txt:   0%|          | 0.00/494k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c13006d5b004420ba24d8035d5096ef7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)main/normalizer.json:   0%|          | 0.00/52.7k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2868015fdbb547208fb76d5ffeed18dc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)in/added_tokens.json:   0%|          | 0.00/2.08k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "828b09e359d04ff18cfa8139444b16c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)cial_tokens_map.json:   0%|          | 0.00/2.08k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# in case of continuing training, run this\n",
    "from transformers import WhisperProcessor\n",
    "\n",
    "processor = WhisperProcessor.from_pretrained(\"ptah23/whisper-small-af-ZA\", language=\"afrikaans\", task=\"transcribe\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5e2045f4-ba78-4358-a55c-a08476a2c704",
   "metadata": {},
   "source": [
    "### Pre-process the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "8e1e04cf-a89b-41c1-964e-bedf8b80a63e",
   "metadata": {},
   "outputs": [],
   "source": [
    "fleurs_dataset = fleurs_dataset.train_test_split(0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "17ed72ab-5634-475a-ba41-ea8095f51be8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'audio': Audio(sampling_rate=16000, mono=True, decode=True, id=None),\n",
       " 'transcription': Value(dtype='string', id=None)}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fleurs_dataset[\"train\"].features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "549642d9-f1ee-4cc2-a002-0f85230a4901",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16000"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from datasets import Audio\n",
    "sampling_rate=processor.feature_extractor.sampling_rate\n",
    "fleurs_dataset = fleurs_dataset.cast_column(\"audio\", Audio(sampling_rate=sampling_rate))\n",
    "sampling_rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "6ba8fa95-1cb1-4a73-b911-728ce3b0d768",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2a769441eaef47ed84b49ef2135902e9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1107 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4ea6e7ae6f7349079944d7fbbec5ffe7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/123 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# load and resample\n",
    "# compute log mel spectrogram\n",
    "# encode transcriptions to label ids with tokenizer\n",
    "def prepare_dataset(example):\n",
    "    audio = example[\"audio\"]\n",
    "    example = processor(audio=audio[\"array\"], sampling_rate=audio[\"sampling_rate\"], text=example[\"transcription\"])\n",
    "    # compute input length of audio sample inseconds\n",
    "    example[\"input_length\"] = len(audio[\"array\"]) / audio[\"sampling_rate\"]\n",
    "    \n",
    "    return example\n",
    "fleurs_dataset = fleurs_dataset.map(prepare_dataset, remove_columns=fleurs_dataset.column_names[\"train\"], num_proc=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "8c98ac28-b19f-4314-aff8-c0e68e991eb3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1e7129858e9f4c238ce3d0b1ad50220d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "max_input_length = 30.0\n",
    "\n",
    "def is_audio_in_length_range(length):\n",
    "    return length < max_input_length\n",
    "\n",
    "# filter\n",
    "\n",
    "fleurs_dataset[\"train\"] = fleurs_dataset[\"train\"].filter(is_audio_in_length_range, input_columns=[\"input_length\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "37bf391c-af42-4242-8cc1-31528a57fb44",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['input_features', 'labels', 'input_length'],\n",
       "    num_rows: 1102\n",
       "})"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fleurs_dataset[\"train\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "16da1b40-f43b-4b9f-831a-e4da43927d8a",
   "metadata": {},
   "source": [
    "### Training and evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7c754b41-8482-441e-b24d-4dea4fc014ed",
   "metadata": {},
   "source": [
    "- Define a data collator: the data collator takes our pre-processed data and prepares PyTorch tensors ready for the model.\n",
    "- Evaluation metrics: during evaluation, we want to evaluate the model using the word error rate (WER) metric. We need to define a compute_metrics function that handles this computation.\n",
    "- Load a pre-trained checkpoint: we need to load a pre-trained checkpoint and configure it correctly for training.\n",
    "- Define the training arguments: these will be used by the 🤗 Trainer in constructing the training schedule."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "92f9191c-f41f-440a-a0b1-d86affb3921a",
   "metadata": {},
   "source": [
    "#### Define a Data Collator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "19fa7bfd-9bcc-4c96-853a-2e8a0faff8c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from dataclasses import dataclass\n",
    "from typing import Any, Dict, List, Union\n",
    "\n",
    "@dataclass\n",
    "class DataCollatorSpeechSeq2SeqWithPadding:\n",
    "    processor: Any\n",
    "    \n",
    "    def __call__(self, features:List[Dict[str,Union[List[int], torch.Tensor]]]) -> Dict[str, torch.Tensor]:\n",
    "        # split inputs and labels since they have to be of different lengths and need different padding methods\n",
    "        # first treat the audio inputs by simply returning torch tensors\n",
    "        input_features = [{\"input_features\":feature[\"input_features\"][0]} for feature in features]\n",
    "        batch = self.processor.feature_extractor.pad(input_features, return_tensors=\"pt\")\n",
    "        # get tokenized label sequences\n",
    "        label_features =[{\"input_ids\":feature[\"labels\"]} for feature in features]\n",
    "        # pad labels\n",
    "        labels_batch = self.processor.tokenizer.pad(label_features, return_tensors=\"pt\")\n",
    "        # replace padding with -100\n",
    "        labels = labels_batch[\"input_ids\"].masked_fill(labels_batch.attention_mask.ne(1), -100)\n",
    "        # remove start of transcript token appended by tokenizer as it's appended later anyway\n",
    "        if(labels[:, 0] == self.processor.tokenizer.bos_token_id).all().cpu().item():\n",
    "            labels = labels[:, 1:]\n",
    "        batch[\"labels\"] = labels\n",
    "        return batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "d297ed6d-8293-434a-9234-068db97cd085",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_collator = DataCollatorSpeechSeq2SeqWithPadding(processor=processor)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ac86e4e-3d71-4655-958c-b13c793f5096",
   "metadata": {},
   "source": [
    "#### Evaluation metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "5a7bf000-6b77-49a1-8fdb-b7dfee3dc1f1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2b27d294f83442dfa7f69f04d715846e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading builder script:   0%|          | 0.00/4.49k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import evaluate\n",
    "metric = evaluate.load(\"wer\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "10f31de0-3251-4c7d-a655-dee55f2c5992",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers.models.whisper.english_normalizer import BasicTextNormalizer\n",
    "normalizer = BasicTextNormalizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "fbb22a3f-9cf5-49ad-bffd-adcd7084d100",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_metrics(pred):\n",
    "    pred_ids = pred.predictions\n",
    "    label_ids = pred.label_ids\n",
    "    \n",
    "    # replace -100 with pad token\n",
    "    label_ids[label_ids == -100] = processor.tokenizer.pad_token_id\n",
    "    # do not group tokens when computing metrics\n",
    "    pred_str = processor.batch_decode(pred_ids, skip_special_tokens=True)\n",
    "    label_str = processor.batch_decode(label_ids, skip_special_tokens=True)\n",
    "    \n",
    "    # compute ortographic wer\n",
    "    wer_ortho = metric.compute(predictions=pred_str, references=label_str)\n",
    "    \n",
    "    # compute normalized wer\n",
    "    pred_str_norm = [normalizer(pred) for pred in pred_str]\n",
    "    label_str_norm = [normalizer(label) for label in label_str]\n",
    "    \n",
    "    # only evaluate non-zero references\n",
    "    pred_str_norm = [pred_str_norm[i] for i in range(len(pred_str_norm)) if len(label_str_norm[i]) > 0]\n",
    "    label_str_norm = [label_str_norm[i] for i in range(len(label_str_norm)) if len(label_str_norm[i]) > 0]\n",
    "\n",
    "    wer = metric.compute(predictions=pred_str_norm, references=label_str_norm)\n",
    "\n",
    "    return {\"wer_ortho\": wer_ortho, \"wer\": wer}\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4e977aa9-9735-4e61-b261-e9fde0a57cdd",
   "metadata": {},
   "source": [
    "#### Load a pre-trained checkpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "6e03d706-d8ea-42b4-baa6-dc1eccd75f8b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5f662841f9c04874826c20775c15e03c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)lve/main/config.json:   0%|          | 0.00/1.97k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0851c2b97bcc40d8822d76e088224226",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading pytorch_model.bin:   0%|          | 0.00/967M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f306b7f04ece4613a1bd5086ff5a0666",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)neration_config.json:   0%|          | 0.00/3.51k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# in case of from scratch training run this\n",
    "from transformers import WhisperForConditionalGeneration\n",
    "\n",
    "model = WhisperForConditionalGeneration.from_pretrained(\"openai/whisper-small\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "c87963b7-4437-4a49-aef6-d823cdb0a66e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "165f2ccbe312454883d1655af95e426a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)lve/main/config.json:   0%|          | 0.00/2.27k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1f3e3e5348da4c27bf70b93c8b49863e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading pytorch_model.bin:   0%|          | 0.00/967M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9f060643289142099c3bb1c79a0359bb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)neration_config.json:   0%|          | 0.00/3.60k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# in case of continuing training run this\n",
    "from transformers import WhisperForConditionalGeneration\n",
    "\n",
    "model = WhisperForConditionalGeneration.from_pretrained(\"ptah23/whisper-small-af-ZA\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "cb7161e6-1f22-4856-a2f4-feb7feb7f5cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "from functools import partial\n",
    "\n",
    "# disable cache during training as it's incompatible with gradient checkpointing\n",
    "model.config.use_cache = False\n",
    "\n",
    "# set language and task for generation and re-enable cache\n",
    "model.generate = partial(model.generate, language=\"afrikaans\", task=\"transcribe\", use_cache=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "044604bc-2c59-41bc-9c09-ab0f9e56160e",
   "metadata": {},
   "source": [
    "### Define a training configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "0163b8b0-d8c3-4e71-bbad-27eaf60ac632",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import Seq2SeqTrainingArguments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "398fbc52-cec1-4e69-97e6-c7087716a6c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "training_args = Seq2SeqTrainingArguments(\n",
    "    output_dir=\"./whisper-small-af-ZA\", # name on HF Hub\n",
    "    per_device_train_batch_size=16,\n",
    "    gradient_accumulation_steps=1, # increase by 2 for every 2x decrease in batch size\n",
    "    learning_rate=1e-5,\n",
    "    lr_scheduler_type=\"constant_with_warmup\",\n",
    "    warmup_steps=5,\n",
    "    max_steps=2000, \n",
    "    gradient_checkpointing=True,\n",
    "    fp16=True,\n",
    "    fp16_full_eval=True,\n",
    "    evaluation_strategy=\"steps\",\n",
    "    per_device_eval_batch_size=16,\n",
    "    predict_with_generate=True,\n",
    "    generation_max_length=225,\n",
    "    save_steps=100,\n",
    "    eval_steps=100,\n",
    "    logging_steps=50,\n",
    "    report_to=[\"wandb\"],\n",
    "    load_best_model_at_end=True,\n",
    "    metric_for_best_model=\"wer\",\n",
    "    greater_is_better=False,\n",
    "    push_to_hub=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "52aac65a-65ec-4860-bf56-0d369242be4b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/notebooks/./whisper-small-af-ZA is already a clone of https://huggingface.co/ptah23/whisper-small-af-ZA. Make sure you pull the latest changes with `repo.git_pull()`.\n"
     ]
    }
   ],
   "source": [
    "from transformers import Seq2SeqTrainer\n",
    "\n",
    "trainer = Seq2SeqTrainer(\n",
    "    args=training_args,\n",
    "    model=model,\n",
    "    train_dataset=fleurs_dataset[\"train\"],\n",
    "    eval_dataset=fleurs_dataset[\"test\"],\n",
    "    data_collator=data_collator,\n",
    "    compute_metrics=compute_metrics,\n",
    "    tokenizer=processor,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d06de2ea-e250-4d32-9da4-1d8315e11ba1",
   "metadata": {},
   "source": [
    "#### Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "074dd210-15d5-4972-9389-16c28dbfce9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "d904eb2e-4f74-4c9b-92e1-35c7fb8384f5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.9/dist-packages/transformers/optimization.py:411: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2001' max='2000' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [2000/2000 00:04, Epoch 29/29]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=2001, training_loss=0.00033052798094361025, metrics={'train_runtime': 79.9278, 'train_samples_per_second': 400.361, 'train_steps_per_second': 25.023, 'total_flos': 9.22261222342656e+18, 'train_loss': 0.00033052798094361025, 'epoch': 29.0})"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train(resume_from_checkpoint=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "6ba40150-edd2-4e19-820c-e1bd0d532863",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7cc7494223344ed9bfef4d1657eac884",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Upload file training_args.bin:   0%|          | 1.00/3.98k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "To https://huggingface.co/ptah23/whisper-small-af-ZA\n",
      "   faa67a1..e007133  main -> main\n",
      "\n",
      "To https://huggingface.co/ptah23/whisper-small-af-ZA\n",
      "   e007133..0d26d00  main -> main\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'https://huggingface.co/ptah23/whisper-small-af-ZA/commit/e007133527c92577a02e1cb01789ac5063448b96'"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kwargs = {\n",
    "    \"dataset_tags\": \"google/fleurs\",\n",
    "    \"finetuned_from\": \"openai/whisper-small\",\n",
    "    \"tasks\": \"automatic-speech-recognition\",\n",
    "}\n",
    "trainer.push_to_hub(**kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "235b6f72-577f-4ce4-9913-ad014853ff55",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}