speech_to_text.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Vv7zm9QnDOhX",
        "outputId": "ae55252c-d31f-437f-c16e-ab99e737db6a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting git+https://github.com/openai/whisper.git\n",
            "  Cloning https://github.com/openai/whisper.git to /tmp/pip-req-build-c4zbfcex\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/openai/whisper.git /tmp/pip-req-build-c4zbfcex\n",
            "  Resolved https://github.com/openai/whisper.git to commit 517a43ecd132a2089d85f4ebc044728a71d49f6e\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: more-itertools in /usr/local/lib/python3.10/dist-packages (from openai-whisper==20240930) (10.5.0)\n",
            "Requirement already satisfied: numba in /usr/local/lib/python3.10/dist-packages (from openai-whisper==20240930) (0.60.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from openai-whisper==20240930) (1.26.4)\n",
            "Requirement already satisfied: tiktoken in /usr/local/lib/python3.10/dist-packages (from openai-whisper==20240930) (0.8.0)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (from openai-whisper==20240930) (2.5.1+cu121)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from openai-whisper==20240930) (4.67.1)\n",
            "Requirement already satisfied: triton>=2 in /usr/local/lib/python3.10/dist-packages (from openai-whisper==20240930) (3.1.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from triton>=2->openai-whisper==20240930) (3.16.1)\n",
            "Requirement already satisfied: llvmlite<0.44,>=0.43.0dev0 in /usr/local/lib/python3.10/dist-packages (from numba->openai-whisper==20240930) (0.43.0)\n",
            "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken->openai-whisper==20240930) (2024.11.6)\n",
            "Requirement already satisfied: requests>=2.26.0 in /usr/local/lib/python3.10/dist-packages (from tiktoken->openai-whisper==20240930) (2.32.3)\n",
            "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch->openai-whisper==20240930) (4.12.2)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch->openai-whisper==20240930) (3.4.2)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch->openai-whisper==20240930) (3.1.5)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch->openai-whisper==20240930) (2024.9.0)\n",
            "Requirement already satisfied: sympy==1.13.1 in /usr/local/lib/python3.10/dist-packages (from torch->openai-whisper==20240930) (1.13.1)\n",
            "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy==1.13.1->torch->openai-whisper==20240930) (1.3.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken->openai-whisper==20240930) (3.4.1)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken->openai-whisper==20240930) (3.10)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken->openai-whisper==20240930) (2.3.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken->openai-whisper==20240930) (2024.12.14)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch->openai-whisper==20240930) (3.0.2)\n"
          ]
        }
      ],
      "source": [
        "!pip install git+https://github.com/openai/whisper.git"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NA96luoHDggo",
        "outputId": "5c80a7c3-5520-499a-ddee-1ea62a37d304"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: datasets in /usr/local/lib/python3.10/dist-packages (3.2.0)\n",
            "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.47.1)\n",
            "Requirement already satisfied: torchaudio in /usr/local/lib/python3.10/dist-packages (2.5.1+cu121)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from datasets) (3.16.1)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from datasets) (1.26.4)\n",
            "Requirement already satisfied: pyarrow>=15.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (17.0.0)\n",
            "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.3.8)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (2.2.2)\n",
            "Requirement already satisfied: requests>=2.32.2 in /usr/local/lib/python3.10/dist-packages (from datasets) (2.32.3)\n",
            "Requirement already satisfied: tqdm>=4.66.3 in /usr/local/lib/python3.10/dist-packages (from datasets) (4.67.1)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets) (3.5.0)\n",
            "Requirement already satisfied: multiprocess<0.70.17 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.70.16)\n",
            "Requirement already satisfied: fsspec<=2024.9.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from fsspec[http]<=2024.9.0,>=2023.1.0->datasets) (2024.9.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.11.11)\n",
            "Requirement already satisfied: huggingface-hub>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.27.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from datasets) (24.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (6.0.2)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2024.11.6)\n",
            "Requirement already satisfied: tokenizers<0.22,>=0.21 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.21.0)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.5.0)\n",
            "Requirement already satisfied: torch==2.5.1 in /usr/local/lib/python3.10/dist-packages (from torchaudio) (2.5.1+cu121)\n",
            "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch==2.5.1->torchaudio) (4.12.2)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch==2.5.1->torchaudio) (3.4.2)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch==2.5.1->torchaudio) (3.1.5)\n",
            "Requirement already satisfied: sympy==1.13.1 in /usr/local/lib/python3.10/dist-packages (from torch==2.5.1->torchaudio) (1.13.1)\n",
            "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy==1.13.1->torch==2.5.1->torchaudio) (1.3.0)\n",
            "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (2.4.4)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.2)\n",
            "Requirement already satisfied: async-timeout<6.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (24.3.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.5.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.1.0)\n",
            "Requirement already satisfied: propcache>=0.2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (0.2.1)\n",
            "Requirement already satisfied: yarl<2.0,>=1.17.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.18.3)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.2->datasets) (3.4.1)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.2->datasets) (3.10)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.2->datasets) (2.3.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.2->datasets) (2024.12.14)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2024.2)\n",
            "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2024.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.17.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch==2.5.1->torchaudio) (3.0.2)\n"
          ]
        }
      ],
      "source": [
        "!pip install datasets transformers torchaudio"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ly5rm-PZE01q",
        "outputId": "f7a51ae1-40cd-474c-8b92-52e3c0248fae"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_auth.py:94: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n",
            "Reading metadata...: 28893it [00:01, 20671.14it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'client_id': 'a90b9c9411bc2b1989b02b27b19c25d11be7de25cfb73ac3c3e056b2ece3051d9c49a09f77d0d54b1e786dafb9d67a463a286e1d3dd1c5868bb8f09dc6043b31', 'path': 'fa_train_0/common_voice_fa_33316953.mp3', 'audio': {'path': 'fa_train_0/common_voice_fa_33316953.mp3', 'array': array([ 3.34477024e-28, -6.18466950e-28, -6.81575822e-28, ...,\n",
            "       -8.99972506e-19,  1.39591030e-18, -1.99899776e-18]), 'sampling_rate': 48000}, 'sentence': 'او مثل آقا رفتار کرد.', 'up_votes': 2, 'down_votes': 0, 'age': '', 'gender': '', 'accent': '', 'locale': 'fa', 'segment': '', 'variant': ''}\n",
            "{'client_id': 'a90b9c9411bc2b1989b02b27b19c25d11be7de25cfb73ac3c3e056b2ece3051d9c49a09f77d0d54b1e786dafb9d67a463a286e1d3dd1c5868bb8f09dc6043b31', 'path': 'fa_train_0/common_voice_fa_33316954.mp3', 'audio': {'path': 'fa_train_0/common_voice_fa_33316954.mp3', 'array': array([-2.84217094e-14,  1.56319402e-13,  1.31450406e-13, ...,\n",
            "        2.63489037e-05, -7.29196472e-08, -8.64365666e-06]), 'sampling_rate': 48000}, 'sentence': 'زیبایی او را نمیتوانم وصف کنم.', 'up_votes': 2, 'down_votes': 0, 'age': '', 'gender': '', 'accent': '', 'locale': 'fa', 'segment': '', 'variant': ''}\n",
            "{'client_id': 'a90b9c9411bc2b1989b02b27b19c25d11be7de25cfb73ac3c3e056b2ece3051d9c49a09f77d0d54b1e786dafb9d67a463a286e1d3dd1c5868bb8f09dc6043b31', 'path': 'fa_train_0/common_voice_fa_33316956.mp3', 'audio': {'path': 'fa_train_0/common_voice_fa_33316956.mp3', 'array': array([ 1.77635684e-15, -5.77315973e-15, -3.01980663e-14, ...,\n",
            "        3.85612736e-07,  1.03579737e-06,  8.92328160e-07]), 'sampling_rate': 48000}, 'sentence': 'بدون قید و شرط', 'up_votes': 2, 'down_votes': 0, 'age': '', 'gender': '', 'accent': '', 'locale': 'fa', 'segment': '', 'variant': ''}\n",
            "{'client_id': 'a90b9c9411bc2b1989b02b27b19c25d11be7de25cfb73ac3c3e056b2ece3051d9c49a09f77d0d54b1e786dafb9d67a463a286e1d3dd1c5868bb8f09dc6043b31', 'path': 'fa_train_0/common_voice_fa_33316957.mp3', 'audio': {'path': 'fa_train_0/common_voice_fa_33316957.mp3', 'array': array([-6.25277607e-13,  3.41060513e-13,  2.27373675e-12, ...,\n",
            "       -1.19191202e-04, -1.37187468e-04, -9.42411789e-05]), 'sampling_rate': 48000}, 'sentence': 'این وظیفهی تو است، سعی نکن آن را به من محول کنی!', 'up_votes': 2, 'down_votes': 0, 'age': '', 'gender': '', 'accent': '', 'locale': 'fa', 'segment': '', 'variant': ''}\n",
            "{'client_id': 'a90b9c9411bc2b1989b02b27b19c25d11be7de25cfb73ac3c3e056b2ece3051d9c49a09f77d0d54b1e786dafb9d67a463a286e1d3dd1c5868bb8f09dc6043b31', 'path': 'fa_train_0/common_voice_fa_33316975.mp3', 'audio': {'path': 'fa_train_0/common_voice_fa_33316975.mp3', 'array': array([ 4.26325641e-14, -1.47792889e-12, -9.66338121e-13, ...,\n",
            "       -4.54122419e-05, -1.01251244e-05,  1.23778955e-06]), 'sampling_rate': 48000}, 'sentence': 'به زودی فیلم شروع خواهد شد.', 'up_votes': 2, 'down_votes': 0, 'age': '', 'gender': '', 'accent': '', 'locale': 'fa', 'segment': '', 'variant': ''}\n"
          ]
        }
      ],
      "source": [
        "from huggingface_hub import login\n",
        "\n",
        "login(token=\"<Your token>\")\n",
        "\n",
        "from datasets import load_dataset\n",
        "\n",
        "# Load the default configuration of the dataset\n",
        "dataset = load_dataset(\"mozilla-foundation/common_voice_17_0\", \"fa\", streaming=True)\n",
        "\n",
        "# Check the first few samples to identify the 'language' field\n",
        "for sample in dataset[\"train\"].take(5):\n",
        "    print(sample)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "--GK4OZOGbMW",
        "outputId": "5d2e3508-836b-492c-de24-e8737bfc5b9f"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Reading metadata...: 28893it [00:01, 27169.88it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'client_id': 'a90b9c9411bc2b1989b02b27b19c25d11be7de25cfb73ac3c3e056b2ece3051d9c49a09f77d0d54b1e786dafb9d67a463a286e1d3dd1c5868bb8f09dc6043b31', 'path': 'fa_train_0/common_voice_fa_33316953.mp3', 'audio': {'path': 'fa_train_0/common_voice_fa_33316953.mp3', 'array': array([ 3.34477024e-28, -6.18466950e-28, -6.81575822e-28, ...,\n",
            "       -8.99972506e-19,  1.39591030e-18, -1.99899776e-18]), 'sampling_rate': 48000}, 'sentence': 'او مثل آقا رفتار کرد.', 'up_votes': 2, 'down_votes': 0, 'age': '', 'gender': '', 'accent': '', 'locale': 'fa', 'segment': '', 'variant': ''}\n",
            "{'client_id': 'a90b9c9411bc2b1989b02b27b19c25d11be7de25cfb73ac3c3e056b2ece3051d9c49a09f77d0d54b1e786dafb9d67a463a286e1d3dd1c5868bb8f09dc6043b31', 'path': 'fa_train_0/common_voice_fa_33316954.mp3', 'audio': {'path': 'fa_train_0/common_voice_fa_33316954.mp3', 'array': array([-2.84217094e-14,  1.56319402e-13,  1.31450406e-13, ...,\n",
            "        2.63489037e-05, -7.29196472e-08, -8.64365666e-06]), 'sampling_rate': 48000}, 'sentence': 'زیبایی او را نمیتوانم وصف کنم.', 'up_votes': 2, 'down_votes': 0, 'age': '', 'gender': '', 'accent': '', 'locale': 'fa', 'segment': '', 'variant': ''}\n",
            "{'client_id': 'a90b9c9411bc2b1989b02b27b19c25d11be7de25cfb73ac3c3e056b2ece3051d9c49a09f77d0d54b1e786dafb9d67a463a286e1d3dd1c5868bb8f09dc6043b31', 'path': 'fa_train_0/common_voice_fa_33316956.mp3', 'audio': {'path': 'fa_train_0/common_voice_fa_33316956.mp3', 'array': array([ 1.77635684e-15, -5.77315973e-15, -3.01980663e-14, ...,\n",
            "        3.85612736e-07,  1.03579737e-06,  8.92328160e-07]), 'sampling_rate': 48000}, 'sentence': 'بدون قید و شرط', 'up_votes': 2, 'down_votes': 0, 'age': '', 'gender': '', 'accent': '', 'locale': 'fa', 'segment': '', 'variant': ''}\n",
            "{'client_id': 'a90b9c9411bc2b1989b02b27b19c25d11be7de25cfb73ac3c3e056b2ece3051d9c49a09f77d0d54b1e786dafb9d67a463a286e1d3dd1c5868bb8f09dc6043b31', 'path': 'fa_train_0/common_voice_fa_33316957.mp3', 'audio': {'path': 'fa_train_0/common_voice_fa_33316957.mp3', 'array': array([-6.25277607e-13,  3.41060513e-13,  2.27373675e-12, ...,\n",
            "       -1.19191202e-04, -1.37187468e-04, -9.42411789e-05]), 'sampling_rate': 48000}, 'sentence': 'این وظیفهی تو است، سعی نکن آن را به من محول کنی!', 'up_votes': 2, 'down_votes': 0, 'age': '', 'gender': '', 'accent': '', 'locale': 'fa', 'segment': '', 'variant': ''}\n",
            "{'client_id': 'a90b9c9411bc2b1989b02b27b19c25d11be7de25cfb73ac3c3e056b2ece3051d9c49a09f77d0d54b1e786dafb9d67a463a286e1d3dd1c5868bb8f09dc6043b31', 'path': 'fa_train_0/common_voice_fa_33316975.mp3', 'audio': {'path': 'fa_train_0/common_voice_fa_33316975.mp3', 'array': array([ 4.26325641e-14, -1.47792889e-12, -9.66338121e-13, ...,\n",
            "       -4.54122419e-05, -1.01251244e-05,  1.23778955e-06]), 'sampling_rate': 48000}, 'sentence': 'به زودی فیلم شروع خواهد شد.', 'up_votes': 2, 'down_votes': 0, 'age': '', 'gender': '', 'accent': '', 'locale': 'fa', 'segment': '', 'variant': ''}\n"
          ]
        }
      ],
      "source": [
        "train_dataset = dataset[\"train\"]\n",
        "validation_dataset = dataset[\"validation\"]\n",
        "\n",
        "# Preview a few samples\n",
        "for sample in train_dataset.take(5):\n",
        "    print(sample)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VWYNC_2wHmjD"
      },
      "outputs": [],
      "source": [
        "from datasets import Audio\n",
        "\n",
        "# Cast the \"audio\" column to download the files locally\n",
        "train_dataset = train_dataset.cast_column(\"audio\", Audio())\n",
        "validation_dataset = validation_dataset.cast_column(\"audio\", Audio())\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cz_fkImTIFLn",
        "outputId": "f08c7d9f-011c-4fee-e138-0a604493d55c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (2.5.1+cu121)\n",
            "Requirement already satisfied: torchaudio in /usr/local/lib/python3.10/dist-packages (2.5.1+cu121)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch) (3.16.1)\n",
            "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch) (4.12.2)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch) (3.4.2)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch) (3.1.5)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch) (2024.9.0)\n",
            "Requirement already satisfied: sympy==1.13.1 in /usr/local/lib/python3.10/dist-packages (from torch) (1.13.1)\n",
            "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy==1.13.1->torch) (1.3.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch) (3.0.2)\n"
          ]
        }
      ],
      "source": [
        "!pip install torch torchaudio"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DAcRGWv7fk3W"
      },
      "outputs": [],
      "source": [
        "def preprocess_and_save(dataset, split_name, chunk_size=100):\n",
        "    import torchaudio\n",
        "\n",
        "    batch = []\n",
        "    chunk_index = 0  # To keep track of the current chunk index\n",
        "\n",
        "    for i, example in enumerate(dataset):\n",
        "        # Access the audio path\n",
        "        audio = example[\"audio\"][\"array\"]  # Extract raw audio data\n",
        "        transcript = example[\"sentence\"]\n",
        "\n",
        "        batch.append({\"audio\": audio, \"transcript\": transcript})\n",
        "\n",
        "        # Save the chunk when the batch reaches the specified size\n",
        "        if (i + 1) % chunk_size == 0:\n",
        "            chunk_path = f\"processed/{split_name}/chunk_{chunk_index}.pt\"\n",
        "            torch.save(batch, chunk_path)\n",
        "            batch = []  # Clear the batch to save memory\n",
        "            chunk_index += 1\n",
        "\n",
        "    # Save any remaining data\n",
        "    if batch:\n",
        "        chunk_path = f\"processed/{split_name}/chunk_{chunk_index}.pt\"\n",
        "        torch.save(batch, chunk_path)\n",
        "\n",
        "# Process and save train and validation datasets\n",
        "preprocess_and_save(train_dataset, \"train\")\n",
        "preprocess_and_save(validation_dataset, \"validation\")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TFWgNKkZM0GS",
        "outputId": "0366a841-8125-45ed-8c9b-41f746c62d61"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train chunks: ['chunk_35.pt', 'chunk_244.pt', 'chunk_132.pt', 'chunk_187.pt', 'chunk_173.pt', 'chunk_77.pt', 'chunk_163.pt', 'chunk_59.pt', 'chunk_148.pt', 'chunk_169.pt', 'chunk_44.pt', 'chunk_123.pt', 'chunk_204.pt', 'chunk_258.pt', 'chunk_23.pt', 'chunk_179.pt', 'chunk_221.pt', 'chunk_8.pt', 'chunk_3.pt', 'chunk_33.pt', 'chunk_282.pt', 'chunk_5.pt', 'chunk_227.pt', 'chunk_6.pt', 'chunk_25.pt', 'chunk_127.pt', 'chunk_256.pt', 'chunk_269.pt', 'chunk_0.pt', 'chunk_69.pt', 'chunk_255.pt', 'chunk_124.pt', 'chunk_245.pt', 'chunk_199.pt', 'chunk_195.pt', 'chunk_128.pt', 'chunk_84.pt', 'chunk_203.pt', 'chunk_215.pt', 'chunk_19.pt', 'chunk_152.pt', 'chunk_97.pt', 'chunk_78.pt', 'chunk_32.pt', 'chunk_263.pt', 'chunk_39.pt', 'chunk_250.pt', 'chunk_28.pt', 'chunk_164.pt', 'chunk_65.pt', 'chunk_36.pt', 'chunk_89.pt', 'chunk_180.pt', 'chunk_24.pt', 'chunk_12.pt', 'chunk_225.pt', 'chunk_175.pt', 'chunk_57.pt', 'chunk_40.pt', 'chunk_133.pt', 'chunk_1.pt', 'chunk_144.pt', 'chunk_117.pt', 'chunk_262.pt', 'chunk_260.pt', 'chunk_267.pt', 'chunk_222.pt', 'chunk_178.pt', 'chunk_241.pt', 'chunk_151.pt', 'chunk_52.pt', 'chunk_281.pt', 'chunk_42.pt', 'chunk_277.pt', 'chunk_219.pt', 'chunk_56.pt', 'chunk_15.pt', 'chunk_70.pt', 'chunk_120.pt', 'chunk_139.pt', 'chunk_43.pt', 'chunk_125.pt', 'chunk_76.pt', 'chunk_209.pt', 'chunk_26.pt', 'chunk_240.pt', 'chunk_83.pt', 'chunk_17.pt', 'chunk_243.pt', 'chunk_58.pt', 'chunk_157.pt', 'chunk_46.pt', 'chunk_53.pt', 'chunk_62.pt', 'chunk_186.pt', 'chunk_60.pt', 'chunk_261.pt', 'chunk_273.pt', 'chunk_212.pt', 'chunk_226.pt', 'chunk_114.pt', 'chunk_207.pt', 'chunk_194.pt', 'chunk_285.pt', 'chunk_142.pt', 'chunk_254.pt', 'chunk_94.pt', 'chunk_161.pt', 'chunk_102.pt', 'chunk_126.pt', 'chunk_283.pt', 'chunk_113.pt', 'chunk_159.pt', 'chunk_13.pt', 'chunk_29.pt', 'chunk_98.pt', 'chunk_129.pt', 'chunk_239.pt', 'chunk_265.pt', 'chunk_201.pt', 'chunk_88.pt', 'chunk_181.pt', 'chunk_183.pt', 'chunk_22.pt', 'chunk_165.pt', 'chunk_196.pt', 'chunk_122.pt', 'chunk_109.pt', 'chunk_166.pt', 'chunk_156.pt', 'chunk_185.pt', 'chunk_238.pt', 'chunk_51.pt', 'chunk_10.pt', 'chunk_249.pt', 'chunk_150.pt', 'chunk_92.pt', 'chunk_266.pt', 'chunk_275.pt', 'chunk_224.pt', 'chunk_279.pt', 'chunk_48.pt', 'chunk_188.pt', 'chunk_233.pt', 'chunk_105.pt', 'chunk_232.pt', 'chunk_223.pt', 'chunk_103.pt', 'chunk_168.pt', 'chunk_284.pt', 'chunk_214.pt', 'chunk_86.pt', 'chunk_34.pt', 'chunk_141.pt', 'chunk_108.pt', 'chunk_153.pt', 'chunk_274.pt', 'chunk_91.pt', 'chunk_27.pt', 'chunk_63.pt', 'chunk_74.pt', 'chunk_174.pt', 'chunk_82.pt', 'chunk_184.pt', 'chunk_119.pt', 'chunk_176.pt', 'chunk_205.pt', 'chunk_229.pt', 'chunk_7.pt', 'chunk_111.pt', 'chunk_182.pt', 'chunk_95.pt', 'chunk_73.pt', 'chunk_158.pt', 'chunk_162.pt', 'chunk_50.pt', 'chunk_41.pt', 'chunk_270.pt', 'chunk_4.pt', 'chunk_11.pt', 'chunk_198.pt', 'chunk_30.pt', 'chunk_218.pt', 'chunk_9.pt', 'chunk_235.pt', 'chunk_210.pt', 'chunk_242.pt', 'chunk_191.pt', 'chunk_202.pt', 'chunk_177.pt', 'chunk_64.pt', 'chunk_220.pt', 'chunk_96.pt', 'chunk_75.pt', 'chunk_140.pt', 'chunk_147.pt', 'chunk_248.pt', 'chunk_37.pt', 'chunk_68.pt', 'chunk_192.pt', 'chunk_115.pt', 'chunk_67.pt', 'chunk_211.pt', 'chunk_136.pt', 'chunk_280.pt', 'chunk_87.pt', 'chunk_160.pt', 'chunk_112.pt', 'chunk_85.pt', 'chunk_208.pt', 'chunk_286.pt', 'chunk_20.pt', 'chunk_93.pt', 'chunk_31.pt', 'chunk_106.pt', 'chunk_252.pt', 'chunk_137.pt', 'chunk_154.pt', 'chunk_54.pt', 'chunk_167.pt', 'chunk_231.pt', 'chunk_107.pt', 'chunk_272.pt', 'chunk_47.pt', 'chunk_146.pt', 'chunk_268.pt', 'chunk_100.pt', 'chunk_234.pt', 'chunk_228.pt', 'chunk_278.pt', 'chunk_138.pt', 'chunk_287.pt', 'chunk_21.pt', 'chunk_170.pt', 'chunk_216.pt', 'chunk_197.pt', 'chunk_45.pt', 'chunk_14.pt', 'chunk_257.pt', 'chunk_135.pt', 'chunk_276.pt', 'chunk_71.pt', 'chunk_145.pt', 'chunk_101.pt', 'chunk_237.pt', 'chunk_217.pt', 'chunk_193.pt', 'chunk_288.pt', 'chunk_90.pt', 'chunk_190.pt', 'chunk_72.pt', 'chunk_143.pt', 'chunk_131.pt', 'chunk_246.pt', 'chunk_121.pt', 'chunk_134.pt', 'chunk_264.pt', 'chunk_200.pt', 'chunk_206.pt', 'chunk_118.pt', 'chunk_99.pt', 'chunk_61.pt', 'chunk_236.pt', 'chunk_189.pt', 'chunk_259.pt', 'chunk_16.pt', 'chunk_2.pt', 'chunk_80.pt', 'chunk_155.pt', 'chunk_79.pt', 'chunk_149.pt', 'chunk_66.pt', 'chunk_49.pt', 'chunk_213.pt', 'chunk_171.pt', 'chunk_230.pt', 'chunk_110.pt', 'chunk_81.pt', 'chunk_38.pt', 'chunk_104.pt', 'chunk_172.pt', 'chunk_271.pt', 'chunk_130.pt', 'chunk_253.pt', 'chunk_18.pt', 'chunk_55.pt', 'chunk_247.pt', 'chunk_116.pt', 'chunk_251.pt']\n",
            "Validation chunks: ['chunk_35.pt', 'chunk_77.pt', 'chunk_59.pt', 'chunk_44.pt', 'chunk_23.pt', 'chunk_8.pt', 'chunk_3.pt', 'chunk_33.pt', 'chunk_5.pt', 'chunk_6.pt', 'chunk_25.pt', 'chunk_0.pt', 'chunk_69.pt', 'chunk_84.pt', 'chunk_19.pt', 'chunk_97.pt', 'chunk_78.pt', 'chunk_32.pt', 'chunk_39.pt', 'chunk_28.pt', 'chunk_65.pt', 'chunk_36.pt', 'chunk_89.pt', 'chunk_24.pt', 'chunk_12.pt', 'chunk_57.pt', 'chunk_40.pt', 'chunk_1.pt', 'chunk_52.pt', 'chunk_42.pt', 'chunk_56.pt', 'chunk_15.pt', 'chunk_70.pt', 'chunk_43.pt', 'chunk_76.pt', 'chunk_26.pt', 'chunk_83.pt', 'chunk_17.pt', 'chunk_58.pt', 'chunk_46.pt', 'chunk_53.pt', 'chunk_62.pt', 'chunk_60.pt', 'chunk_94.pt', 'chunk_102.pt', 'chunk_13.pt', 'chunk_29.pt', 'chunk_98.pt', 'chunk_88.pt', 'chunk_22.pt', 'chunk_51.pt', 'chunk_10.pt', 'chunk_92.pt', 'chunk_48.pt', 'chunk_105.pt', 'chunk_103.pt', 'chunk_86.pt', 'chunk_34.pt', 'chunk_91.pt', 'chunk_27.pt', 'chunk_63.pt', 'chunk_74.pt', 'chunk_82.pt', 'chunk_7.pt', 'chunk_95.pt', 'chunk_73.pt', 'chunk_50.pt', 'chunk_41.pt', 'chunk_4.pt', 'chunk_11.pt', 'chunk_30.pt', 'chunk_9.pt', 'chunk_64.pt', 'chunk_96.pt', 'chunk_75.pt', 'chunk_37.pt', 'chunk_68.pt', 'chunk_67.pt', 'chunk_87.pt', 'chunk_85.pt', 'chunk_20.pt', 'chunk_93.pt', 'chunk_31.pt', 'chunk_54.pt', 'chunk_47.pt', 'chunk_100.pt', 'chunk_21.pt', 'chunk_45.pt', 'chunk_14.pt', 'chunk_71.pt', 'chunk_101.pt', 'chunk_90.pt', 'chunk_72.pt', 'chunk_99.pt', 'chunk_61.pt', 'chunk_16.pt', 'chunk_2.pt', 'chunk_80.pt', 'chunk_79.pt', 'chunk_66.pt', 'chunk_49.pt', 'chunk_81.pt', 'chunk_38.pt', 'chunk_104.pt', 'chunk_18.pt', 'chunk_55.pt']\n"
          ]
        }
      ],
      "source": [
        "import os\n",
        "\n",
        "train_chunks = os.listdir(\"processed/train\")\n",
        "validation_chunks = os.listdir(\"processed/validation\")\n",
        "print(f\"Train chunks: {train_chunks}\")\n",
        "print(f\"Validation chunks: {validation_chunks}\")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eRWCWOPoNuam"
      },
      "outputs": [],
      "source": [
        "def load_chunk(split_name, chunk_index):\n",
        "    import torch\n",
        "\n",
        "    chunk_path = f\"processed/{split_name}/chunk_{chunk_index}.pt\"\n",
        "    return torch.load(chunk_path)  # Returns a list of dictionaries with 'audio' and 'transcript'\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cgTEhrrSNywh",
        "outputId": "de581ee0-c641-4b7a-cd38-893a078428c2"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "<ipython-input-8-7ef96ad826c7>:5: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
            "  return torch.load(chunk_path)  # Returns a list of dictionaries with 'audio' and 'transcript'\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded 100 examples from the first training chunk.\n"
          ]
        }
      ],
      "source": [
        "# Load the first chunk from the training set\n",
        "train_chunk_0 = load_chunk(\"train\", 0)\n",
        "print(f\"Loaded {len(train_chunk_0)} examples from the first training chunk.\")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "naPNoTHlOFEE",
        "outputId": "a21dc820-6b00-4b2e-8e30-2ed775dbd66f"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "<ipython-input-8-7ef96ad826c7>:5: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
            "  return torch.load(chunk_path)  # Returns a list of dictionaries with 'audio' and 'transcript'\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[{'audio': array([0., 0., 0., ..., 0., 0., 0.]), 'transcript': 'او مثل آقا رفتار کرد.'}, {'audio': array([ 0.00000000e+00,  1.32521056e-13, -7.95415166e-14, ...,\n",
            "        3.36417179e-05,  2.67675605e-05, -8.01658098e-06]), 'transcript': 'زیبایی او را نمیتوانم وصف کنم.'}, {'audio': array([ 0.00000000e+00, -5.71812630e-15, -3.87703074e-14, ...,\n",
            "        5.19313403e-07,  3.44763549e-07,  1.15781017e-06]), 'transcript': 'بدون قید و شرط'}, {'audio': array([ 0.00000000e+00,  1.23193534e-12,  2.48242160e-12, ...,\n",
            "       -1.42956735e-04, -1.15126917e-04, -1.31362525e-04]), 'transcript': 'این وظیفهی تو است، سعی نکن آن را به من محول کنی!'}]\n"
          ]
        }
      ],
      "source": [
        "def stream_batches(split_name, total_chunks, batch_size=4):\n",
        "    \"\"\"\n",
        "    Generator that streams batches from preprocessed chunks.\n",
        "    \"\"\"\n",
        "    for chunk_index in range(total_chunks):\n",
        "        chunk = load_chunk(split_name, chunk_index)\n",
        "        for i in range(0, len(chunk), batch_size):\n",
        "            batch = chunk[i:i+batch_size]\n",
        "            yield batch\n",
        "\n",
        "# Example: Stream batches from the train set\n",
        "train_batches = stream_batches(\"train\", total_chunks=len(train_chunks), batch_size=4)\n",
        "for batch in train_batches:\n",
        "    print(batch)\n",
        "    break  # Print one batch\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "49lG6exJORdN"
      },
      "outputs": [],
      "source": [
        "from torch.utils.data import Dataset\n",
        "\n",
        "class ChunkedDataset(Dataset):\n",
        "    def __init__(self, split_name, total_chunks):\n",
        "        self.split_name = split_name\n",
        "        self.total_chunks = total_chunks\n",
        "        self.current_chunk = None\n",
        "        self.chunk_index = -1\n",
        "\n",
        "    def load_next_chunk(self):\n",
        "        self.chunk_index += 1\n",
        "        if self.chunk_index < self.total_chunks:\n",
        "            self.current_chunk = load_chunk(self.split_name, self.chunk_index)\n",
        "        else:\n",
        "            self.current_chunk = None\n",
        "\n",
        "    def __len__(self):\n",
        "        return self.total_chunks * len(self.current_chunk)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        chunk_idx = idx // len(self.current_chunk)\n",
        "        within_chunk_idx = idx % len(self.current_chunk)\n",
        "        if chunk_idx != self.chunk_index:\n",
        "            self.chunk_index = chunk_idx\n",
        "            self.current_chunk = load_chunk(self.split_name, self.chunk_index)\n",
        "        return self.current_chunk[within_chunk_idx]\n",
        "\n",
        "# Example: Create train dataset\n",
        "train_dataset = ChunkedDataset(\"train\", total_chunks=len(train_chunks))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "S8ygKG3k99mR"
      },
      "outputs": [],
      "source": [
        "from transformers import WhisperForConditionalGeneration, WhisperProcessor\n",
        "\n",
        "# Load the Whisper model\n",
        "model_name = \"openai/whisper-large-v3\"\n",
        "model = WhisperForConditionalGeneration.from_pretrained(model_name)\n",
        "\n",
        "# Load the processor (tokenizer + feature extractor)\n",
        "processor = WhisperProcessor.from_pretrained(model_name)\n",
        "tokenizer = processor.tokenizer\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZAsvf-Z2-apw",
        "outputId": "512a8b20-6f4b-444a-ef6f-3e0b9fd392ec"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
            "  warnings.warn(\n"
          ]
        }
      ],
      "source": [
        "from transformers import Seq2SeqTrainingArguments\n",
        "\n",
        "training_args = Seq2SeqTrainingArguments(\n",
        "    output_dir=\"./whisper_fine_tuned\",    # Directory to save model checkpoints\n",
        "    per_device_train_batch_size=4,       # Adjust batch size to fit in memory\n",
        "    per_device_eval_batch_size=4,\n",
        "    learning_rate=5e-5,                  # Learning rate\n",
        "    num_train_epochs=3,                  # Number of training epochs\n",
        "    evaluation_strategy=\"epoch\",         # Evaluate at the end of each epoch\n",
        "    save_strategy=\"epoch\",               # Save model checkpoints\n",
        "    logging_steps=1,\n",
        "    max_steps=1000,\n",
        "    logging_dir=\"./logs\",                # Directory for logs\n",
        "    predict_with_generate=True,          # Generate text for evaluation\n",
        "    fp16=True,                           # Enable mixed precision training (if supported)\n",
        "    save_total_limit=2,                  # Limit the number of checkpoints\n",
        ")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "id": "8eKuEabR-crC"
      },
      "outputs": [],
      "source": [
        "from transformers import DataCollatorForSeq2Seq\n",
        "\n",
        "data_collator = DataCollatorForSeq2Seq(processor.tokenizer, model=model)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "id": "IhJv0JwdDh-F"
      },
      "outputs": [],
      "source": [
        "from torch.utils.data import Dataset\n",
        "import torch\n",
        "\n",
        "class CustomDataset(Dataset):\n",
        "    def __init__(self, chunks_path, total_chunks):\n",
        "        self.chunks_path = chunks_path\n",
        "        self.total_chunks = total_chunks\n",
        "\n",
        "    def __len__(self):\n",
        "        return sum(len(torch.load(f\"{self.chunks_path}/chunk_{i}.pt\")) for i in range(self.total_chunks))\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        chunk_idx = idx // len(torch.load(f\"{self.chunks_path}/chunk_0.pt\"))  # Determine which chunk\n",
        "        local_idx = idx % len(torch.load(f\"{self.chunks_path}/chunk_0.pt\"))  # Index within the chunk\n",
        "        chunk = torch.load(f\"{self.chunks_path}/chunk_{chunk_idx}.pt\")\n",
        "        return chunk[local_idx]\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "id": "3XWBe1mwDZAY"
      },
      "outputs": [],
      "source": [
        "train_dataset = CustomDataset(\"processed/train\", total_chunks=len(train_chunks))\n",
        "validation_dataset = CustomDataset(\"processed/validation\", total_chunks=len(validation_chunks))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JAPDEkWC-gwX",
        "outputId": "c9d281b3-27f8-40db-9e59-ff4bbc7c2e35"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "<ipython-input-24-08c8795bec9e>:3: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Seq2SeqTrainer.__init__`. Use `processing_class` instead.\n",
            "  trainer = Seq2SeqTrainer(\n",
            "<ipython-input-22-83299940537d>:10: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
            "  return sum(len(torch.load(f\"{self.chunks_path}/chunk_{i}.pt\")) for i in range(self.total_chunks))\n"
          ]
        }
      ],
      "source": [
        "from transformers import Seq2SeqTrainer\n",
        "\n",
        "trainer = Seq2SeqTrainer(\n",
        "    model=model,\n",
        "    args=training_args,\n",
        "    train_dataset=train_dataset,\n",
        "    eval_dataset=ChunkedDataset(\"validation\", total_chunks=len(validation_chunks)),\n",
        "    tokenizer=tokenizer,\n",
        "    data_collator=data_collator,\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FhfkGb6qgr-j",
        "outputId": "9b35a06a-d6db-492a-d642-ef8ede893a1e"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "<ipython-input-22-83299940537d>:10: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
            "  return sum(len(torch.load(f\"{self.chunks_path}/chunk_{i}.pt\")) for i in range(self.total_chunks))\n"
          ]
        }
      ],
      "source": [
        "from transformers import TrainerCallback\n",
        "import time\n",
        "\n",
        "class TimeEstimatorCallback(TrainerCallback):\n",
        "    def __init__(self, total_steps):\n",
        "        self.total_steps = total_steps\n",
        "        self.start_time = time.time()\n",
        "\n",
        "    def on_log(self, args, state, control, **kwargs):\n",
        "        elapsed_time = time.time() - self.start_time\n",
        "        steps_done = state.global_step\n",
        "        if steps_done > 0:\n",
        "            remaining_steps = self.total_steps - steps_done\n",
        "            time_per_step = elapsed_time / steps_done\n",
        "            remaining_time = remaining_steps * time_per_step\n",
        "            print(f\"Time remaining: {remaining_time / 60:.2f} minutes\")\n",
        "\n",
        "# Add total steps calculation\n",
        "train_steps_per_epoch = len(train_dataset) // training_args.per_device_train_batch_size\n",
        "total_train_steps = train_steps_per_epoch * training_args.num_train_epochs\n",
        "\n",
        "# Attach callback\n",
        "trainer.add_callback(TimeEstimatorCallback(total_train_steps))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MfHDWpEV-myE",
        "outputId": "b85cea37-73c0-402f-ae68-c8a9ced55af5"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "<ipython-input-22-83299940537d>:10: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
            "  return sum(len(torch.load(f\"{self.chunks_path}/chunk_{i}.pt\")) for i in range(self.total_chunks))\n"
          ]
        }
      ],
      "source": [
        "trainer.train()"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}