train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f424a491-9864-427c-a438-a4f55971cba1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wed Aug 21 14:32:19 2024       \n",
      "+-----------------------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 555.42.06              Driver Version: 555.42.06      CUDA Version: 12.5     |\n",
      "|-----------------------------------------+------------------------+----------------------+\n",
      "| GPU  Name                 Persistence-M | Bus-Id          Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp   Perf          Pwr:Usage/Cap |           Memory-Usage | GPU-Util  Compute M. |\n",
      "|                                         |                        |               MIG M. |\n",
      "|=========================================+========================+======================|\n",
      "|   0  NVIDIA H100 80GB HBM3          On  |   00000000:BF:00.0 Off |                    0 |\n",
      "| N/A   25C    P0             68W /  700W |       1MiB /  81559MiB |      0%      Default |\n",
      "|                                         |                        |             Disabled |\n",
      "+-----------------------------------------+------------------------+----------------------+\n",
      "                                                                                         \n",
      "+-----------------------------------------------------------------------------------------+\n",
      "| Processes:                                                                              |\n",
      "|  GPU   GI   CI        PID   Type   Process name                              GPU Memory |\n",
      "|        ID   ID                                                               Usage      |\n",
      "|=========================================================================================|\n",
      "|  No running processes found                                                             |\n",
      "+-----------------------------------------------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "gpu_info = !nvidia-smi\n",
    "gpu_info = '\\n'.join(gpu_info)\n",
    "if gpu_info.find('failed') >= 0:\n",
    "  print('Not connected to a GPU')\n",
    "else:\n",
    "  print(gpu_info)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3e74d77e-fad7-4fa9-9836-19da0c64d6d2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'llama_fim_finetuning'...\n",
      "remote: Enumerating objects: 30, done.\u001b[K\n",
      "remote: Counting objects: 100% (30/30), done.\u001b[K\n",
      "remote: Compressing objects: 100% (21/21), done.\u001b[K\n",
      "remote: Total 30 (delta 9), reused 28 (delta 8), pack-reused 0 (from 0)\u001b[K\n",
      "Receiving objects: 100% (30/30), 14.90 KiB | 181.00 KiB/s, done.\n",
      "Resolving deltas: 100% (9/9), done.\n"
     ]
    }
   ],
   "source": [
    "!git clone https://github.com/ardalaan/llama_fim_finetuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "94a5062c-ee61-425b-9e63-43b607a6518e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (23.2)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython -m pip install --upgrade pip\u001b[0m\n",
      "\u001b[33mWARNING: Skipping ninja as it is not installed.\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0mCollecting ninja\n",
      "  Downloading ninja-1.11.1.1-py2.py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl.metadata (5.3 kB)\n",
      "Downloading ninja-1.11.1.1-py2.py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl (307 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m307.2/307.2 kB\u001b[0m \u001b[31m3.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: ninja\n",
      "Successfully installed ninja-1.11.1.1\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython -m pip install --upgrade pip\u001b[0m\n",
      "1.11.1.git.kitware.jobserver-1\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "!pip install packaging\n",
    "!pip uninstall -y ninja && pip install ninja\n",
    "!ninja --version\n",
    "!echo $?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b64aa427-3c85-4a3f-a9c3-babef38dae0a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git\n",
      "  Cloning https://github.com/unslothai/unsloth.git to /tmp/pip-install-0ph6h9rz/unsloth_f317fb7e490d41499e67345dad799c7a\n",
      "  Running command git clone --filter=blob:none --quiet https://github.com/unslothai/unsloth.git /tmp/pip-install-0ph6h9rz/unsloth_f317fb7e490d41499e67345dad799c7a\n",
      "  Resolved https://github.com/unslothai/unsloth.git to commit 600ffe2a175312e7f2fbcdffbe6b99e3df20c417\n",
      "  Installing build dependencies ... \u001b[?25ldone\n",
      "\u001b[?25h  Getting requirements to build wheel ... \u001b[?25ldone\n",
      "\u001b[?25h  Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n",
      "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (23.2)\n",
      "Collecting tyro (from unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading tyro-0.8.8-py3-none-any.whl.metadata (8.4 kB)\n",
      "Collecting transformers>=4.43.2 (from unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading transformers-4.44.1-py3-none-any.whl.metadata (43 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.7/43.7 kB\u001b[0m \u001b[31m1.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hCollecting datasets>=2.16.0 (from unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading datasets-2.21.0-py3-none-any.whl.metadata (21 kB)\n",
      "Collecting sentencepiece>=0.2.0 (from unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading sentencepiece-0.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (7.7 kB)\n",
      "Collecting tqdm (from unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading tqdm-4.66.5-py3-none-any.whl.metadata (57 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.6/57.6 kB\u001b[0m \u001b[31m5.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (5.9.6)\n",
      "Collecting wheel>=0.42.0 (from unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading wheel-0.44.0-py3-none-any.whl.metadata (2.3 kB)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (1.24.1)\n",
      "Collecting protobuf<4.0.0 (from unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading protobuf-3.20.3-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl.metadata (679 bytes)\n",
      "Collecting huggingface-hub (from unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading huggingface_hub-0.24.6-py3-none-any.whl.metadata (13 kB)\n",
      "Collecting hf-transfer (from unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading hf_transfer-0.1.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (1.7 kB)\n",
      "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from datasets>=2.16.0->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (3.9.0)\n",
      "Collecting pyarrow>=15.0.0 (from datasets>=2.16.0->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading pyarrow-17.0.0-cp310-cp310-manylinux_2_28_x86_64.whl.metadata (3.3 kB)\n",
      "Collecting dill<0.3.9,>=0.3.0 (from datasets>=2.16.0->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading dill-0.3.8-py3-none-any.whl.metadata (10 kB)\n",
      "Collecting pandas (from datasets>=2.16.0->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading pandas-2.2.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (19 kB)\n",
      "Collecting requests>=2.32.2 (from datasets>=2.16.0->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading requests-2.32.3-py3-none-any.whl.metadata (4.6 kB)\n",
      "Collecting xxhash (from datasets>=2.16.0->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading xxhash-3.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (12 kB)\n",
      "Collecting multiprocess (from datasets>=2.16.0->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading multiprocess-0.70.16-py310-none-any.whl.metadata (7.2 kB)\n",
      "Requirement already satisfied: fsspec<=2024.6.1,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from fsspec[http]<=2024.6.1,>=2023.1.0->datasets>=2.16.0->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (2023.4.0)\n",
      "Collecting aiohttp (from datasets>=2.16.0->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading aiohttp-3.10.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (7.5 kB)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.16.0->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (6.0.1)\n",
      "Collecting fsspec<=2024.6.1,>=2023.1.0 (from fsspec[http]<=2024.6.1,>=2023.1.0->datasets>=2.16.0->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading fsspec-2024.6.1-py3-none-any.whl.metadata (11 kB)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (4.4.0)\n",
      "Collecting regex!=2019.12.17 (from transformers>=4.43.2->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading regex-2024.7.24-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (40 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m40.5/40.5 kB\u001b[0m \u001b[31m11.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting safetensors>=0.4.1 (from transformers>=4.43.2->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading safetensors-0.4.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.8 kB)\n",
      "Collecting tokenizers<0.20,>=0.19 (from transformers>=4.43.2->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading tokenizers-0.19.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.7 kB)\n",
      "Collecting docstring-parser>=0.16 (from tyro->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading docstring_parser-0.16-py3-none-any.whl.metadata (3.0 kB)\n",
      "Collecting typing-extensions>=3.7.4.3 (from huggingface-hub->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading typing_extensions-4.12.2-py3-none-any.whl.metadata (3.0 kB)\n",
      "Collecting rich>=11.1.0 (from tyro->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading rich-13.7.1-py3-none-any.whl.metadata (18 kB)\n",
      "Collecting shtab>=1.5.6 (from tyro->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading shtab-1.7.1-py3-none-any.whl.metadata (7.3 kB)\n",
      "INFO: pip is looking at multiple versions of fsspec[http] to determine which version is compatible with other requirements. This could take a while.\n",
      "Collecting aiohappyeyeballs>=2.3.0 (from aiohttp->datasets>=2.16.0->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading aiohappyeyeballs-2.4.0-py3-none-any.whl.metadata (5.9 kB)\n",
      "Collecting aiosignal>=1.1.2 (from aiohttp->datasets>=2.16.0->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading aiosignal-1.3.1-py3-none-any.whl.metadata (4.0 kB)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.16.0->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (23.1.0)\n",
      "Collecting frozenlist>=1.1.1 (from aiohttp->datasets>=2.16.0->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading frozenlist-1.4.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (12 kB)\n",
      "Collecting multidict<7.0,>=4.5 (from aiohttp->datasets>=2.16.0->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading multidict-6.0.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.2 kB)\n",
      "Collecting yarl<2.0,>=1.0 (from aiohttp->datasets>=2.16.0->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading yarl-1.9.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (31 kB)\n",
      "Collecting async-timeout<5.0,>=4.0 (from aiohttp->datasets>=2.16.0->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading async_timeout-4.0.3-py3-none-any.whl.metadata (4.2 kB)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.2->datasets>=2.16.0->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (2.1.1)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.2->datasets>=2.16.0->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.2->datasets>=2.16.0->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (1.26.13)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.2->datasets>=2.16.0->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (2022.12.7)\n",
      "Collecting markdown-it-py>=2.2.0 (from rich>=11.1.0->tyro->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading markdown_it_py-3.0.0-py3-none-any.whl.metadata (6.9 kB)\n",
      "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich>=11.1.0->tyro->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (2.16.1)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets>=2.16.0->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (2.8.2)\n",
      "Collecting pytz>=2020.1 (from pandas->datasets>=2.16.0->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading pytz-2024.1-py2.py3-none-any.whl.metadata (22 kB)\n",
      "Collecting tzdata>=2022.7 (from pandas->datasets>=2.16.0->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading tzdata-2024.1-py2.py3-none-any.whl.metadata (1.4 kB)\n",
      "Collecting mdurl~=0.1 (from markdown-it-py>=2.2.0->rich>=11.1.0->tyro->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git)\n",
      "  Downloading mdurl-0.1.2-py3-none-any.whl.metadata (1.6 kB)\n",
      "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.8.2->pandas->datasets>=2.16.0->unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git) (1.16.0)\n",
      "Downloading datasets-2.21.0-py3-none-any.whl (527 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m527.3/527.3 kB\u001b[0m \u001b[31m8.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hDownloading huggingface_hub-0.24.6-py3-none-any.whl (417 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m417.5/417.5 kB\u001b[0m \u001b[31m55.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading protobuf-3.20.3-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.1 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m41.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading sentencepiece-0.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m83.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading tqdm-4.66.5-py3-none-any.whl (78 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m78.4/78.4 kB\u001b[0m \u001b[31m17.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading transformers-4.44.1-py3-none-any.whl (9.5 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m9.5/9.5 MB\u001b[0m \u001b[31m46.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hDownloading wheel-0.44.0-py3-none-any.whl (67 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.1/67.1 kB\u001b[0m \u001b[31m40.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading hf_transfer-0.1.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.6 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.6/3.6 MB\u001b[0m \u001b[31m45.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hDownloading tyro-0.8.8-py3-none-any.whl (104 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m104.6/104.6 kB\u001b[0m \u001b[31m59.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading dill-0.3.8-py3-none-any.whl (116 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m69.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading docstring_parser-0.16-py3-none-any.whl (36 kB)\n",
      "Downloading fsspec-2024.6.1-py3-none-any.whl (177 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m177.6/177.6 kB\u001b[0m \u001b[31m91.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading aiohttp-3.10.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m84.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading pyarrow-17.0.0-cp310-cp310-manylinux_2_28_x86_64.whl (39.9 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m39.9/39.9 MB\u001b[0m \u001b[31m44.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hDownloading regex-2024.7.24-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (776 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m776.5/776.5 kB\u001b[0m \u001b[31m126.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading requests-2.32.3-py3-none-any.whl (64 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m64.9/64.9 kB\u001b[0m \u001b[31m35.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading rich-13.7.1-py3-none-any.whl (240 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m240.7/240.7 kB\u001b[0m \u001b[31m121.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading safetensors-0.4.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (435 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m435.5/435.5 kB\u001b[0m \u001b[31m65.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading shtab-1.7.1-py3-none-any.whl (14 kB)\n",
      "Downloading tokenizers-0.19.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.6 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.6/3.6 MB\u001b[0m \u001b[31m56.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hDownloading typing_extensions-4.12.2-py3-none-any.whl (37 kB)\n",
      "Downloading multiprocess-0.70.16-py310-none-any.whl (134 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m27.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading pandas-2.2.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (13.0 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m13.0/13.0 MB\u001b[0m \u001b[31m54.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hDownloading xxhash-3.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m42.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading aiohappyeyeballs-2.4.0-py3-none-any.whl (12 kB)\n",
      "Downloading aiosignal-1.3.1-py3-none-any.whl (7.6 kB)\n",
      "Downloading async_timeout-4.0.3-py3-none-any.whl (5.7 kB)\n",
      "Downloading frozenlist-1.4.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (239 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m239.5/239.5 kB\u001b[0m \u001b[31m43.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading markdown_it_py-3.0.0-py3-none-any.whl (87 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m87.5/87.5 kB\u001b[0m \u001b[31m19.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading multidict-6.0.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (124 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m124.3/124.3 kB\u001b[0m \u001b[31m27.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading pytz-2024.1-py2.py3-none-any.whl (505 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m505.5/505.5 kB\u001b[0m \u001b[31m63.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading tzdata-2024.1-py2.py3-none-any.whl (345 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m345.4/345.4 kB\u001b[0m \u001b[31m53.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading yarl-1.9.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (301 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m301.6/301.6 kB\u001b[0m \u001b[31m49.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading mdurl-0.1.2-py3-none-any.whl (10.0 kB)\n",
      "Building wheels for collected packages: unsloth\n",
      "  Building wheel for unsloth (pyproject.toml) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for unsloth: filename=unsloth-2024.8-py3-none-any.whl size=145665 sha256=1d42cfad1a93587c5641e560257e23e29815c4f134313e7984b15bd521cbf093\n",
      "  Stored in directory: /tmp/pip-ephem-wheel-cache-lkbwc1c2/wheels/ed/d4/e9/76fb290ee3df0a5fc21ce5c2c788e29e9607a2353d8342fd0d\n",
      "Successfully built unsloth\n",
      "Installing collected packages: sentencepiece, pytz, xxhash, wheel, unsloth, tzdata, typing-extensions, tqdm, shtab, safetensors, requests, regex, pyarrow, protobuf, multidict, mdurl, hf-transfer, fsspec, frozenlist, docstring-parser, dill, async-timeout, aiohappyeyeballs, yarl, pandas, multiprocess, markdown-it-py, huggingface-hub, aiosignal, tokenizers, rich, aiohttp, tyro, transformers, datasets\n",
      "  Attempting uninstall: wheel\n",
      "    Found existing installation: wheel 0.41.3\n",
      "    Uninstalling wheel-0.41.3:\n",
      "      Successfully uninstalled wheel-0.41.3\n",
      "  Attempting uninstall: typing-extensions\n",
      "    Found existing installation: typing_extensions 4.4.0\n",
      "    Uninstalling typing_extensions-4.4.0:\n",
      "      Successfully uninstalled typing_extensions-4.4.0\n",
      "  Attempting uninstall: requests\n",
      "    Found existing installation: requests 2.31.0\n",
      "    Uninstalling requests-2.31.0:\n",
      "      Successfully uninstalled requests-2.31.0\n",
      "  Attempting uninstall: fsspec\n",
      "    Found existing installation: fsspec 2023.4.0\n",
      "    Uninstalling fsspec-2023.4.0:\n",
      "      Successfully uninstalled fsspec-2023.4.0\n",
      "Successfully installed aiohappyeyeballs-2.4.0 aiohttp-3.10.5 aiosignal-1.3.1 async-timeout-4.0.3 datasets-2.21.0 dill-0.3.8 docstring-parser-0.16 frozenlist-1.4.1 fsspec-2024.6.1 hf-transfer-0.1.8 huggingface-hub-0.24.6 markdown-it-py-3.0.0 mdurl-0.1.2 multidict-6.0.5 multiprocess-0.70.16 pandas-2.2.2 protobuf-3.20.3 pyarrow-17.0.0 pytz-2024.1 regex-2024.7.24 requests-2.32.3 rich-13.7.1 safetensors-0.4.4 sentencepiece-0.2.0 shtab-1.7.1 tokenizers-0.19.1 tqdm-4.66.5 transformers-4.44.1 typing-extensions-4.12.2 tyro-0.8.8 tzdata-2024.1 unsloth-2024.8 wheel-0.44.0 xxhash-3.5.0 yarl-1.9.4\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython -m pip install --upgrade pip\u001b[0m\n",
      "Collecting xformers<0.0.27\n",
      "  Downloading xformers-0.0.26.post1-cp310-cp310-manylinux2014_x86_64.whl.metadata (1.0 kB)\n",
      "Collecting trl<0.9.0\n",
      "  Downloading trl-0.8.6-py3-none-any.whl.metadata (11 kB)\n",
      "Collecting peft\n",
      "  Downloading peft-0.12.0-py3-none-any.whl.metadata (13 kB)\n",
      "Collecting accelerate\n",
      "  Downloading accelerate-0.33.0-py3-none-any.whl.metadata (18 kB)\n",
      "Collecting bitsandbytes\n",
      "  Downloading bitsandbytes-0.43.3-py3-none-manylinux_2_24_x86_64.whl.metadata (3.5 kB)\n",
      "Downloading xformers-0.0.26.post1-cp310-cp310-manylinux2014_x86_64.whl (222.7 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m222.7/222.7 MB\u001b[0m \u001b[31m27.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hDownloading trl-0.8.6-py3-none-any.whl (245 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m245.2/245.2 kB\u001b[0m \u001b[31m104.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading peft-0.12.0-py3-none-any.whl (296 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m296.4/296.4 kB\u001b[0m \u001b[31m109.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading accelerate-0.33.0-py3-none-any.whl (315 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m315.1/315.1 kB\u001b[0m \u001b[31m110.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading bitsandbytes-0.43.3-py3-none-manylinux_2_24_x86_64.whl (137.5 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m137.5/137.5 MB\u001b[0m \u001b[31m39.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: bitsandbytes, xformers, trl, peft, accelerate\n",
      "Successfully installed accelerate-0.33.0 bitsandbytes-0.43.3 peft-0.12.0 trl-0.8.6 xformers-0.0.26.post1\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython -m pip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install \"unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git\"\n",
    "!pip install --no-deps \"xformers<0.0.27\" \"trl<0.9.0\" peft accelerate bitsandbytes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a0b17ac7-0790-4bc8-95d4-97b2d42b6faa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting wandb\n",
      "  Downloading wandb-0.17.7-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (10 kB)\n",
      "Collecting click!=8.0.0,>=7.1 (from wandb)\n",
      "  Downloading click-8.1.7-py3-none-any.whl.metadata (3.0 kB)\n",
      "Collecting docker-pycreds>=0.4.0 (from wandb)\n",
      "  Downloading docker_pycreds-0.4.0-py2.py3-none-any.whl.metadata (1.8 kB)\n",
      "Collecting gitpython!=3.1.29,>=1.0.0 (from wandb)\n",
      "  Downloading GitPython-3.1.43-py3-none-any.whl.metadata (13 kB)\n",
      "Requirement already satisfied: platformdirs in /usr/local/lib/python3.10/dist-packages (from wandb) (3.11.0)\n",
      "Requirement already satisfied: protobuf!=4.21.0,<6,>=3.19.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (3.20.3)\n",
      "Requirement already satisfied: psutil>=5.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (5.9.6)\n",
      "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from wandb) (6.0.1)\n",
      "Requirement already satisfied: requests<3,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb) (2.32.3)\n",
      "Collecting sentry-sdk>=1.0.0 (from wandb)\n",
      "  Downloading sentry_sdk-2.13.0-py2.py3-none-any.whl.metadata (9.7 kB)\n",
      "Collecting setproctitle (from wandb)\n",
      "  Downloading setproctitle-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (9.9 kB)\n",
      "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from wandb) (68.2.2)\n",
      "Requirement already satisfied: six>=1.4.0 in /usr/lib/python3/dist-packages (from docker-pycreds>=0.4.0->wandb) (1.16.0)\n",
      "Collecting gitdb<5,>=4.0.1 (from gitpython!=3.1.29,>=1.0.0->wandb)\n",
      "  Downloading gitdb-4.0.11-py3-none-any.whl.metadata (1.2 kB)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.0.0->wandb) (2.1.1)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.0.0->wandb) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.0.0->wandb) (1.26.13)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.0.0->wandb) (2022.12.7)\n",
      "Collecting smmap<6,>=3.0.1 (from gitdb<5,>=4.0.1->gitpython!=3.1.29,>=1.0.0->wandb)\n",
      "  Downloading smmap-5.0.1-py3-none-any.whl.metadata (4.3 kB)\n",
      "Downloading wandb-0.17.7-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (9.3 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m9.3/9.3 MB\u001b[0m \u001b[31m38.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hDownloading click-8.1.7-py3-none-any.whl (97 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m97.9/97.9 kB\u001b[0m \u001b[31m55.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading docker_pycreds-0.4.0-py2.py3-none-any.whl (9.0 kB)\n",
      "Downloading GitPython-3.1.43-py3-none-any.whl (207 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m207.3/207.3 kB\u001b[0m \u001b[31m109.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading sentry_sdk-2.13.0-py2.py3-none-any.whl (309 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m309.1/309.1 kB\u001b[0m \u001b[31m131.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading setproctitle-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (30 kB)\n",
      "Downloading gitdb-4.0.11-py3-none-any.whl (62 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.7/62.7 kB\u001b[0m \u001b[31m40.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading smmap-5.0.1-py3-none-any.whl (24 kB)\n",
      "Installing collected packages: smmap, setproctitle, sentry-sdk, docker-pycreds, click, gitdb, gitpython, wandb\n",
      "Successfully installed click-8.1.7 docker-pycreds-0.4.0 gitdb-4.0.11 gitpython-3.1.43 sentry-sdk-2.13.0 setproctitle-1.3.3 smmap-5.0.1 wandb-0.17.7\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython -m pip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install wandb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "cf4264c7-9577-42ad-843e-4edb594c948e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Logging into wandb.ai. (Learn how to deploy a W&B server locally: https://wandb.me/wandb-server)\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: You can find your API key in your browser here: https://wandb.ai/authorize\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Paste an API key from your profile and hit enter, or press ctrl+c to quit:"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "  ········\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import wandb\n",
    "wandb.login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "41400f4f-ce61-4806-87e8-537cd3fc225c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d03ac188b6af46d7b15288360f2a2adc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from huggingface_hub import notebook_login\n",
    "notebook_login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1891dec5-3665-489e-a87c-ddf35cc6a935",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting xformers==0.0.23\n",
      "  Downloading xformers-0.0.23-cp310-cp310-manylinux2014_x86_64.whl.metadata (1.0 kB)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from xformers==0.0.23) (1.24.1)\n",
      "Collecting torch==2.1.1 (from xformers==0.0.23)\n",
      "  Downloading torch-2.1.1-cp310-cp310-manylinux1_x86_64.whl.metadata (25 kB)\n",
      "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch==2.1.1->xformers==0.0.23) (3.9.0)\n",
      "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch==2.1.1->xformers==0.0.23) (4.12.2)\n",
      "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch==2.1.1->xformers==0.0.23) (1.12)\n",
      "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch==2.1.1->xformers==0.0.23) (3.0)\n",
      "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch==2.1.1->xformers==0.0.23) (3.1.2)\n",
      "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch==2.1.1->xformers==0.0.23) (2024.6.1)\n",
      "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch==2.1.1->xformers==0.0.23)\n",
      "  Downloading nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
      "Collecting nvidia-cuda-runtime-cu12==12.1.105 (from torch==2.1.1->xformers==0.0.23)\n",
      "  Downloading nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
      "Collecting nvidia-cuda-cupti-cu12==12.1.105 (from torch==2.1.1->xformers==0.0.23)\n",
      "  Downloading nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n",
      "Collecting nvidia-cudnn-cu12==8.9.2.26 (from torch==2.1.1->xformers==0.0.23)\n",
      "  Downloading nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n",
      "Collecting nvidia-cublas-cu12==12.1.3.1 (from torch==2.1.1->xformers==0.0.23)\n",
      "  Downloading nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
      "Collecting nvidia-cufft-cu12==11.0.2.54 (from torch==2.1.1->xformers==0.0.23)\n",
      "  Downloading nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
      "Collecting nvidia-curand-cu12==10.3.2.106 (from torch==2.1.1->xformers==0.0.23)\n",
      "  Downloading nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
      "Collecting nvidia-cusolver-cu12==11.4.5.107 (from torch==2.1.1->xformers==0.0.23)\n",
      "  Downloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n",
      "Collecting nvidia-cusparse-cu12==12.1.0.106 (from torch==2.1.1->xformers==0.0.23)\n",
      "  Downloading nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n",
      "Collecting nvidia-nccl-cu12==2.18.1 (from torch==2.1.1->xformers==0.0.23)\n",
      "  Downloading nvidia_nccl_cu12-2.18.1-py3-none-manylinux1_x86_64.whl.metadata (1.8 kB)\n",
      "Collecting nvidia-nvtx-cu12==12.1.105 (from torch==2.1.1->xformers==0.0.23)\n",
      "  Downloading nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.7 kB)\n",
      "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch==2.1.1->xformers==0.0.23) (2.1.0)\n",
      "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch==2.1.1->xformers==0.0.23)\n",
      "  Downloading nvidia_nvjitlink_cu12-12.6.20-py3-none-manylinux2014_x86_64.whl.metadata (1.5 kB)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch==2.1.1->xformers==0.0.23) (2.1.2)\n",
      "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch==2.1.1->xformers==0.0.23) (1.3.0)\n",
      "Downloading xformers-0.0.23-cp310-cp310-manylinux2014_x86_64.whl (213.0 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m213.0/213.0 MB\u001b[0m \u001b[31m26.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hDownloading torch-2.1.1-cp310-cp310-manylinux1_x86_64.whl (670.2 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m670.2/670.2 MB\u001b[0m \u001b[31m13.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hDownloading nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m410.6/410.6 MB\u001b[0m \u001b[31m21.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hDownloading nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.1/14.1 MB\u001b[0m \u001b[31m74.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hDownloading nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m23.7/23.7 MB\u001b[0m \u001b[31m67.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hDownloading nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m823.6/823.6 kB\u001b[0m \u001b[31m156.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m731.7/731.7 MB\u001b[0m \u001b[31m13.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hDownloading nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.6/121.6 MB\u001b[0m \u001b[31m44.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hDownloading nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.5/56.5 MB\u001b[0m \u001b[31m60.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hDownloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m124.2/124.2 MB\u001b[0m \u001b[31m43.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hDownloading nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m196.0/196.0 MB\u001b[0m \u001b[31m34.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hDownloading nvidia_nccl_cu12-2.18.1-py3-none-manylinux1_x86_64.whl (209.8 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m209.8/209.8 MB\u001b[0m \u001b[31m33.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hDownloading nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m99.1/99.1 kB\u001b[0m \u001b[31m60.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading nvidia_nvjitlink_cu12-12.6.20-py3-none-manylinux2014_x86_64.whl (19.7 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m19.7/19.7 MB\u001b[0m \u001b[31m80.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, nvidia-cusparse-cu12, nvidia-cudnn-cu12, nvidia-cusolver-cu12, torch, xformers\n",
      "  Attempting uninstall: torch\n",
      "    Found existing installation: torch 2.1.0+cu118\n",
      "    Uninstalling torch-2.1.0+cu118:\n",
      "      Successfully uninstalled torch-2.1.0+cu118\n",
      "  Attempting uninstall: xformers\n",
      "    Found existing installation: xformers 0.0.26.post1\n",
      "    Uninstalling xformers-0.0.26.post1:\n",
      "      Successfully uninstalled xformers-0.0.26.post1\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "torchaudio 2.1.0+cu118 requires torch==2.1.0, but you have torch 2.1.1 which is incompatible.\n",
      "torchvision 0.16.0+cu118 requires torch==2.1.0, but you have torch 2.1.1 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0mSuccessfully installed nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.18.1 nvidia-nvjitlink-cu12-12.6.20 nvidia-nvtx-cu12-12.1.105 torch-2.1.1 xformers-0.0.23\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython -m pip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install xformers==0.0.23"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8e58b2ed-b4e3-44b3-b8bb-c124d1b6233e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/workspace/llama_fim_finetuning\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/dist-packages/IPython/core/magics/osm.py:417: UserWarning: using dhist requires you to install the `pickleshare` library.\n",
      "  self.shell.db['dhist'] = compress_dhist(dhist)[-100:]\n"
     ]
    }
   ],
   "source": [
    "%cd llama_fim_finetuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "80e92737-03a5-4029-9c2d-7414f3d8e528",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Already up to date.\n",
      "/usr/local/lib/python3.10/dist-packages/transformers/training_args.py:1525: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
      "  warnings.warn(\n",
      "tokenizer_config.json: 100%|███████████████| 1.87k/1.87k [00:00<00:00, 12.1MB/s]\n",
      "tokenizer.json: 100%|██████████████████████| 1.37M/1.37M [00:00<00:00, 8.32MB/s]\n",
      "Downloading readme: 100%|██████████████████████| 509/509 [00:00<00:00, 5.71kB/s]\n",
      "Downloading data: 100%|████████████████████| 32.3M/32.3M [00:00<00:00, 33.7MB/s]\n",
      "Generating train split: 100%|█████| 6579/6579 [00:00<00:00, 15208.71 examples/s]\n",
      "Size of the train set: 5921. Size of the validation set: 658\n",
      "  0%|                                                   | 0/400 [00:00<?, ?it/s]Token indices sequence length is longer than the specified maximum sequence length for this model (22453 > 16384). Running this sequence through the model will result in indexing errors\n",
      "100%|█████████████████████████████████████████| 400/400 [00:04<00:00, 88.41it/s]\n",
      "The character to token ratio of the dataset is: 3.01\n",
      "A sample of valid dataset: {'input_ids': tensor([32013, 32016,  3154,  ...,  3563,  4941,  2949]), 'labels': tensor([32013, 32016,  3154,  ...,  3563,  4941,  2949])}\n",
      "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n",
      "Unrecognized keys in `rope_scaling` for 'rope_type'='linear': {'type'}\n",
      "==((====))==  Unsloth 2024.8: Fast Llama patching. Transformers = 4.44.1.\n",
      "   \\\\   /|    GPU: NVIDIA H100 80GB HBM3. Max memory: 79.216 GB. Platform = Linux.\n",
      "O^O/ \\_/ \\    Pytorch: 2.1.1+cu121. CUDA = 9.0. CUDA Toolkit = 12.1.\n",
      "\\        /    Bfloat16 = TRUE. FA [Xformers = 0.0.23. FA2 = False]\n",
      " \"-____-\"     Free Apache license: http://github.com/unslothai/unsloth\n",
      "Unrecognized keys in `rope_scaling` for 'rope_type'='linear': {'type'}\n",
      "Unrecognized keys in `rope_scaling` for 'rope_type'='linear': {'type'}\n",
      "model.safetensors: 100%|███████████████████| 2.69G/2.69G [00:50<00:00, 53.2MB/s]\n",
      "generation_config.json: 100%|███████████████████| 119/119 [00:00<00:00, 287kB/s]\n",
      "Unsloth: Dropout = 0 is supported for fast patching. You are using dropout = 0.1.\n",
      "Unsloth will patch all other layers, except LoRA matrices, causing a performance hit.\n",
      "Unsloth 2024.8 patched 24 layers with 0 QKV layers, 0 O layers and 0 MLP layers.\n",
      "max_steps is given, it will override any value given in num_train_epochs\n",
      "Using auto half precision backend\n",
      "PeftModelForCausalLM(\n",
      "  (base_model): LoraModel(\n",
      "    (model): LlamaForCausalLM(\n",
      "      (model): LlamaModel(\n",
      "        (embed_tokens): Embedding(32256, 2048)\n",
      "        (layers): ModuleList(\n",
      "          (0-23): 24 x LlamaDecoderLayer(\n",
      "            (self_attn): LlamaAttention(\n",
      "              (q_proj): lora.Linear4bit(\n",
      "                (base_layer): Linear4bit(in_features=2048, out_features=2048, bias=False)\n",
      "                (lora_dropout): ModuleDict(\n",
      "                  (default): Dropout(p=0.1, inplace=False)\n",
      "                )\n",
      "                (lora_A): ModuleDict(\n",
      "                  (default): Linear(in_features=2048, out_features=16, bias=False)\n",
      "                )\n",
      "                (lora_B): ModuleDict(\n",
      "                  (default): Linear(in_features=16, out_features=2048, bias=False)\n",
      "                )\n",
      "                (lora_embedding_A): ParameterDict()\n",
      "                (lora_embedding_B): ParameterDict()\n",
      "                (lora_magnitude_vector): ModuleDict()\n",
      "              )\n",
      "              (k_proj): lora.Linear4bit(\n",
      "                (base_layer): Linear4bit(in_features=2048, out_features=2048, bias=False)\n",
      "                (lora_dropout): ModuleDict(\n",
      "                  (default): Dropout(p=0.1, inplace=False)\n",
      "                )\n",
      "                (lora_A): ModuleDict(\n",
      "                  (default): Linear(in_features=2048, out_features=16, bias=False)\n",
      "                )\n",
      "                (lora_B): ModuleDict(\n",
      "                  (default): Linear(in_features=16, out_features=2048, bias=False)\n",
      "                )\n",
      "                (lora_embedding_A): ParameterDict()\n",
      "                (lora_embedding_B): ParameterDict()\n",
      "                (lora_magnitude_vector): ModuleDict()\n",
      "              )\n",
      "              (v_proj): lora.Linear4bit(\n",
      "                (base_layer): Linear4bit(in_features=2048, out_features=2048, bias=False)\n",
      "                (lora_dropout): ModuleDict(\n",
      "                  (default): Dropout(p=0.1, inplace=False)\n",
      "                )\n",
      "                (lora_A): ModuleDict(\n",
      "                  (default): Linear(in_features=2048, out_features=16, bias=False)\n",
      "                )\n",
      "                (lora_B): ModuleDict(\n",
      "                  (default): Linear(in_features=16, out_features=2048, bias=False)\n",
      "                )\n",
      "                (lora_embedding_A): ParameterDict()\n",
      "                (lora_embedding_B): ParameterDict()\n",
      "                (lora_magnitude_vector): ModuleDict()\n",
      "              )\n",
      "              (o_proj): lora.Linear4bit(\n",
      "                (base_layer): Linear4bit(in_features=2048, out_features=2048, bias=False)\n",
      "                (lora_dropout): ModuleDict(\n",
      "                  (default): Dropout(p=0.1, inplace=False)\n",
      "                )\n",
      "                (lora_A): ModuleDict(\n",
      "                  (default): Linear(in_features=2048, out_features=16, bias=False)\n",
      "                )\n",
      "                (lora_B): ModuleDict(\n",
      "                  (default): Linear(in_features=16, out_features=2048, bias=False)\n",
      "                )\n",
      "                (lora_embedding_A): ParameterDict()\n",
      "                (lora_embedding_B): ParameterDict()\n",
      "                (lora_magnitude_vector): ModuleDict()\n",
      "              )\n",
      "              (rotary_emb): LlamaLinearScalingRotaryEmbedding()\n",
      "            )\n",
      "            (mlp): LlamaMLP(\n",
      "              (gate_proj): lora.Linear4bit(\n",
      "                (base_layer): Linear4bit(in_features=2048, out_features=5504, bias=False)\n",
      "                (lora_dropout): ModuleDict(\n",
      "                  (default): Dropout(p=0.1, inplace=False)\n",
      "                )\n",
      "                (lora_A): ModuleDict(\n",
      "                  (default): Linear(in_features=2048, out_features=16, bias=False)\n",
      "                )\n",
      "                (lora_B): ModuleDict(\n",
      "                  (default): Linear(in_features=16, out_features=5504, bias=False)\n",
      "                )\n",
      "                (lora_embedding_A): ParameterDict()\n",
      "                (lora_embedding_B): ParameterDict()\n",
      "                (lora_magnitude_vector): ModuleDict()\n",
      "              )\n",
      "              (up_proj): lora.Linear4bit(\n",
      "                (base_layer): Linear4bit(in_features=2048, out_features=5504, bias=False)\n",
      "                (lora_dropout): ModuleDict(\n",
      "                  (default): Dropout(p=0.1, inplace=False)\n",
      "                )\n",
      "                (lora_A): ModuleDict(\n",
      "                  (default): Linear(in_features=2048, out_features=16, bias=False)\n",
      "                )\n",
      "                (lora_B): ModuleDict(\n",
      "                  (default): Linear(in_features=16, out_features=5504, bias=False)\n",
      "                )\n",
      "                (lora_embedding_A): ParameterDict()\n",
      "                (lora_embedding_B): ParameterDict()\n",
      "                (lora_magnitude_vector): ModuleDict()\n",
      "              )\n",
      "              (down_proj): lora.Linear4bit(\n",
      "                (base_layer): Linear4bit(in_features=5504, out_features=2048, bias=False)\n",
      "                (lora_dropout): ModuleDict(\n",
      "                  (default): Dropout(p=0.1, inplace=False)\n",
      "                )\n",
      "                (lora_A): ModuleDict(\n",
      "                  (default): Linear(in_features=5504, out_features=16, bias=False)\n",
      "                )\n",
      "                (lora_B): ModuleDict(\n",
      "                  (default): Linear(in_features=16, out_features=2048, bias=False)\n",
      "                )\n",
      "                (lora_embedding_A): ParameterDict()\n",
      "                (lora_embedding_B): ParameterDict()\n",
      "                (lora_magnitude_vector): ModuleDict()\n",
      "              )\n",
      "              (act_fn): SiLU()\n",
      "            )\n",
      "            (input_layernorm): LlamaRMSNorm((2048,), eps=1e-06)\n",
      "            (post_attention_layernorm): LlamaRMSNorm((2048,), eps=1e-06)\n",
      "          )\n",
      "        )\n",
      "        (norm): LlamaRMSNorm((2048,), eps=1e-06)\n",
      "        (rotary_emb): LlamaRotaryEmbedding()\n",
      "      )\n",
      "      (lm_head): Linear(in_features=2048, out_features=32256, bias=False)\n",
      "    )\n",
      "  )\n",
      ")\n",
      "trainable params: 14,991,360 || all params: 1,361,463,296 || trainable%: 1.1011\n",
      "==((====))==  Unsloth - 2x faster free finetuning | Num GPUs = 1\n",
      "   \\\\   /|    Num examples = 128,000 | Num Epochs = 9,223,372,036,854,775,807\n",
      "O^O/ \\_/ \\    Batch size per device = 16 | Gradient Accumulation steps = 4\n",
      "\\        /    Total batch size = 64 | Total steps = 2,000\n",
      " \"-____-\"     Number of trainable parameters = 14,991,360\n",
      "Automatic Weights & Biases logging enabled, to disable set os.environ[\"WANDB_DISABLED\"] = \"true\"\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m The `run_name` is currently set to the same value as `TrainingArguments.output_dir`. If this was not intended, please specify a different run name by setting the `TrainingArguments.run_name` parameter.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mardalan405\u001b[0m (\u001b[33mardalan405-university-of-guilan\u001b[0m). Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.17.7\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/workspace/llama_fim_finetuning/wandb/run-20240821_143916-ys4ug9sd\u001b[0m\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33mfim_llama\u001b[0m\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/ardalan405-university-of-guilan/personal-code-copilot\u001b[0m\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/ardalan405-university-of-guilan/personal-code-copilot/runs/ys4ug9sd\u001b[0m\n",
      "{'loss': 0.9914, 'grad_norm': 0.08931465446949005, 'learning_rate': 5e-06, 'epoch': 0.0}\n",
      "{'loss': 0.9587, 'grad_norm': 0.09965350478887558, 'learning_rate': 1e-05, 'epoch': 0.01}\n",
      "{'loss': 0.9845, 'grad_norm': 0.10502968728542328, 'learning_rate': 1.5e-05, 'epoch': 0.01}\n",
      "{'loss': 0.942, 'grad_norm': 0.1073104590177536, 'learning_rate': 2e-05, 'epoch': 0.01}\n",
      "{'loss': 0.9168, 'grad_norm': 0.10257415473461151, 'learning_rate': 2.5e-05, 'epoch': 0.01}\n",
      "{'loss': 0.9338, 'grad_norm': 0.13417939841747284, 'learning_rate': 3e-05, 'epoch': 0.01}\n",
      "{'loss': 0.9083, 'grad_norm': 0.11853798478841782, 'learning_rate': 3.5e-05, 'epoch': 0.02}\n",
      "{'loss': 0.9478, 'grad_norm': 0.0973772406578064, 'learning_rate': 4e-05, 'epoch': 0.02}\n",
      "{'loss': 0.939, 'grad_norm': 0.05733523145318031, 'learning_rate': 4.5e-05, 'epoch': 0.02}\n",
      "{'loss': 0.8555, 'grad_norm': 0.08896207064390182, 'learning_rate': 5e-05, 'epoch': 0.03}\n",
      "{'loss': 0.8211, 'grad_norm': 0.0747579038143158, 'learning_rate': 5.500000000000001e-05, 'epoch': 0.03}\n",
      "{'loss': 0.799, 'grad_norm': 0.04328622668981552, 'learning_rate': 6e-05, 'epoch': 0.03}\n",
      "{'loss': 0.8207, 'grad_norm': 0.04649871215224266, 'learning_rate': 6.500000000000001e-05, 'epoch': 0.03}\n",
      "{'loss': 0.8386, 'grad_norm': 0.04743924364447594, 'learning_rate': 7e-05, 'epoch': 0.04}\n",
      "{'loss': 0.8504, 'grad_norm': 0.03946412354707718, 'learning_rate': 7.500000000000001e-05, 'epoch': 0.04}\n",
      "{'loss': 0.8354, 'grad_norm': 0.042876649647951126, 'learning_rate': 8e-05, 'epoch': 0.04}\n",
      "{'loss': 0.7966, 'grad_norm': 0.03933627903461456, 'learning_rate': 8.5e-05, 'epoch': 0.04}\n",
      "{'loss': 0.795, 'grad_norm': 0.040357768535614014, 'learning_rate': 9e-05, 'epoch': 0.04}\n",
      "{'loss': 0.7541, 'grad_norm': 0.03959975391626358, 'learning_rate': 9.5e-05, 'epoch': 0.05}\n",
      "{'loss': 0.7856, 'grad_norm': 0.04722171276807785, 'learning_rate': 0.0001, 'epoch': 0.05}\n",
      "  5%|█▉                                    | 100/2000 [06:45<2:08:25,  4.06s/it]\n",
      "***** Running Evaluation *****\n",
      "  Num examples: Unknown\n",
      "  Batch size = 16\n",
      "{'eval_loss': 0.7991667985916138, 'eval_runtime': 27.4214, 'eval_samples_per_second': 53.681, 'eval_steps_per_second': 3.355, 'epoch': 0.05}\n",
      "{'loss': 0.7826, 'grad_norm': 0.046507563441991806, 'learning_rate': 0.000105, 'epoch': 0.05}\n",
      "{'loss': 0.7817, 'grad_norm': 0.046817634254693985, 'learning_rate': 0.00011000000000000002, 'epoch': 0.06}\n",
      "{'loss': 0.762, 'grad_norm': 0.05649365484714508, 'learning_rate': 0.00011499999999999999, 'epoch': 0.06}\n",
      "{'loss': 0.7933, 'grad_norm': 0.059349190443754196, 'learning_rate': 0.00012, 'epoch': 0.06}\n",
      "{'loss': 0.788, 'grad_norm': 0.04952576011419296, 'learning_rate': 0.000125, 'epoch': 0.06}\n",
      "{'loss': 0.7797, 'grad_norm': 0.05525524169206619, 'learning_rate': 0.00013000000000000002, 'epoch': 0.07}\n",
      "{'loss': 0.7099, 'grad_norm': 0.06189746409654617, 'learning_rate': 0.00013500000000000003, 'epoch': 0.07}\n",
      "{'loss': 0.7221, 'grad_norm': 0.06159722059965134, 'learning_rate': 0.00014, 'epoch': 0.07}\n",
      "{'loss': 0.671, 'grad_norm': 0.06149214506149292, 'learning_rate': 0.000145, 'epoch': 0.07}\n",
      "{'loss': 0.7408, 'grad_norm': 0.05969501659274101, 'learning_rate': 0.00015000000000000001, 'epoch': 0.07}\n",
      "{'loss': 0.7379, 'grad_norm': 0.062133047729730606, 'learning_rate': 0.000155, 'epoch': 0.08}\n",
      "{'loss': 0.7387, 'grad_norm': 0.0722159743309021, 'learning_rate': 0.00016, 'epoch': 0.08}\n",
      "{'loss': 0.7361, 'grad_norm': 0.07085324078798294, 'learning_rate': 0.000165, 'epoch': 0.08}\n",
      "{'loss': 0.7538, 'grad_norm': 0.07338175922632217, 'learning_rate': 0.00017, 'epoch': 0.09}\n",
      "{'loss': 0.7647, 'grad_norm': 0.07789365947246552, 'learning_rate': 0.000175, 'epoch': 0.09}\n",
      "{'loss': 0.7332, 'grad_norm': 0.07396231591701508, 'learning_rate': 0.00018, 'epoch': 0.09}\n",
      "{'loss': 0.6795, 'grad_norm': 0.08217714726924896, 'learning_rate': 0.00018500000000000002, 'epoch': 0.09}\n",
      "{'loss': 0.7216, 'grad_norm': 0.08340702205896378, 'learning_rate': 0.00019, 'epoch': 0.1}\n",
      "{'loss': 0.7055, 'grad_norm': 0.07243268191814423, 'learning_rate': 0.000195, 'epoch': 0.1}\n",
      "{'loss': 0.7197, 'grad_norm': 0.08899835497140884, 'learning_rate': 0.0002, 'epoch': 0.1}\n",
      " 10%|███▊                                  | 200/2000 [13:55<2:00:11,  4.01s/it]\n",
      "***** Running Evaluation *****\n",
      "  Num examples: Unknown\n",
      "  Batch size = 16\n",
      "{'eval_loss': 0.7235196828842163, 'eval_runtime': 27.4519, 'eval_samples_per_second': 53.621, 'eval_steps_per_second': 3.351, 'epoch': 0.1}\n",
      "{'loss': 0.7049, 'grad_norm': 0.08276673406362534, 'learning_rate': 0.00019999619230641713, 'epoch': 0.1}\n",
      "{'loss': 0.7298, 'grad_norm': 0.07591517269611359, 'learning_rate': 0.00019998476951563915, 'epoch': 0.1}\n",
      "{'loss': 0.7328, 'grad_norm': 0.08176608383655548, 'learning_rate': 0.00019996573249755572, 'epoch': 0.11}\n",
      "{'loss': 0.6975, 'grad_norm': 0.0809355229139328, 'learning_rate': 0.0001999390827019096, 'epoch': 0.11}\n",
      "{'loss': 0.7312, 'grad_norm': 0.1017574891448021, 'learning_rate': 0.0001999048221581858, 'epoch': 0.11}\n",
      "{'loss': 0.7113, 'grad_norm': 0.08066695183515549, 'learning_rate': 0.0001998629534754574, 'epoch': 0.12}\n",
      "{'loss': 0.7144, 'grad_norm': 0.06965827196836472, 'learning_rate': 0.0001998134798421867, 'epoch': 0.12}\n",
      "{'loss': 0.7184, 'grad_norm': 0.07197012007236481, 'learning_rate': 0.00019975640502598244, 'epoch': 0.12}\n",
      "{'loss': 0.7298, 'grad_norm': 0.07070869207382202, 'learning_rate': 0.0001996917333733128, 'epoch': 0.12}\n",
      "{'loss': 0.747, 'grad_norm': 0.07369192689657211, 'learning_rate': 0.00019961946980917456, 'epoch': 0.12}\n",
      "{'loss': 0.7636, 'grad_norm': 0.09922760725021362, 'learning_rate': 0.00019953961983671788, 'epoch': 0.13}\n",
      "{'loss': 0.7065, 'grad_norm': 0.08758189529180527, 'learning_rate': 0.00019945218953682734, 'epoch': 0.13}\n",
      "{'loss': 0.6269, 'grad_norm': 0.08969839662313461, 'learning_rate': 0.00019935718556765876, 'epoch': 0.13}\n",
      "{'loss': 0.6086, 'grad_norm': 0.09615705162286758, 'learning_rate': 0.00019925461516413223, 'epoch': 0.14}\n",
      "{'loss': 0.6395, 'grad_norm': 0.08039776980876923, 'learning_rate': 0.00019914448613738106, 'epoch': 0.14}\n",
      "{'loss': 0.6756, 'grad_norm': 0.08975008875131607, 'learning_rate': 0.00019902680687415705, 'epoch': 0.14}\n",
      "{'loss': 0.6645, 'grad_norm': 0.09787575900554657, 'learning_rate': 0.0001989015863361917, 'epoch': 0.14}\n",
      "{'loss': 0.6997, 'grad_norm': 0.11832548677921295, 'learning_rate': 0.00019876883405951377, 'epoch': 0.14}\n",
      "{'loss': 0.7018, 'grad_norm': 0.07970436662435532, 'learning_rate': 0.00019862856015372317, 'epoch': 0.15}\n",
      "{'loss': 0.6471, 'grad_norm': 0.08168169856071472, 'learning_rate': 0.00019848077530122083, 'epoch': 0.15}\n",
      " 15%|█████▋                                | 300/2000 [21:06<1:53:22,  4.00s/it]\n",
      "***** Running Evaluation *****\n",
      "  Num examples: Unknown\n",
      "  Batch size = 16\n",
      "{'eval_loss': 0.6876130104064941, 'eval_runtime': 27.3512, 'eval_samples_per_second': 53.818, 'eval_steps_per_second': 3.364, 'epoch': 0.15}\n",
      "{'loss': 0.6527, 'grad_norm': 0.08630518615245819, 'learning_rate': 0.0001983254907563955, 'epoch': 0.15}\n",
      "{'loss': 0.6708, 'grad_norm': 0.07388246804475784, 'learning_rate': 0.00019816271834476642, 'epoch': 0.15}\n",
      "{'loss': 0.6562, 'grad_norm': 0.08780287951231003, 'learning_rate': 0.00019799247046208297, 'epoch': 0.16}\n",
      "{'loss': 0.6772, 'grad_norm': 0.07254461944103241, 'learning_rate': 0.00019781476007338058, 'epoch': 0.16}\n",
      "{'loss': 0.691, 'grad_norm': 0.08565288782119751, 'learning_rate': 0.00019762960071199333, 'epoch': 0.16}\n",
      "{'loss': 0.6914, 'grad_norm': 0.07783597707748413, 'learning_rate': 0.00019743700647852354, 'epoch': 0.17}\n",
      "{'loss': 0.6655, 'grad_norm': 0.07758184522390366, 'learning_rate': 0.00019723699203976766, 'epoch': 0.17}\n",
      "{'loss': 0.6724, 'grad_norm': 0.08369570225477219, 'learning_rate': 0.00019702957262759965, 'epoch': 0.17}\n",
      "{'loss': 0.6094, 'grad_norm': 0.08514421433210373, 'learning_rate': 0.0001968147640378108, 'epoch': 0.17}\n",
      "{'loss': 0.6453, 'grad_norm': 0.08423178642988205, 'learning_rate': 0.00019659258262890683, 'epoch': 0.17}\n",
      "{'loss': 0.6046, 'grad_norm': 0.07619503885507584, 'learning_rate': 0.0001963630453208623, 'epoch': 0.18}\n",
      "{'loss': 0.6808, 'grad_norm': 0.09945912659168243, 'learning_rate': 0.0001961261695938319, 'epoch': 0.18}\n",
      "{'loss': 0.666, 'grad_norm': 0.08663097769021988, 'learning_rate': 0.0001958819734868193, 'epoch': 0.18}\n",
      "{'loss': 0.6372, 'grad_norm': 0.09123697131872177, 'learning_rate': 0.00019563047559630357, 'epoch': 0.18}\n",
      "{'loss': 0.6842, 'grad_norm': 0.11710835248231888, 'learning_rate': 0.0001953716950748227, 'epoch': 0.19}\n",
      "{'loss': 0.6701, 'grad_norm': 0.08709461987018585, 'learning_rate': 0.00019510565162951537, 'epoch': 0.19}\n",
      "{'loss': 0.6705, 'grad_norm': 0.07744211703538895, 'learning_rate': 0.00019483236552061994, 'epoch': 0.19}\n",
      "{'loss': 0.6188, 'grad_norm': 0.08408669382333755, 'learning_rate': 0.0001945518575599317, 'epoch': 0.2}\n",
      "{'loss': 0.6355, 'grad_norm': 0.10033920407295227, 'learning_rate': 0.00019426414910921787, 'epoch': 0.2}\n",
      "{'loss': 0.6383, 'grad_norm': 0.07906177639961243, 'learning_rate': 0.00019396926207859084, 'epoch': 0.2}\n",
      " 20%|███████▌                              | 400/2000 [28:17<1:46:14,  3.98s/it]\n",
      "***** Running Evaluation *****\n",
      "  Num examples: Unknown\n",
      "  Batch size = 16\n",
      "{'eval_loss': 0.6619904637336731, 'eval_runtime': 27.4542, 'eval_samples_per_second': 53.617, 'eval_steps_per_second': 3.351, 'epoch': 0.2}\n",
      "{'loss': 0.6235, 'grad_norm': 0.08375341445207596, 'learning_rate': 0.00019366721892483978, 'epoch': 0.2}\n",
      "{'loss': 0.6654, 'grad_norm': 0.09167716652154922, 'learning_rate': 0.00019335804264972018, 'epoch': 0.2}\n",
      "{'loss': 0.6394, 'grad_norm': 0.08455649763345718, 'learning_rate': 0.00019304175679820247, 'epoch': 0.21}\n",
      "{'loss': 0.6365, 'grad_norm': 0.08978413790464401, 'learning_rate': 0.00019271838545667876, 'epoch': 0.21}\n",
      "{'loss': 0.6625, 'grad_norm': 0.08237719535827637, 'learning_rate': 0.0001923879532511287, 'epoch': 0.21}\n",
      "{'loss': 0.6473, 'grad_norm': 0.08573328703641891, 'learning_rate': 0.00019205048534524406, 'epoch': 0.21}\n",
      "{'loss': 0.6745, 'grad_norm': 0.07707343995571136, 'learning_rate': 0.0001917060074385124, 'epoch': 0.22}\n",
      "{'loss': 0.6536, 'grad_norm': 0.07993399351835251, 'learning_rate': 0.0001913545457642601, 'epoch': 0.22}\n",
      "{'loss': 0.643, 'grad_norm': 0.07474994659423828, 'learning_rate': 0.00019099612708765434, 'epoch': 0.22}\n",
      "{'loss': 0.6454, 'grad_norm': 0.07723818719387054, 'learning_rate': 0.000190630778703665, 'epoch': 0.23}\n",
      "{'loss': 0.7014, 'grad_norm': 0.08291968703269958, 'learning_rate': 0.00019025852843498607, 'epoch': 0.23}\n",
      "{'loss': 0.7024, 'grad_norm': 0.07827633619308472, 'learning_rate': 0.0001898794046299167, 'epoch': 0.23}\n",
      "{'loss': 0.6799, 'grad_norm': 0.07579002529382706, 'learning_rate': 0.00018949343616020252, 'epoch': 0.23}\n",
      "{'loss': 0.6191, 'grad_norm': 0.09009797871112823, 'learning_rate': 0.0001891006524188368, 'epoch': 0.23}\n",
      "{'loss': 0.5735, 'grad_norm': 0.09737949073314667, 'learning_rate': 0.00018870108331782217, 'epoch': 0.24}\n",
      "{'loss': 0.5652, 'grad_norm': 0.09152916818857193, 'learning_rate': 0.00018829475928589271, 'epoch': 0.24}\n",
      "{'loss': 0.5586, 'grad_norm': 0.08465364575386047, 'learning_rate': 0.00018788171126619653, 'epoch': 0.24}\n",
      "{'loss': 0.625, 'grad_norm': 0.07967010140419006, 'learning_rate': 0.00018746197071393958, 'epoch': 0.24}\n",
      "{'loss': 0.6209, 'grad_norm': 0.09266957640647888, 'learning_rate': 0.00018703556959398998, 'epoch': 0.25}\n",
      "{'loss': 0.6352, 'grad_norm': 0.07705790549516678, 'learning_rate': 0.00018660254037844388, 'epoch': 0.25}\n",
      " 25%|█████████▌                            | 500/2000 [35:28<1:45:25,  4.22s/it]\n",
      "***** Running Evaluation *****\n",
      "  Num examples: Unknown\n",
      "  Batch size = 16\n",
      "{'eval_loss': 0.639828085899353, 'eval_runtime': 27.7055, 'eval_samples_per_second': 53.13, 'eval_steps_per_second': 3.321, 'epoch': 0.25}\n",
      " 25%|█████████▌                            | 500/2000 [35:56<1:45:25,  4.22s/it]Saving model checkpoint to fim_llama/checkpoint-500\n",
      "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--deepseek-ai--deepseek-coder-1.3b-instruct/snapshots/e063262dac8366fc1f28a4da0ff3c50ea66259ca/config.json\n",
      "Unrecognized keys in `rope_scaling` for 'rope_type'='linear': {'type'}\n",
      "Model config LlamaConfig {\n",
      "  \"architectures\": [\n",
      "    \"LlamaForCausalLM\"\n",
      "  ],\n",
      "  \"attention_bias\": false,\n",
      "  \"attention_dropout\": 0.0,\n",
      "  \"bos_token_id\": 32013,\n",
      "  \"eos_token_id\": 32021,\n",
      "  \"hidden_act\": \"silu\",\n",
      "  \"hidden_size\": 2048,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 5504,\n",
      "  \"max_position_embeddings\": 16384,\n",
      "  \"mlp_bias\": false,\n",
      "  \"model_type\": \"llama\",\n",
      "  \"num_attention_heads\": 16,\n",
      "  \"num_hidden_layers\": 24,\n",
      "  \"num_key_value_heads\": 16,\n",
      "  \"pretraining_tp\": 1,\n",
      "  \"rms_norm_eps\": 1e-06,\n",
      "  \"rope_scaling\": {\n",
      "    \"factor\": 4.0,\n",
      "    \"rope_type\": \"linear\",\n",
      "    \"type\": \"linear\"\n",
      "  },\n",
      "  \"rope_theta\": 100000,\n",
      "  \"tie_word_embeddings\": false,\n",
      "  \"torch_dtype\": \"bfloat16\",\n",
      "  \"transformers_version\": \"4.44.1\",\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 32256\n",
      "}\n",
      "\n",
      "{'loss': 0.5951, 'grad_norm': 0.0934540256857872, 'learning_rate': 0.00018616291604415258, 'epoch': 0.25}\n",
      "{'loss': 0.6306, 'grad_norm': 0.08586423099040985, 'learning_rate': 0.00018571673007021123, 'epoch': 0.26}\n",
      "{'loss': 0.5935, 'grad_norm': 0.0792413279414177, 'learning_rate': 0.00018526401643540922, 'epoch': 0.26}\n",
      "{'loss': 0.6186, 'grad_norm': 0.07678309082984924, 'learning_rate': 0.0001848048096156426, 'epoch': 0.26}\n",
      "{'loss': 0.6124, 'grad_norm': 0.08484379947185516, 'learning_rate': 0.0001843391445812886, 'epoch': 0.26}\n",
      "{'loss': 0.6134, 'grad_norm': 0.07602684199810028, 'learning_rate': 0.00018386705679454242, 'epoch': 0.27}\n",
      "{'loss': 0.6426, 'grad_norm': 0.08804088085889816, 'learning_rate': 0.00018338858220671682, 'epoch': 0.27}\n",
      "{'loss': 0.6429, 'grad_norm': 0.08014117926359177, 'learning_rate': 0.00018290375725550417, 'epoch': 0.27}\n",
      "{'loss': 0.6507, 'grad_norm': 0.08792722970247269, 'learning_rate': 0.00018241261886220154, 'epoch': 0.27}\n",
      "{'loss': 0.5973, 'grad_norm': 0.0783887729048729, 'learning_rate': 0.0001819152044288992, 'epoch': 0.28}\n",
      "{'loss': 0.5692, 'grad_norm': 0.10323406010866165, 'learning_rate': 0.00018141155183563193, 'epoch': 0.28}\n",
      "{'loss': 0.5394, 'grad_norm': 0.08329281210899353, 'learning_rate': 0.00018090169943749476, 'epoch': 0.28}\n",
      "{'loss': 0.5885, 'grad_norm': 0.09396232664585114, 'learning_rate': 0.00018038568606172173, 'epoch': 0.28}\n",
      "{'loss': 0.5928, 'grad_norm': 0.07952198386192322, 'learning_rate': 0.00017986355100472928, 'epoch': 0.28}\n",
      "{'loss': 0.6001, 'grad_norm': 0.07985331118106842, 'learning_rate': 0.00017933533402912354, 'epoch': 0.29}\n",
      "{'loss': 0.604, 'grad_norm': 0.08425755798816681, 'learning_rate': 0.00017880107536067218, 'epoch': 0.29}\n",
      "{'loss': 0.6039, 'grad_norm': 0.10370580852031708, 'learning_rate': 0.0001782608156852414, 'epoch': 0.29}\n",
      "{'loss': 0.617, 'grad_norm': 0.10013758391141891, 'learning_rate': 0.0001777145961456971, 'epoch': 0.29}\n",
      "{'loss': 0.6019, 'grad_norm': 0.08401121944189072, 'learning_rate': 0.00017716245833877201, 'epoch': 0.3}\n",
      "{'loss': 0.6103, 'grad_norm': 0.1284884810447693, 'learning_rate': 0.0001766044443118978, 'epoch': 0.3}\n",
      " 30%|███████████▍                          | 600/2000 [42:42<1:34:41,  4.06s/it]\n",
      "***** Running Evaluation *****\n",
      "  Num examples: Unknown\n",
      "  Batch size = 16\n",
      "{'eval_loss': 0.6243872046470642, 'eval_runtime': 27.55, 'eval_samples_per_second': 53.43, 'eval_steps_per_second': 3.339, 'epoch': 0.3}\n",
      "{'loss': 0.5822, 'grad_norm': 0.10390838980674744, 'learning_rate': 0.0001760405965600031, 'epoch': 0.3}\n",
      "{'loss': 0.6114, 'grad_norm': 0.0830380842089653, 'learning_rate': 0.00017547095802227723, 'epoch': 0.3}\n",
      "{'loss': 0.5864, 'grad_norm': 0.07518042623996735, 'learning_rate': 0.00017489557207890023, 'epoch': 0.31}\n",
      "{'loss': 0.6102, 'grad_norm': 0.08863691240549088, 'learning_rate': 0.00017431448254773944, 'epoch': 0.31}\n",
      "{'loss': 0.5992, 'grad_norm': 0.08606618642807007, 'learning_rate': 0.0001737277336810124, 'epoch': 0.31}\n",
      "{'loss': 0.6282, 'grad_norm': 0.08565220236778259, 'learning_rate': 0.00017313537016191706, 'epoch': 0.32}\n",
      "{'loss': 0.6582, 'grad_norm': 0.08975878357887268, 'learning_rate': 0.00017253743710122875, 'epoch': 0.32}\n",
      "{'loss': 0.6093, 'grad_norm': 0.09197087585926056, 'learning_rate': 0.0001719339800338651, 'epoch': 0.32}\n",
      "{'loss': 0.6041, 'grad_norm': 0.08055991679430008, 'learning_rate': 0.00017132504491541818, 'epoch': 0.32}\n",
      "{'loss': 0.5888, 'grad_norm': 0.09446295350790024, 'learning_rate': 0.00017071067811865476, 'epoch': 0.33}\n",
      "{'loss': 0.6263, 'grad_norm': 0.08155995607376099, 'learning_rate': 0.0001700909264299851, 'epoch': 0.33}\n",
      "{'loss': 0.5857, 'grad_norm': 0.10109683126211166, 'learning_rate': 0.00016946583704589973, 'epoch': 0.33}\n",
      "{'loss': 0.6622, 'grad_norm': 0.09784011542797089, 'learning_rate': 0.0001688354575693754, 'epoch': 0.33}\n",
      "{'loss': 0.6462, 'grad_norm': 0.10091932117938995, 'learning_rate': 0.00016819983600624986, 'epoch': 0.34}\n",
      "{'loss': 0.6528, 'grad_norm': 0.08444683998823166, 'learning_rate': 0.00016755902076156604, 'epoch': 0.34}\n",
      "{'loss': 0.5882, 'grad_norm': 0.10533221811056137, 'learning_rate': 0.00016691306063588583, 'epoch': 0.34}\n",
      "{'loss': 0.5676, 'grad_norm': 0.08271635323762894, 'learning_rate': 0.00016626200482157378, 'epoch': 0.34}\n",
      "{'loss': 0.5322, 'grad_norm': 0.0861879289150238, 'learning_rate': 0.00016560590289905073, 'epoch': 0.34}\n",
      "{'loss': 0.5511, 'grad_norm': 0.08479107171297073, 'learning_rate': 0.00016494480483301836, 'epoch': 0.35}\n",
      "{'loss': 0.5722, 'grad_norm': 0.09172743558883667, 'learning_rate': 0.00016427876096865394, 'epoch': 0.35}\n",
      " 35%|█████████████▎                        | 700/2000 [49:52<1:26:43,  4.00s/it]\n",
      "***** Running Evaluation *****\n",
      "  Num examples: Unknown\n",
      "  Batch size = 16\n",
      "{'eval_loss': 0.6107571721076965, 'eval_runtime': 27.3903, 'eval_samples_per_second': 53.742, 'eval_steps_per_second': 3.359, 'epoch': 0.35}\n",
      "{'loss': 0.563, 'grad_norm': 0.08303554356098175, 'learning_rate': 0.0001636078220277764, 'epoch': 0.35}\n",
      "{'loss': 0.5497, 'grad_norm': 0.09457661211490631, 'learning_rate': 0.00016293203910498376, 'epoch': 0.35}\n",
      "{'loss': 0.5683, 'grad_norm': 0.08191928267478943, 'learning_rate': 0.00016225146366376198, 'epoch': 0.36}\n",
      "{'loss': 0.5534, 'grad_norm': 0.09998566657304764, 'learning_rate': 0.0001615661475325658, 'epoch': 0.36}\n",
      "{'loss': 0.5768, 'grad_norm': 0.08539795875549316, 'learning_rate': 0.00016087614290087208, 'epoch': 0.36}\n",
      "{'loss': 0.5657, 'grad_norm': 0.09718336910009384, 'learning_rate': 0.00016018150231520486, 'epoch': 0.36}\n",
      "{'loss': 0.5708, 'grad_norm': 0.08935309946537018, 'learning_rate': 0.00015948227867513415, 'epoch': 0.37}\n",
      "{'loss': 0.567, 'grad_norm': 0.0888625830411911, 'learning_rate': 0.00015877852522924732, 'epoch': 0.37}\n",
      "{'loss': 0.5912, 'grad_norm': 0.0897168442606926, 'learning_rate': 0.00015807029557109398, 'epoch': 0.37}\n",
      "{'loss': 0.6033, 'grad_norm': 0.08346230536699295, 'learning_rate': 0.0001573576436351046, 'epoch': 0.38}\n",
      "{'loss': 0.6206, 'grad_norm': 0.08081517368555069, 'learning_rate': 0.00015664062369248328, 'epoch': 0.38}\n",
      "{'loss': 0.591, 'grad_norm': 0.08316104114055634, 'learning_rate': 0.0001559192903470747, 'epoch': 0.38}\n",
      "{'loss': 0.5442, 'grad_norm': 0.08549479395151138, 'learning_rate': 0.0001551936985312058, 'epoch': 0.38}\n",
      "{'loss': 0.5301, 'grad_norm': 0.0870632454752922, 'learning_rate': 0.00015446390350150273, 'epoch': 0.39}\n",
      "{'loss': 0.544, 'grad_norm': 0.0850161761045456, 'learning_rate': 0.0001537299608346824, 'epoch': 0.39}\n",
      "{'loss': 0.543, 'grad_norm': 0.08457230031490326, 'learning_rate': 0.0001529919264233205, 'epoch': 0.39}\n",
      "{'loss': 0.5695, 'grad_norm': 0.08823264390230179, 'learning_rate': 0.0001522498564715949, 'epoch': 0.39}\n",
      "{'loss': 0.5695, 'grad_norm': 0.08264882117509842, 'learning_rate': 0.00015150380749100545, 'epoch': 0.4}\n",
      "{'loss': 0.5769, 'grad_norm': 0.09058960527181625, 'learning_rate': 0.00015075383629607042, 'epoch': 0.4}\n",
      "{'loss': 0.5584, 'grad_norm': 0.08464445918798447, 'learning_rate': 0.00015000000000000001, 'epoch': 0.4}\n",
      " 40%|███████████████▏                      | 800/2000 [57:03<1:19:57,  4.00s/it]\n",
      "***** Running Evaluation *****\n",
      "  Num examples: Unknown\n",
      "  Batch size = 16\n",
      "{'eval_loss': 0.6007864475250244, 'eval_runtime': 27.4279, 'eval_samples_per_second': 53.668, 'eval_steps_per_second': 3.354, 'epoch': 0.4}\n",
      "{'loss': 0.5653, 'grad_norm': 0.08724399656057358, 'learning_rate': 0.00014924235601034672, 'epoch': 0.4}\n",
      "{'loss': 0.5764, 'grad_norm': 0.10610764473676682, 'learning_rate': 0.00014848096202463372, 'epoch': 0.41}\n",
      "{'loss': 0.5863, 'grad_norm': 0.0920855849981308, 'learning_rate': 0.00014771587602596084, 'epoch': 0.41}\n",
      "{'loss': 0.5511, 'grad_norm': 0.08935829997062683, 'learning_rate': 0.00014694715627858908, 'epoch': 0.41}\n",
      "{'loss': 0.5788, 'grad_norm': 0.08164089918136597, 'learning_rate': 0.00014617486132350343, 'epoch': 0.41}\n",
      "{'loss': 0.591, 'grad_norm': 0.08427812904119492, 'learning_rate': 0.00014539904997395468, 'epoch': 0.41}\n",
      "{'loss': 0.5457, 'grad_norm': 0.08997779339551926, 'learning_rate': 0.00014461978131098088, 'epoch': 0.42}\n",
      "{'loss': 0.5742, 'grad_norm': 0.08972011506557465, 'learning_rate': 0.00014383711467890774, 'epoch': 0.42}\n",
      "{'loss': 0.5758, 'grad_norm': 0.09121856838464737, 'learning_rate': 0.00014305110968082952, 'epoch': 0.42}\n",
      "{'loss': 0.5906, 'grad_norm': 0.09059849381446838, 'learning_rate': 0.00014226182617406996, 'epoch': 0.42}\n",
      "{'loss': 0.5808, 'grad_norm': 0.09304126352071762, 'learning_rate': 0.00014146932426562392, 'epoch': 0.43}\n",
      "{'loss': 0.558, 'grad_norm': 0.08804154396057129, 'learning_rate': 0.00014067366430758004, 'epoch': 0.43}\n",
      "{'loss': 0.5956, 'grad_norm': 0.09386904537677765, 'learning_rate': 0.00013987490689252463, 'epoch': 0.43}\n",
      "{'loss': 0.5612, 'grad_norm': 0.08774534612894058, 'learning_rate': 0.00013907311284892736, 'epoch': 0.43}\n",
      "{'loss': 0.5995, 'grad_norm': 0.08660001307725906, 'learning_rate': 0.000138268343236509, 'epoch': 0.44}\n",
      "{'loss': 0.5835, 'grad_norm': 0.08281690627336502, 'learning_rate': 0.00013746065934159123, 'epoch': 0.44}\n",
      "{'loss': 0.5765, 'grad_norm': 0.08895985782146454, 'learning_rate': 0.00013665012267242974, 'epoch': 0.44}\n",
      "{'loss': 0.5535, 'grad_norm': 0.08906779438257217, 'learning_rate': 0.00013583679495453, 'epoch': 0.45}\n",
      "{'loss': 0.5339, 'grad_norm': 0.08768098801374435, 'learning_rate': 0.00013502073812594675, 'epoch': 0.45}\n",
      "{'loss': 0.5639, 'grad_norm': 0.08671879023313522, 'learning_rate': 0.00013420201433256689, 'epoch': 0.45}\n",
      " 45%|████████████████▏                   | 900/2000 [1:04:13<1:13:06,  3.99s/it]\n",
      "***** Running Evaluation *****\n",
      "  Num examples: Unknown\n",
      "  Batch size = 16\n",
      "{'eval_loss': 0.5910969972610474, 'eval_runtime': 27.4221, 'eval_samples_per_second': 53.679, 'eval_steps_per_second': 3.355, 'epoch': 0.45}\n",
      "{'loss': 0.5544, 'grad_norm': 0.09496685862541199, 'learning_rate': 0.0001333806859233771, 'epoch': 0.45}\n",
      "{'loss': 0.5452, 'grad_norm': 0.09727366268634796, 'learning_rate': 0.00013255681544571568, 'epoch': 0.46}\n",
      "{'loss': 0.5532, 'grad_norm': 0.08747061342000961, 'learning_rate': 0.00013173046564050924, 'epoch': 0.46}\n",
      "{'loss': 0.5207, 'grad_norm': 0.09103508293628693, 'learning_rate': 0.00013090169943749476, 'epoch': 0.46}\n",
      "{'loss': 0.5162, 'grad_norm': 0.08973684161901474, 'learning_rate': 0.00013007057995042732, 'epoch': 0.46}\n",
      "{'loss': 0.5056, 'grad_norm': 0.08825654536485672, 'learning_rate': 0.00012923717047227368, 'epoch': 0.47}\n",
      "{'loss': 0.5536, 'grad_norm': 0.09327245503664017, 'learning_rate': 0.00012840153447039228, 'epoch': 0.47}\n",
      "{'loss': 0.5514, 'grad_norm': 0.09571614116430283, 'learning_rate': 0.0001275637355816999, 'epoch': 0.47}\n",
      "{'loss': 0.5461, 'grad_norm': 0.09461065381765366, 'learning_rate': 0.00012672383760782568, 'epoch': 0.47}\n",
      "{'loss': 0.5421, 'grad_norm': 0.09444981068372726, 'learning_rate': 0.00012588190451025207, 'epoch': 0.47}\n",
      "{'loss': 0.5789, 'grad_norm': 0.10599493980407715, 'learning_rate': 0.00012503800040544416, 'epoch': 0.48}\n",
      "{'loss': 0.5964, 'grad_norm': 0.0882829949259758, 'learning_rate': 0.00012419218955996676, 'epoch': 0.48}\n",
      "{'loss': 0.5856, 'grad_norm': 0.09173424541950226, 'learning_rate': 0.00012334453638559057, 'epoch': 0.48}\n",
      "{'loss': 0.5397, 'grad_norm': 0.08677025139331818, 'learning_rate': 0.0001224951054343865, 'epoch': 0.48}\n",
      "{'loss': 0.5156, 'grad_norm': 0.0865790918469429, 'learning_rate': 0.00012164396139381029, 'epoch': 0.49}\n",
      "{'loss': 0.5377, 'grad_norm': 0.08314566314220428, 'learning_rate': 0.00012079116908177593, 'epoch': 0.49}\n",
      "{'loss': 0.5224, 'grad_norm': 0.10139483213424683, 'learning_rate': 0.00011993679344171973, 'epoch': 0.49}\n",
      "{'loss': 0.5297, 'grad_norm': 0.09137145429849625, 'learning_rate': 0.00011908089953765449, 'epoch': 0.49}\n",
      "{'loss': 0.4995, 'grad_norm': 0.09090246260166168, 'learning_rate': 0.00011822355254921478, 'epoch': 0.5}\n",
      "{'loss': 0.5433, 'grad_norm': 0.09456104040145874, 'learning_rate': 0.00011736481776669306, 'epoch': 0.5}\n",
      " 50%|█████████████████▌                 | 1000/2000 [1:11:24<1:10:31,  4.23s/it]\n",
      "***** Running Evaluation *****\n",
      "  Num examples: Unknown\n",
      "  Batch size = 16\n",
      "{'eval_loss': 0.5847486257553101, 'eval_runtime': 27.3877, 'eval_samples_per_second': 53.747, 'eval_steps_per_second': 3.359, 'epoch': 0.5}\n",
      " 50%|█████████████████▌                 | 1000/2000 [1:11:52<1:10:31,  4.23s/it]Saving model checkpoint to fim_llama/checkpoint-1000\n",
      "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--deepseek-ai--deepseek-coder-1.3b-instruct/snapshots/e063262dac8366fc1f28a4da0ff3c50ea66259ca/config.json\n",
      "Unrecognized keys in `rope_scaling` for 'rope_type'='linear': {'type'}\n",
      "Model config LlamaConfig {\n",
      "  \"architectures\": [\n",
      "    \"LlamaForCausalLM\"\n",
      "  ],\n",
      "  \"attention_bias\": false,\n",
      "  \"attention_dropout\": 0.0,\n",
      "  \"bos_token_id\": 32013,\n",
      "  \"eos_token_id\": 32021,\n",
      "  \"hidden_act\": \"silu\",\n",
      "  \"hidden_size\": 2048,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 5504,\n",
      "  \"max_position_embeddings\": 16384,\n",
      "  \"mlp_bias\": false,\n",
      "  \"model_type\": \"llama\",\n",
      "  \"num_attention_heads\": 16,\n",
      "  \"num_hidden_layers\": 24,\n",
      "  \"num_key_value_heads\": 16,\n",
      "  \"pretraining_tp\": 1,\n",
      "  \"rms_norm_eps\": 1e-06,\n",
      "  \"rope_scaling\": {\n",
      "    \"factor\": 4.0,\n",
      "    \"rope_type\": \"linear\",\n",
      "    \"type\": \"linear\"\n",
      "  },\n",
      "  \"rope_theta\": 100000,\n",
      "  \"tie_word_embeddings\": false,\n",
      "  \"torch_dtype\": \"bfloat16\",\n",
      "  \"transformers_version\": \"4.44.1\",\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 32256\n",
      "}\n",
      "\n",
      "{'loss': 0.5705, 'grad_norm': 0.10289566218852997, 'learning_rate': 0.00011650476058606777, 'epoch': 0.5}\n",
      "{'loss': 0.5249, 'grad_norm': 0.09769364446401596, 'learning_rate': 0.0001156434465040231, 'epoch': 0.51}\n",
      "{'loss': 0.5044, 'grad_norm': 0.09591488540172577, 'learning_rate': 0.00011478094111296109, 'epoch': 0.51}\n",
      "{'loss': 0.5319, 'grad_norm': 0.09418535232543945, 'learning_rate': 0.00011391731009600654, 'epoch': 0.51}\n",
      "{'loss': 0.5682, 'grad_norm': 0.09838850796222687, 'learning_rate': 0.00011305261922200519, 'epoch': 0.51}\n",
      "{'loss': 0.5647, 'grad_norm': 0.09282592684030533, 'learning_rate': 0.00011218693434051475, 'epoch': 0.52}\n",
      "{'loss': 0.5382, 'grad_norm': 0.09485801309347153, 'learning_rate': 0.0001113203213767907, 'epoch': 0.52}\n",
      "{'loss': 0.5248, 'grad_norm': 0.08749143034219742, 'learning_rate': 0.00011045284632676536, 'epoch': 0.52}\n",
      "{'loss': 0.5738, 'grad_norm': 0.0900052934885025, 'learning_rate': 0.00010958457525202241, 'epoch': 0.52}\n",
      "{'loss': 0.5498, 'grad_norm': 0.09491677582263947, 'learning_rate': 0.00010871557427476583, 'epoch': 0.53}\n",
      "{'loss': 0.5513, 'grad_norm': 0.10258976370096207, 'learning_rate': 0.0001078459095727845, 'epoch': 0.53}\n",
      "{'loss': 0.5293, 'grad_norm': 0.10348208993673325, 'learning_rate': 0.00010697564737441252, 'epoch': 0.53}\n",
      "{'loss': 0.5631, 'grad_norm': 0.10764230042695999, 'learning_rate': 0.00010610485395348571, 'epoch': 0.53}\n",
      "{'loss': 0.567, 'grad_norm': 0.09270712733268738, 'learning_rate': 0.0001052335956242944, 'epoch': 0.54}\n",
      "{'loss': 0.5701, 'grad_norm': 0.09114719182252884, 'learning_rate': 0.00010436193873653361, 'epoch': 0.54}\n",
      "{'loss': 0.5704, 'grad_norm': 0.10268360376358032, 'learning_rate': 0.00010348994967025012, 'epoch': 0.54}\n",
      "{'loss': 0.5306, 'grad_norm': 0.09275121241807938, 'learning_rate': 0.00010261769483078733, 'epoch': 0.54}\n",
      "{'loss': 0.5467, 'grad_norm': 0.10746346414089203, 'learning_rate': 0.00010174524064372837, 'epoch': 0.55}\n",
      "{'loss': 0.5336, 'grad_norm': 0.09254515171051025, 'learning_rate': 0.0001008726535498374, 'epoch': 0.55}\n",
      "{'loss': 0.5717, 'grad_norm': 0.09638430923223495, 'learning_rate': 0.0001, 'epoch': 0.55}\n",
      " 55%|███████████████████▎               | 1100/2000 [1:18:38<1:00:36,  4.04s/it]\n",
      "***** Running Evaluation *****\n",
      "  Num examples: Unknown\n",
      "  Batch size = 16\n",
      "{'eval_loss': 0.5790347456932068, 'eval_runtime': 27.3405, 'eval_samples_per_second': 53.839, 'eval_steps_per_second': 3.365, 'epoch': 0.55}\n",
      "{'loss': 0.578, 'grad_norm': 0.09018813073635101, 'learning_rate': 9.912734645016263e-05, 'epoch': 0.55}\n",
      "{'loss': 0.5587, 'grad_norm': 0.09178995341062546, 'learning_rate': 9.825475935627165e-05, 'epoch': 0.56}\n",
      "{'loss': 0.4994, 'grad_norm': 0.10171325504779816, 'learning_rate': 9.73823051692127e-05, 'epoch': 0.56}\n",
      "{'loss': 0.4915, 'grad_norm': 0.09648846834897995, 'learning_rate': 9.651005032974994e-05, 'epoch': 0.56}\n",
      "{'loss': 0.504, 'grad_norm': 0.10033725202083588, 'learning_rate': 9.563806126346642e-05, 'epoch': 0.56}\n",
      "{'loss': 0.5064, 'grad_norm': 0.09371436387300491, 'learning_rate': 9.476640437570562e-05, 'epoch': 0.56}\n",
      "{'loss': 0.5467, 'grad_norm': 0.09619645774364471, 'learning_rate': 9.38951460465143e-05, 'epoch': 0.57}\n",
      "{'loss': 0.4829, 'grad_norm': 0.09484288841485977, 'learning_rate': 9.302435262558747e-05, 'epoch': 0.57}\n",
      "{'loss': 0.5096, 'grad_norm': 0.09577333927154541, 'learning_rate': 9.215409042721552e-05, 'epoch': 0.57}\n",
      "{'loss': 0.5421, 'grad_norm': 0.09283590316772461, 'learning_rate': 9.128442572523417e-05, 'epoch': 0.57}\n",
      "{'loss': 0.5406, 'grad_norm': 0.09694782644510269, 'learning_rate': 9.04154247479776e-05, 'epoch': 0.58}\n",
      "{'loss': 0.537, 'grad_norm': 0.09547742456197739, 'learning_rate': 8.954715367323468e-05, 'epoch': 0.58}\n",
      "{'loss': 0.5362, 'grad_norm': 0.10363823175430298, 'learning_rate': 8.867967862320934e-05, 'epoch': 0.58}\n",
      "{'loss': 0.5336, 'grad_norm': 0.09808262437582016, 'learning_rate': 8.781306565948528e-05, 'epoch': 0.58}\n",
      "{'loss': 0.5593, 'grad_norm': 0.09522387385368347, 'learning_rate': 8.694738077799488e-05, 'epoch': 0.59}\n",
      "{'loss': 0.5433, 'grad_norm': 0.08995317667722702, 'learning_rate': 8.608268990399349e-05, 'epoch': 0.59}\n",
      "{'loss': 0.5235, 'grad_norm': 0.09353280067443848, 'learning_rate': 8.521905888703893e-05, 'epoch': 0.59}\n",
      "{'loss': 0.5131, 'grad_norm': 0.09289946407079697, 'learning_rate': 8.435655349597689e-05, 'epoch': 0.59}\n",
      "{'loss': 0.5182, 'grad_norm': 0.09709266573190689, 'learning_rate': 8.349523941393224e-05, 'epoch': 0.6}\n",
      "{'loss': 0.4931, 'grad_norm': 0.08833959698677063, 'learning_rate': 8.263518223330697e-05, 'epoch': 0.6}\n",
      " 60%|██████████████████████▏              | 1200/2000 [1:25:48<53:22,  4.00s/it]\n",
      "***** Running Evaluation *****\n",
      "  Num examples: Unknown\n",
      "  Batch size = 16\n",
      "{'eval_loss': 0.5747049450874329, 'eval_runtime': 27.3882, 'eval_samples_per_second': 53.746, 'eval_steps_per_second': 3.359, 'epoch': 0.6}\n",
      "{'loss': 0.5014, 'grad_norm': 0.09783495962619781, 'learning_rate': 8.177644745078526e-05, 'epoch': 0.6}\n",
      "{'loss': 0.4857, 'grad_norm': 0.09421593695878983, 'learning_rate': 8.091910046234552e-05, 'epoch': 0.6}\n",
      "{'loss': 0.5074, 'grad_norm': 0.10091498494148254, 'learning_rate': 8.00632065582803e-05, 'epoch': 0.61}\n",
      "{'loss': 0.5137, 'grad_norm': 0.10100143402814865, 'learning_rate': 7.920883091822408e-05, 'epoch': 0.61}\n",
      "{'loss': 0.5061, 'grad_norm': 0.09478582441806793, 'learning_rate': 7.835603860618972e-05, 'epoch': 0.61}\n",
      "{'loss': 0.5272, 'grad_norm': 0.09374094754457474, 'learning_rate': 7.750489456561352e-05, 'epoch': 0.61}\n",
      "{'loss': 0.5253, 'grad_norm': 0.10347336530685425, 'learning_rate': 7.66554636144095e-05, 'epoch': 0.62}\n",
      "{'loss': 0.5588, 'grad_norm': 0.14736586809158325, 'learning_rate': 7.580781044003324e-05, 'epoch': 0.62}\n",
      "{'loss': 0.5503, 'grad_norm': 0.10534188151359558, 'learning_rate': 7.496199959455584e-05, 'epoch': 0.62}\n",
      "{'loss': 0.5599, 'grad_norm': 0.09792864322662354, 'learning_rate': 7.411809548974792e-05, 'epoch': 0.62}\n",
      "{'loss': 0.5431, 'grad_norm': 0.11646681278944016, 'learning_rate': 7.327616239217431e-05, 'epoch': 0.63}\n",
      "{'loss': 0.5304, 'grad_norm': 0.094825878739357, 'learning_rate': 7.243626441830009e-05, 'epoch': 0.63}\n",
      "{'loss': 0.5375, 'grad_norm': 0.09488825500011444, 'learning_rate': 7.159846552960774e-05, 'epoch': 0.63}\n",
      "{'loss': 0.5205, 'grad_norm': 0.09067409485578537, 'learning_rate': 7.076282952772633e-05, 'epoch': 0.64}\n",
      "{'loss': 0.5304, 'grad_norm': 0.09226901084184647, 'learning_rate': 6.992942004957271e-05, 'epoch': 0.64}\n",
      "{'loss': 0.5161, 'grad_norm': 0.10008667409420013, 'learning_rate': 6.909830056250527e-05, 'epoch': 0.64}\n",
      "{'loss': 0.5488, 'grad_norm': 0.09680334478616714, 'learning_rate': 6.826953435949081e-05, 'epoch': 0.64}\n",
      "{'loss': 0.5268, 'grad_norm': 0.101612389087677, 'learning_rate': 6.744318455428436e-05, 'epoch': 0.65}\n",
      "{'loss': 0.5373, 'grad_norm': 0.0983649268746376, 'learning_rate': 6.661931407662292e-05, 'epoch': 0.65}\n",
      "{'loss': 0.5338, 'grad_norm': 0.09590578079223633, 'learning_rate': 6.579798566743314e-05, 'epoch': 0.65}\n",
      " 65%|████████████████████████             | 1300/2000 [1:32:59<46:34,  3.99s/it]\n",
      "***** Running Evaluation *****\n",
      "  Num examples: Unknown\n",
      "  Batch size = 16\n",
      "{'eval_loss': 0.5708640813827515, 'eval_runtime': 27.4628, 'eval_samples_per_second': 53.6, 'eval_steps_per_second': 3.35, 'epoch': 0.65}\n",
      "{'loss': 0.5465, 'grad_norm': 0.09611256420612335, 'learning_rate': 6.497926187405326e-05, 'epoch': 0.65}\n",
      "{'loss': 0.5703, 'grad_norm': 0.10238052904605865, 'learning_rate': 6.416320504546997e-05, 'epoch': 0.66}\n",
      "{'loss': 0.5534, 'grad_norm': 0.09668735414743423, 'learning_rate': 6.334987732757029e-05, 'epoch': 0.66}\n",
      "{'loss': 0.5511, 'grad_norm': 0.09508249908685684, 'learning_rate': 6.25393406584088e-05, 'epoch': 0.66}\n",
      "{'loss': 0.4828, 'grad_norm': 0.09814571589231491, 'learning_rate': 6.173165676349103e-05, 'epoch': 0.66}\n",
      "{'loss': 0.4784, 'grad_norm': 0.09756369888782501, 'learning_rate': 6.092688715107264e-05, 'epoch': 0.67}\n",
      "{'loss': 0.472, 'grad_norm': 0.10181044042110443, 'learning_rate': 6.012509310747538e-05, 'epoch': 0.67}\n",
      "{'loss': 0.5222, 'grad_norm': 0.09295830875635147, 'learning_rate': 5.9326335692419995e-05, 'epoch': 0.67}\n",
      "{'loss': 0.5092, 'grad_norm': 0.10007476061582565, 'learning_rate': 5.853067573437612e-05, 'epoch': 0.67}\n",
      "{'loss': 0.5117, 'grad_norm': 0.09404541552066803, 'learning_rate': 5.773817382593008e-05, 'epoch': 0.68}\n",
      "{'loss': 0.5148, 'grad_norm': 0.09672043472528458, 'learning_rate': 5.694889031917047e-05, 'epoch': 0.68}\n",
      "{'loss': 0.5207, 'grad_norm': 0.09787816554307938, 'learning_rate': 5.616288532109225e-05, 'epoch': 0.68}\n",
      "{'loss': 0.5097, 'grad_norm': 0.09687553346157074, 'learning_rate': 5.5380218689019125e-05, 'epoch': 0.68}\n",
      "{'loss': 0.509, 'grad_norm': 0.10128545016050339, 'learning_rate': 5.4600950026045326e-05, 'epoch': 0.69}\n",
      "{'loss': 0.5222, 'grad_norm': 0.10490205138921738, 'learning_rate': 5.382513867649663e-05, 'epoch': 0.69}\n",
      "{'loss': 0.5336, 'grad_norm': 0.09976710379123688, 'learning_rate': 5.305284372141095e-05, 'epoch': 0.69}\n",
      "{'loss': 0.5115, 'grad_norm': 0.09301279485225677, 'learning_rate': 5.2284123974039154e-05, 'epoch': 0.69}\n",
      "{'loss': 0.5555, 'grad_norm': 0.1000623106956482, 'learning_rate': 5.15190379753663e-05, 'epoch': 0.69}\n",
      "{'loss': 0.5754, 'grad_norm': 0.1027800515294075, 'learning_rate': 5.07576439896533e-05, 'epoch': 0.7}\n",
      "{'loss': 0.5168, 'grad_norm': 0.10245217382907867, 'learning_rate': 5.000000000000002e-05, 'epoch': 0.7}\n",
      " 70%|█████████████████████████▉           | 1400/2000 [1:40:09<39:51,  3.99s/it]\n",
      "***** Running Evaluation *****\n",
      "  Num examples: Unknown\n",
      "  Batch size = 16\n",
      "{'eval_loss': 0.568744421005249, 'eval_runtime': 27.3949, 'eval_samples_per_second': 53.733, 'eval_steps_per_second': 3.358, 'epoch': 0.7}\n",
      "{'loss': 0.4778, 'grad_norm': 0.09361349046230316, 'learning_rate': 4.924616370392961e-05, 'epoch': 0.7}\n",
      "{'loss': 0.4623, 'grad_norm': 0.1013522520661354, 'learning_rate': 4.8496192508994576e-05, 'epoch': 0.7}\n",
      "{'loss': 0.4708, 'grad_norm': 0.09926623106002808, 'learning_rate': 4.7750143528405126e-05, 'epoch': 0.71}\n",
      "{'loss': 0.4277, 'grad_norm': 0.10308998823165894, 'learning_rate': 4.700807357667952e-05, 'epoch': 0.71}\n",
      "{'loss': 0.4862, 'grad_norm': 0.09574003517627716, 'learning_rate': 4.6270039165317605e-05, 'epoch': 0.71}\n",
      "{'loss': 0.4893, 'grad_norm': 0.09930822253227234, 'learning_rate': 4.5536096498497295e-05, 'epoch': 0.71}\n",
      "{'loss': 0.5752, 'grad_norm': 0.09586738795042038, 'learning_rate': 4.480630146879419e-05, 'epoch': 0.72}\n",
      "{'loss': 0.5445, 'grad_norm': 0.10016771405935287, 'learning_rate': 4.4080709652925336e-05, 'epoch': 0.72}\n",
      "{'loss': 0.5072, 'grad_norm': 0.10638781636953354, 'learning_rate': 4.335937630751674e-05, 'epoch': 0.72}\n",
      "{'loss': 0.5177, 'grad_norm': 0.10528813302516937, 'learning_rate': 4.264235636489542e-05, 'epoch': 0.72}\n",
      "{'loss': 0.4985, 'grad_norm': 0.1065158098936081, 'learning_rate': 4.1929704428906026e-05, 'epoch': 0.73}\n",
      "{'loss': 0.5351, 'grad_norm': 0.09731660038232803, 'learning_rate': 4.12214747707527e-05, 'epoch': 0.73}\n",
      "{'loss': 0.5214, 'grad_norm': 0.09655523300170898, 'learning_rate': 4.0517721324865884e-05, 'epoch': 0.73}\n",
      "{'loss': 0.5244, 'grad_norm': 0.10056781023740768, 'learning_rate': 3.981849768479517e-05, 'epoch': 0.73}\n",
      "{'loss': 0.4769, 'grad_norm': 0.09545408934354782, 'learning_rate': 3.9123857099127936e-05, 'epoch': 0.74}\n",
      "{'loss': 0.5194, 'grad_norm': 0.09666460752487183, 'learning_rate': 3.843385246743417e-05, 'epoch': 0.74}\n",
      "{'loss': 0.5283, 'grad_norm': 0.10808345675468445, 'learning_rate': 3.774853633623806e-05, 'epoch': 0.74}\n",
      "{'loss': 0.5344, 'grad_norm': 0.10320992022752762, 'learning_rate': 3.7067960895016275e-05, 'epoch': 0.74}\n",
      "{'loss': 0.5184, 'grad_norm': 0.10091155767440796, 'learning_rate': 3.6392177972223594e-05, 'epoch': 0.75}\n",
      "{'loss': 0.5527, 'grad_norm': 0.09729717671871185, 'learning_rate': 3.5721239031346066e-05, 'epoch': 0.75}\n",
      " 75%|███████████████████████████▊         | 1500/2000 [1:47:21<34:38,  4.16s/it]\n",
      "***** Running Evaluation *****\n",
      "  Num examples: Unknown\n",
      "  Batch size = 16\n",
      "{'eval_loss': 0.5662122964859009, 'eval_runtime': 27.3736, 'eval_samples_per_second': 53.774, 'eval_steps_per_second': 3.361, 'epoch': 0.75}\n",
      " 75%|███████████████████████████▊         | 1500/2000 [1:47:48<34:38,  4.16s/it]Saving model checkpoint to fim_llama/checkpoint-1500\n",
      "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--deepseek-ai--deepseek-coder-1.3b-instruct/snapshots/e063262dac8366fc1f28a4da0ff3c50ea66259ca/config.json\n",
      "Unrecognized keys in `rope_scaling` for 'rope_type'='linear': {'type'}\n",
      "Model config LlamaConfig {\n",
      "  \"architectures\": [\n",
      "    \"LlamaForCausalLM\"\n",
      "  ],\n",
      "  \"attention_bias\": false,\n",
      "  \"attention_dropout\": 0.0,\n",
      "  \"bos_token_id\": 32013,\n",
      "  \"eos_token_id\": 32021,\n",
      "  \"hidden_act\": \"silu\",\n",
      "  \"hidden_size\": 2048,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 5504,\n",
      "  \"max_position_embeddings\": 16384,\n",
      "  \"mlp_bias\": false,\n",
      "  \"model_type\": \"llama\",\n",
      "  \"num_attention_heads\": 16,\n",
      "  \"num_hidden_layers\": 24,\n",
      "  \"num_key_value_heads\": 16,\n",
      "  \"pretraining_tp\": 1,\n",
      "  \"rms_norm_eps\": 1e-06,\n",
      "  \"rope_scaling\": {\n",
      "    \"factor\": 4.0,\n",
      "    \"rope_type\": \"linear\",\n",
      "    \"type\": \"linear\"\n",
      "  },\n",
      "  \"rope_theta\": 100000,\n",
      "  \"tie_word_embeddings\": false,\n",
      "  \"torch_dtype\": \"bfloat16\",\n",
      "  \"transformers_version\": \"4.44.1\",\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 32256\n",
      "}\n",
      "\n",
      "{'loss': 0.5396, 'grad_norm': 0.0967901200056076, 'learning_rate': 3.5055195166981645e-05, 'epoch': 0.75}\n",
      "{'loss': 0.5271, 'grad_norm': 0.10106580704450607, 'learning_rate': 3.439409710094929e-05, 'epoch': 0.76}\n",
      "{'loss': 0.5567, 'grad_norm': 0.10108551383018494, 'learning_rate': 3.373799517842627e-05, 'epoch': 0.76}\n",
      "{'loss': 0.5354, 'grad_norm': 0.0979294553399086, 'learning_rate': 3.308693936411421e-05, 'epoch': 0.76}\n",
      "{'loss': 0.5917, 'grad_norm': 0.09627845883369446, 'learning_rate': 3.244097923843398e-05, 'epoch': 0.76}\n",
      "{'loss': 0.5755, 'grad_norm': 0.0967583879828453, 'learning_rate': 3.1800163993750166e-05, 'epoch': 0.77}\n",
      "{'loss': 0.4584, 'grad_norm': 0.10873382538557053, 'learning_rate': 3.116454243062459e-05, 'epoch': 0.77}\n",
      "{'loss': 0.4104, 'grad_norm': 0.10114021599292755, 'learning_rate': 3.053416295410026e-05, 'epoch': 0.77}\n",
      "{'loss': 0.441, 'grad_norm': 0.09938845783472061, 'learning_rate': 2.9909073570014912e-05, 'epoch': 0.77}\n",
      "{'loss': 0.49, 'grad_norm': 0.10098407417535782, 'learning_rate': 2.9289321881345254e-05, 'epoch': 0.78}\n",
      "{'loss': 0.5067, 'grad_norm': 0.10960653424263, 'learning_rate': 2.8674955084581857e-05, 'epoch': 0.78}\n",
      "{'loss': 0.5196, 'grad_norm': 0.09750140458345413, 'learning_rate': 2.8066019966134904e-05, 'epoch': 0.78}\n",
      "{'loss': 0.5176, 'grad_norm': 0.10484049469232559, 'learning_rate': 2.746256289877126e-05, 'epoch': 0.78}\n",
      "{'loss': 0.4994, 'grad_norm': 0.10100133717060089, 'learning_rate': 2.6864629838082956e-05, 'epoch': 0.79}\n",
      "{'loss': 0.5052, 'grad_norm': 0.09987310320138931, 'learning_rate': 2.6272266318987603e-05, 'epoch': 0.79}\n",
      "{'loss': 0.4939, 'grad_norm': 0.10390567034482956, 'learning_rate': 2.5685517452260567e-05, 'epoch': 0.79}\n",
      "{'loss': 0.4946, 'grad_norm': 0.09876112639904022, 'learning_rate': 2.5104427921099782e-05, 'epoch': 0.79}\n",
      "{'loss': 0.508, 'grad_norm': 0.09935278445482254, 'learning_rate': 2.45290419777228e-05, 'epoch': 0.8}\n",
      "{'loss': 0.4891, 'grad_norm': 0.10090654343366623, 'learning_rate': 2.3959403439996907e-05, 'epoch': 0.8}\n",
      "{'loss': 0.5538, 'grad_norm': 0.09791342914104462, 'learning_rate': 2.339555568810221e-05, 'epoch': 0.8}\n",
      " 80%|█████████████████████████████▌       | 1600/2000 [1:54:34<26:56,  4.04s/it]\n",
      "***** Running Evaluation *****\n",
      "  Num examples: Unknown\n",
      "  Batch size = 16\n",
      "{'eval_loss': 0.5642468929290771, 'eval_runtime': 27.4661, 'eval_samples_per_second': 53.593, 'eval_steps_per_second': 3.35, 'epoch': 0.8}\n",
      "{'loss': 0.5358, 'grad_norm': 0.09601961076259613, 'learning_rate': 2.2837541661228025e-05, 'epoch': 0.8}\n",
      "{'loss': 0.552, 'grad_norm': 0.10148713737726212, 'learning_rate': 2.2285403854302912e-05, 'epoch': 0.81}\n",
      "{'loss': 0.4985, 'grad_norm': 0.10025947540998459, 'learning_rate': 2.173918431475861e-05, 'epoch': 0.81}\n",
      "{'loss': 0.4499, 'grad_norm': 0.09809647500514984, 'learning_rate': 2.119892463932781e-05, 'epoch': 0.81}\n",
      "{'loss': 0.4662, 'grad_norm': 0.10345581918954849, 'learning_rate': 2.0664665970876496e-05, 'epoch': 0.81}\n",
      "{'loss': 0.4565, 'grad_norm': 0.09733357280492783, 'learning_rate': 2.013644899527074e-05, 'epoch': 0.81}\n",
      "{'loss': 0.5037, 'grad_norm': 0.10066165030002594, 'learning_rate': 1.9614313938278272e-05, 'epoch': 0.82}\n",
      "{'loss': 0.49, 'grad_norm': 0.10075569897890091, 'learning_rate': 1.9098300562505266e-05, 'epoch': 0.82}\n",
      "{'loss': 0.4874, 'grad_norm': 0.11035048216581345, 'learning_rate': 1.858844816436809e-05, 'epoch': 0.82}\n",
      "{'loss': 0.5243, 'grad_norm': 0.10338343679904938, 'learning_rate': 1.808479557110081e-05, 'epoch': 0.82}\n",
      "{'loss': 0.5121, 'grad_norm': 0.10518023371696472, 'learning_rate': 1.7587381137798432e-05, 'epoch': 0.83}\n",
      "{'loss': 0.5121, 'grad_norm': 0.1024111732840538, 'learning_rate': 1.7096242744495837e-05, 'epoch': 0.83}\n",
      "{'loss': 0.5086, 'grad_norm': 0.0998847633600235, 'learning_rate': 1.661141779328319e-05, 'epoch': 0.83}\n",
      "{'loss': 0.5236, 'grad_norm': 0.10017276555299759, 'learning_rate': 1.6132943205457606e-05, 'epoch': 0.83}\n",
      "{'loss': 0.4846, 'grad_norm': 0.09713669121265411, 'learning_rate': 1.566085541871145e-05, 'epoch': 0.84}\n",
      "{'loss': 0.4909, 'grad_norm': 0.10252225399017334, 'learning_rate': 1.5195190384357404e-05, 'epoch': 0.84}\n",
      "{'loss': 0.5242, 'grad_norm': 0.10146605223417282, 'learning_rate': 1.4735983564590783e-05, 'epoch': 0.84}\n",
      "{'loss': 0.5184, 'grad_norm': 0.10046922415494919, 'learning_rate': 1.4283269929788779e-05, 'epoch': 0.84}\n",
      "{'loss': 0.5053, 'grad_norm': 0.100851871073246, 'learning_rate': 1.3837083955847418e-05, 'epoch': 0.85}\n",
      "{'loss': 0.5105, 'grad_norm': 0.09544802457094193, 'learning_rate': 1.339745962155613e-05, 'epoch': 0.85}\n",
      " 85%|███████████████████████████████▍     | 1700/2000 [2:01:44<19:57,  3.99s/it]\n",
      "***** Running Evaluation *****\n",
      "  Num examples: Unknown\n",
      "  Batch size = 16\n",
      "{'eval_loss': 0.5637705326080322, 'eval_runtime': 27.392, 'eval_samples_per_second': 53.738, 'eval_steps_per_second': 3.359, 'epoch': 0.85}\n",
      "{'loss': 0.509, 'grad_norm': 0.10336119681596756, 'learning_rate': 1.296443040601003e-05, 'epoch': 0.85}\n",
      "{'loss': 0.5605, 'grad_norm': 0.09586534649133682, 'learning_rate': 1.2538029286060426e-05, 'epoch': 0.85}\n",
      "{'loss': 0.52, 'grad_norm': 0.1037215068936348, 'learning_rate': 1.2118288733803473e-05, 'epoch': 0.86}\n",
      "{'loss': 0.5479, 'grad_norm': 0.09067007899284363, 'learning_rate': 1.1705240714107302e-05, 'epoch': 0.86}\n",
      "{'loss': 0.5529, 'grad_norm': 0.09873151779174805, 'learning_rate': 1.129891668217783e-05, 'epoch': 0.86}\n",
      "{'loss': 0.5612, 'grad_norm': 0.09816864132881165, 'learning_rate': 1.0899347581163221e-05, 'epoch': 0.86}\n",
      "{'loss': 0.5817, 'grad_norm': 0.09817475825548172, 'learning_rate': 1.0506563839797501e-05, 'epoch': 0.87}\n",
      "{'loss': 0.5476, 'grad_norm': 0.10102558135986328, 'learning_rate': 1.0120595370083318e-05, 'epoch': 0.87}\n",
      "{'loss': 0.4361, 'grad_norm': 0.10010476410388947, 'learning_rate': 9.74147156501396e-06, 'epoch': 0.87}\n",
      "{'loss': 0.4433, 'grad_norm': 0.10134933143854141, 'learning_rate': 9.369221296335006e-06, 'epoch': 0.88}\n",
      "{'loss': 0.393, 'grad_norm': 0.10211031883955002, 'learning_rate': 9.00387291234569e-06, 'epoch': 0.88}\n",
      "{'loss': 0.5003, 'grad_norm': 0.11144755780696869, 'learning_rate': 8.645454235739903e-06, 'epoch': 0.88}\n",
      "{'loss': 0.5293, 'grad_norm': 0.09581740200519562, 'learning_rate': 8.293992561487596e-06, 'epoch': 0.88}\n",
      "{'loss': 0.4995, 'grad_norm': 0.09995877742767334, 'learning_rate': 7.949514654755962e-06, 'epoch': 0.89}\n",
      "{'loss': 0.4982, 'grad_norm': 0.09831967204809189, 'learning_rate': 7.612046748871327e-06, 'epoch': 0.89}\n",
      "{'loss': 0.498, 'grad_norm': 0.09853954613208771, 'learning_rate': 7.281614543321269e-06, 'epoch': 0.89}\n",
      "{'loss': 0.5021, 'grad_norm': 0.10075218975543976, 'learning_rate': 6.958243201797554e-06, 'epoch': 0.89}\n",
      "{'loss': 0.5204, 'grad_norm': 0.10033264756202698, 'learning_rate': 6.6419573502798374e-06, 'epoch': 0.9}\n",
      "{'eval_loss': 0.5632106065750122, 'eval_runtime': 27.3887, 'eval_samples_per_second': 53.745, 'eval_steps_per_second': 3.359, 'epoch': 0.9}\n",
      "{'loss': 0.512, 'grad_norm': 0.09812013059854507, 'learning_rate': 5.735850890782157e-06, 'epoch': 0.9}\n",
      "{'loss': 0.5198, 'grad_norm': 0.1010688990354538, 'learning_rate': 5.448142440068316e-06, 'epoch': 0.91}\n",
      "{'loss': 0.5416, 'grad_norm': 0.10282190144062042, 'learning_rate': 5.167634479380068e-06, 'epoch': 0.91}\n",
      "{'loss': 0.522, 'grad_norm': 0.10158905386924744, 'learning_rate': 4.8943483704846475e-06, 'epoch': 0.91}\n",
      "{'loss': 0.4693, 'grad_norm': 0.10576950013637543, 'learning_rate': 4.628304925177318e-06, 'epoch': 0.91}\n",
      "{'loss': 0.4563, 'grad_norm': 0.1012519896030426, 'learning_rate': 4.369524403696457e-06, 'epoch': 0.92}\n",
      "{'loss': 0.4523, 'grad_norm': 0.09914936125278473, 'learning_rate': 4.118026513180695e-06, 'epoch': 0.92}\n",
      "{'loss': 0.4781, 'grad_norm': 0.1119132712483406, 'learning_rate': 3.873830406168111e-06, 'epoch': 0.92}\n",
      "{'loss': 0.4801, 'grad_norm': 0.09970618039369583, 'learning_rate': 3.6369546791377052e-06, 'epoch': 0.92}\n",
      "{'loss': 0.5105, 'grad_norm': 0.10010836273431778, 'learning_rate': 3.40741737109318e-06, 'epoch': 0.93}\n",
      "{'loss': 0.5119, 'grad_norm': 0.10445805639028549, 'learning_rate': 3.1852359621892367e-06, 'epoch': 0.93}\n",
      "{'loss': 0.5038, 'grad_norm': 0.10609171539545059, 'learning_rate': 2.970427372400353e-06, 'epoch': 0.93}\n",
      "{'loss': 0.5216, 'grad_norm': 0.11233904957771301, 'learning_rate': 2.7630079602323442e-06, 'epoch': 0.93}\n",
      "{'loss': 0.5177, 'grad_norm': 0.0962633416056633, 'learning_rate': 2.5629935214764865e-06, 'epoch': 0.94}\n",
      "{'loss': 0.4892, 'grad_norm': 0.10372447222471237, 'learning_rate': 2.3703992880066638e-06, 'epoch': 0.94}\n",
      "{'loss': 0.4943, 'grad_norm': 0.09837695956230164, 'learning_rate': 2.1852399266194314e-06, 'epoch': 0.94}\n",
      "{'loss': 0.4918, 'grad_norm': 0.09740300476551056, 'learning_rate': 2.0075295379170412e-06, 'epoch': 0.94}\n",
      "{'loss': 0.5272, 'grad_norm': 0.10358048975467682, 'learning_rate': 1.8372816552336026e-06, 'epoch': 0.94}\n",
      "{'loss': 0.5251, 'grad_norm': 0.09530334919691086, 'learning_rate': 1.6745092436045494e-06, 'epoch': 0.95}\n",
      "{'loss': 0.4867, 'grad_norm': 0.09998327493667603, 'learning_rate': 1.5192246987791981e-06, 'epoch': 0.95}\n",
      " 95%|███████████████████████████████████▏ | 1900/2000 [2:16:05<06:38,  3.98s/it]\n",
      "***** Running Evaluation *****\n",
      "  Num examples: Unknown\n",
      "  Batch size = 16\n",
      "{'eval_loss': 0.5630667209625244, 'eval_runtime': 27.4759, 'eval_samples_per_second': 53.574, 'eval_steps_per_second': 3.348, 'epoch': 0.95}\n",
      "{'loss': 0.5001, 'grad_norm': 0.0997118204832077, 'learning_rate': 1.3714398462768563e-06, 'epoch': 0.95}\n",
      "{'loss': 0.4957, 'grad_norm': 0.10226450115442276, 'learning_rate': 1.231165940486234e-06, 'epoch': 0.95}\n",
      "{'loss': 0.515, 'grad_norm': 0.09989296644926071, 'learning_rate': 1.0984136638083177e-06, 'epoch': 0.96}\n",
      "{'loss': 0.5734, 'grad_norm': 0.10892973840236664, 'learning_rate': 9.731931258429638e-07, 'epoch': 0.96}\n",
      "{'loss': 0.534, 'grad_norm': 0.10010427236557007, 'learning_rate': 8.555138626189618e-07, 'epoch': 0.96}\n",
      "{'loss': 0.5032, 'grad_norm': 0.10048697888851166, 'learning_rate': 7.453848358678017e-07, 'epoch': 0.96}\n",
      "{'loss': 0.5425, 'grad_norm': 0.10035979002714157, 'learning_rate': 6.428144323412544e-07, 'epoch': 0.97}\n",
      "{'loss': 0.5834, 'grad_norm': 0.10086975246667862, 'learning_rate': 5.478104631726711e-07, 'epoch': 0.97}\n",
      "{'loss': 0.5695, 'grad_norm': 0.10318192839622498, 'learning_rate': 4.6038016328211476e-07, 'epoch': 0.97}\n",
      "{'loss': 0.5355, 'grad_norm': 0.10292799025774002, 'learning_rate': 3.805301908254455e-07, 'epoch': 0.97}\n",
      "{'loss': 0.4005, 'grad_norm': 0.10501635074615479, 'learning_rate': 3.0826662668720364e-07, 'epoch': 0.98}\n",
      "{'loss': 0.4282, 'grad_norm': 0.10128708183765411, 'learning_rate': 2.4359497401758024e-07, 'epoch': 0.98}\n",
      "{'loss': 0.4397, 'grad_norm': 0.1050596684217453, 'learning_rate': 1.86520157813308e-07, 'epoch': 0.98}\n",
      "{'loss': 0.5077, 'grad_norm': 0.09914371371269226, 'learning_rate': 1.3704652454261668e-07, 'epoch': 0.98}\n",
      "{'loss': 0.5353, 'grad_norm': 0.09811881929636002, 'learning_rate': 9.517784181422019e-08, 'epoch': 0.99}\n",
      "{'loss': 0.5035, 'grad_norm': 0.09990497678518295, 'learning_rate': 6.09172980904238e-08, 'epoch': 0.99}\n",
      "{'loss': 0.4834, 'grad_norm': 0.1039889007806778, 'learning_rate': 3.4267502444274015e-08, 'epoch': 0.99}\n",
      "{'loss': 0.4799, 'grad_norm': 0.09593120962381363, 'learning_rate': 1.5230484360873044e-08, 'epoch': 0.99}\n",
      "{'loss': 0.5203, 'grad_norm': 0.09616655856370926, 'learning_rate': 3.807693582869032e-09, 'epoch': 1.0}\n",
      "{'loss': 0.4668, 'grad_norm': 0.0990811362862587, 'learning_rate': 0.0, 'epoch': 1.0}\n",
      "100%|█████████████████████████████████████| 2000/2000 [2:23:17<00:00,  4.10s/it]\n",
      "***** Running Evaluation *****\n",
      "  Num examples: Unknown\n",
      "  Batch size = 16\n",
      "{'eval_loss': 0.5631296634674072, 'eval_runtime': 27.5074, 'eval_samples_per_second': 53.513, 'eval_steps_per_second': 3.345, 'epoch': 1.0}\n",
      "100%|█████████████████████████████████████| 2000/2000 [2:23:45<00:00,  4.10s/it]Saving model checkpoint to fim_llama/checkpoint-2000\n",
      "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--deepseek-ai--deepseek-coder-1.3b-instruct/snapshots/e063262dac8366fc1f28a4da0ff3c50ea66259ca/config.json\n",
      "Unrecognized keys in `rope_scaling` for 'rope_type'='linear': {'type'}\n",
      "Model config LlamaConfig {\n",
      "  \"architectures\": [\n",
      "    \"LlamaForCausalLM\"\n",
      "  ],\n",
      "  \"attention_bias\": false,\n",
      "  \"attention_dropout\": 0.0,\n",
      "  \"bos_token_id\": 32013,\n",
      "  \"eos_token_id\": 32021,\n",
      "  \"hidden_act\": \"silu\",\n",
      "  \"hidden_size\": 2048,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 5504,\n",
      "  \"max_position_embeddings\": 16384,\n",
      "  \"mlp_bias\": false,\n",
      "  \"model_type\": \"llama\",\n",
      "  \"num_attention_heads\": 16,\n",
      "  \"num_hidden_layers\": 24,\n",
      "  \"num_key_value_heads\": 16,\n",
      "  \"pretraining_tp\": 1,\n",
      "  \"rms_norm_eps\": 1e-06,\n",
      "  \"rope_scaling\": {\n",
      "    \"factor\": 4.0,\n",
      "    \"rope_type\": \"linear\",\n",
      "    \"type\": \"linear\"\n",
      "  },\n",
      "  \"rope_theta\": 100000,\n",
      "  \"tie_word_embeddings\": false,\n",
      "  \"torch_dtype\": \"bfloat16\",\n",
      "  \"transformers_version\": \"4.44.1\",\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 32256\n",
      "}\n",
      "\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "{'train_runtime': 8627.9966, 'train_samples_per_second': 14.835, 'train_steps_per_second': 0.232, 'train_loss': 0.5831610231995582, 'epoch': 1.0}\n",
      "100%|█████████████████████████████████████| 2000/2000 [2:23:46<00:00,  4.31s/it]\n",
      "Waiting for the current checkpoint push to be finished, this might take a couple of minutes.\n",
      "Saving model checkpoint to fim_llama\n",
      "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--deepseek-ai--deepseek-coder-1.3b-instruct/snapshots/e063262dac8366fc1f28a4da0ff3c50ea66259ca/config.json\n",
      "Unrecognized keys in `rope_scaling` for 'rope_type'='linear': {'type'}\n",
      "Model config LlamaConfig {\n",
      "  \"architectures\": [\n",
      "    \"LlamaForCausalLM\"\n",
      "  ],\n",
      "  \"attention_bias\": false,\n",
      "  \"attention_dropout\": 0.0,\n",
      "  \"bos_token_id\": 32013,\n",
      "  \"eos_token_id\": 32021,\n",
      "  \"hidden_act\": \"silu\",\n",
      "  \"hidden_size\": 2048,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 5504,\n",
      "  \"max_position_embeddings\": 16384,\n",
      "  \"mlp_bias\": false,\n",
      "  \"model_type\": \"llama\",\n",
      "  \"num_attention_heads\": 16,\n",
      "  \"num_hidden_layers\": 24,\n",
      "  \"num_key_value_heads\": 16,\n",
      "  \"pretraining_tp\": 1,\n",
      "  \"rms_norm_eps\": 1e-06,\n",
      "  \"rope_scaling\": {\n",
      "    \"factor\": 4.0,\n",
      "    \"rope_type\": \"linear\",\n",
      "    \"type\": \"linear\"\n",
      "  },\n",
      "  \"rope_theta\": 100000,\n",
      "  \"tie_word_embeddings\": false,\n",
      "  \"torch_dtype\": \"bfloat16\",\n",
      "  \"transformers_version\": \"4.44.1\",\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 32256\n",
      "}\n",
      "\n",
      "Saving model checkpoint to fim_llama\n",
      "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--deepseek-ai--deepseek-coder-1.3b-instruct/snapshots/e063262dac8366fc1f28a4da0ff3c50ea66259ca/config.json\n",
      "Unrecognized keys in `rope_scaling` for 'rope_type'='linear': {'type'}\n",
      "Model config LlamaConfig {\n",
      "  \"architectures\": [\n",
      "    \"LlamaForCausalLM\"\n",
      "  ],\n",
      "  \"attention_bias\": false,\n",
      "  \"attention_dropout\": 0.0,\n",
      "  \"bos_token_id\": 32013,\n",
      "  \"eos_token_id\": 32021,\n",
      "  \"hidden_act\": \"silu\",\n",
      "  \"hidden_size\": 2048,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 5504,\n",
      "  \"max_position_embeddings\": 16384,\n",
      "  \"mlp_bias\": false,\n",
      "  \"model_type\": \"llama\",\n",
      "  \"num_attention_heads\": 16,\n",
      "  \"num_hidden_layers\": 24,\n",
      "  \"num_key_value_heads\": 16,\n",
      "  \"pretraining_tp\": 1,\n",
      "  \"rms_norm_eps\": 1e-06,\n",
      "  \"rope_scaling\": {\n",
      "    \"factor\": 4.0,\n",
      "    \"rope_type\": \"linear\",\n",
      "    \"type\": \"linear\"\n",
      "  },\n",
      "  \"rope_theta\": 100000,\n",
      "  \"tie_word_embeddings\": false,\n",
      "  \"torch_dtype\": \"bfloat16\",\n",
      "  \"transformers_version\": \"4.44.1\",\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 32256\n",
      "}\n",
      "\n",
      "Saving model checkpoint to fim_llama\n",
      "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--deepseek-ai--deepseek-coder-1.3b-instruct/snapshots/e063262dac8366fc1f28a4da0ff3c50ea66259ca/config.json\n",
      "Unrecognized keys in `rope_scaling` for 'rope_type'='linear': {'type'}\n",
      "Model config LlamaConfig {\n",
      "  \"architectures\": [\n",
      "    \"LlamaForCausalLM\"\n",
      "  ],\n",
      "  \"attention_bias\": false,\n",
      "  \"attention_dropout\": 0.0,\n",
      "  \"bos_token_id\": 32013,\n",
      "  \"eos_token_id\": 32021,\n",
      "  \"hidden_act\": \"silu\",\n",
      "  \"hidden_size\": 2048,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 5504,\n",
      "  \"max_position_embeddings\": 16384,\n",
      "  \"mlp_bias\": false,\n",
      "  \"model_type\": \"llama\",\n",
      "  \"num_attention_heads\": 16,\n",
      "  \"num_hidden_layers\": 24,\n",
      "  \"num_key_value_heads\": 16,\n",
      "  \"pretraining_tp\": 1,\n",
      "  \"rms_norm_eps\": 1e-06,\n",
      "  \"rope_scaling\": {\n",
      "    \"factor\": 4.0,\n",
      "    \"rope_type\": \"linear\",\n",
      "    \"type\": \"linear\"\n",
      "  },\n",
      "  \"rope_theta\": 100000,\n",
      "  \"tie_word_embeddings\": false,\n",
      "  \"torch_dtype\": \"bfloat16\",\n",
      "  \"transformers_version\": \"4.44.1\",\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 32256\n",
      "}\n",
      "\n",
      "No files have been modified since last commit. Skipping to prevent empty commit.\n"
     ]
    }
   ],
   "source": [
    "!git pull\n",
    "!CUDA_VISIBLE_DEVICES=0 WANDB_PROJECT=personal-code-copilot python train.py \\\n",
    "--seed 11 \\\n",
    "--model_name_or_path \"deepseek-ai/deepseek-coder-1.3b-instruct\" \\\n",
    "--dataset_name \"smangrul/hug_stack\" \\\n",
    "--splits \"train\" \\\n",
    "--max_seq_len 2048 \\\n",
    "--max_steps 2000 \\\n",
    "--save_steps 500 \\\n",
    "--eval_steps 100 \\\n",
    "--logging_steps 5 \\\n",
    "--log_level \"info\" \\\n",
    "--logging_strategy \"steps\" \\\n",
    "--evaluation_strategy \"steps\" \\\n",
    "--save_strategy \"steps\" \\\n",
    "--push_to_hub \\\n",
    "--hub_private_repo True \\\n",
    "--hub_strategy \"every_save\" \\\n",
    "--bf16 True \\\n",
    "--learning_rate 2e-4 \\\n",
    "--lr_scheduler_type \"cosine\" \\\n",
    "--weight_decay 0.1 \\\n",
    "--warmup_ratio 0.1 \\\n",
    "--max_grad_norm 1.0 \\\n",
    "--output_dir \"fim_llama\" \\\n",
    "--per_device_train_batch_size 16 \\\n",
    "--per_device_eval_batch_size 16 \\\n",
    "--gradient_accumulation_steps 4 \\\n",
    "--gradient_checkpointing True \\\n",
    "--use_reentrant True \\\n",
    "--dataset_text_field \"text\" \\\n",
    "--test_size 0.1 \\\n",
    "--fim_rate 0.9 \\\n",
    "--fim_spm_rate 0.5 \\\n",
    "--use_peft_lora True \\\n",
    "--lora_r 16 \\\n",
    "--lora_alpha 16 \\\n",
    "--lora_dropout 0.1 \\\n",
    "--lora_target_modules \"q_proj,k_proj,v_proj,o_proj,down_proj,up_proj,gate_proj\" \\\n",
    "--use_4bit_quantization True \\\n",
    "--use_nested_quant True \\\n",
    "--bnb_4bit_compute_dtype \"bfloat16\" \\\n",
    "--use_flash_attn True \\\n",
    "--use_unsloth True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "848a5a38-525c-4699-8b70-fc679b8103cd",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}