prototyping.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "from transformers import AutoTokenizer\n",
    "\n",
    "model = 'distilbert/distilgpt2'\n",
    "tokenizer = AutoTokenizer.from_pretrained(model)\n",
    "tokenizer.pad_token = tokenizer.eos_token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 1.5616,  0.9337,  1.8573],\n",
       "        [ 1.0641, -0.7764,  1.0184],\n",
       "        [ 4.4964, -2.8384, -1.0813],\n",
       "        [ 1.7908, -0.6957, -8.6970],\n",
       "        [-0.4853,  3.9855,    -inf]])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "x = torch.normal(0, 3, (5, 3))\n",
    "x[-1, -1] = float('-inf')\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.5935,  1.4084, -0.8150],\n",
       "        [-1.4142,  0.7082,  0.7060],\n",
       "        [ 0.4957, -1.3949,  0.8992],\n",
       "        [ 1.0119, -1.3615,  0.3496],\n",
       "        [    nan,     nan,     nan]])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "F.layer_norm(x, (3,))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x21e45afe9f0>]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGdCAYAAADAAnMpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAAA9EklEQVR4nO3deXxU9b3/8ffMJJkkkJVAQkIg7PuiIGlQtGqUKkW9rRbRWyh1qV5q1dhexSpUvTVuVXuVFutPxV6vV9RaukBBRAGRqKwq+74FkkAgmZCQbeb7+yPJQCQJmZDkzExez8djHpk58z0zn8NhJu+c7/d8j80YYwQAAGARu9UFAACAjo0wAgAALEUYAQAAliKMAAAASxFGAACApQgjAADAUoQRAABgKcIIAACwVIjVBTSHx+PR4cOHFRUVJZvNZnU5AACgGYwxKikpUXJysuz2xo9/BEQYOXz4sFJTU60uAwAAtMDBgwfVo0ePRp8PiDASFRUlqWZjoqOjLa4GAAA0h8vlUmpqqvf3eGMCIozUdc1ER0cTRgAACDDnGmLBAFYAAGApwggAALAUYQQAAFiKMAIAACxFGAEAAJYijAAAAEsRRgAAgKUIIwAAwFKEEQAAYCmfw8jKlSs1adIkJScny2azacGCBedcZ/ny5brwwgvldDrVr18/zZs3rwWlAgCAYORzGCktLdXIkSM1Z86cZrXfu3evJk6cqMsvv1wbN27Ufffdp9tvv11LlizxuVgAABB8fL42zTXXXKNrrrmm2e3nzp2r3r1763e/+50kafDgwVq1apVeeOEFTZgwwde3BwAAQabNL5SXk5OjzMzMessmTJig++67r9F1KioqVFFR4X3scrnaqjwAQABze4yq3J7aW839ymqP3B6jak/Nspr7Rm6PR9VnPjZGbnfNT0/tY49RzX2PkccYGSN56pYbI3PGfY+RTG0bozPb1NR2+jnVa1PX4Iy7MjpjvW8t1xnL615XZ7Q5UwOLzmpnzmh15nO3XdJbqfGRzfhXb31tHkby8vKUmJhYb1liYqJcLpdOnTqliIiIs9bJzs7WY4891talAQDagdtjVFJepeJTVXKdqparvEol5VU6WeFWaUW1TlZUq7SiWmWVbp2qdKusyq1TldU6VeVWeZVHFdU1P8ur3Kqs9qiiuiZwVLprQgdax3WjkoM3jLTEzJkzlZWV5X3scrmUmppqYUUAgDNVVnuU7yrXkeJyHS2p0NGSchWUVOhoSYWOl1bqeFmlTpRW6nhppVzl1e1Wl8NuU6jDphC7XSF1P+222vs2Oby3muV2u00OW816dpvN+7Nuud1mk81mk90m2Wrb1Ty2yaaaZTVtVG+ZTTXLbDZJdfe//VxtzbaaRt7nTt8/rbbJ6bbeJ85oc8a63/btRQ21SYwOb84/cZto8zCSlJSk/Pz8esvy8/MVHR3d4FERSXI6nXI6nW1dGgCgEcYY5bsqtK+wVPsLS7W/sEz7C8t0qOiUjhSd0tGTFQ12EzQlItSh6IgQxUSEqrMzRJ2cIYoKD1GnsJr7kWEORYY5FBFWcz8i1KHwULucoQ6FhzjkDLXLGVJ3cygsxK5Qh732p02hdrvs9gZ+y8LvtXkYycjI0KJFi+otW7p0qTIyMtr6rQEAzeAqr9KmQ8XalleiHfkl2p5fop35J3WyoukjGmEhdnWPCVdiVLi6Rjm9ty6dwhTXKUxxkWGK7xSq2MgwRYeHKiyEqa3QMJ/DyMmTJ7Vr1y7v471792rjxo2Kj49Xz549NXPmTOXm5urPf/6zJOmuu+7Syy+/rP/8z//UT3/6U3388cd69913tXDhwtbbCgBAs7g9RpsPF2vd/hP6+lCxvjpYpD3HShts67Db1CMuQr26dFJal0j1jI9Uj7hIpcRGKDk2XPGdwrzdBsD58DmMrF27Vpdffrn3cd3YjmnTpmnevHk6cuSIDhw44H2+d+/eWrhwoe6//379/ve/V48ePfT//t//47ReAGgHxhhtOeJSzu5C5ewu1Jd7j6ukgSMeqfERGtI9WgMTozQgKUoDE6OUltBJoQ6OZqDt2Yzxtdev/blcLsXExKi4uFjR0dFWlwMAfq2i2q2c3YVauiVfH23NV76rot7zUeEhGtMrTqNS4zQiNUYje8QqvlOYRdUimDX397dfnk0DAPCN22P06c6j+sv6XH2yraDeeI/IMIfSe8cro28XZfRJ0JDkaDkY6Ak/QhgBgAC2M79E768/pL+uz1VByekjIN2inMockqirhiQqo08XhYc6LKwSaBphBAACjMdj9Mn2Ar366R59vue4d3lcZKiuH5Wi60cla2SPWE5zRcAgjABAgCivcmvBhly9+uke7T5acwaMw27T5QO76cbRPXTFoG6cPouARBgBAD9X7fZo/tqDevGjnTpa2xUT5QzRLek99ZOL09Q9puEJJIFAQRgBAD9ljNGHW/L19OJt2lN7JCQ5Jlw/vaS3Jl+UqqjwUIsrBFoHYQQA/NDXh4r02D+2aN3+E5Kk+E5h+sUV/XRLei+6YhB0CCMA4EfKq9x64aMdenXlHnmMFB5q1+2X9NHPLuvDkRAELcIIAPiJNfuO68H3v/ZOz37dyGQ9fO1gJcVYdzVVoD0QRgDAYuVVbj31r216M2efjKmZI+S3/zZcVw1JtLo0oF0QRgDAQodOlOnut9brm9xiSdKPxvTQrycOUUwEXTLoOAgjAGCRT3ce1S/+b4NOlFUpLjJUz08epcsHdrO6LKDdEUYAoJ0ZY/THFbv13JLt8hhpeEqM/vjvF6pHXKTVpQGWIIwAQDuqrPYo692N+ufXRyTVdMs8fv0wrh2DDo0wAgDt5FSlW3e9tU4rdhxVqMOmx64bpiljU2WzcQ0ZdGyEEQBoB67yKt02b43W7DuhiFCH/jR1tMb372p1WYBfIIwAQBsrPFmhqa9/qc2HXYoKD9G86RdpdK94q8sC/AZhBADaUIGrXFNe/Vy7j5aqS6cw/fm2sRqaHGN1WYBfIYwAQBtxlVdp2htrtPtoqZJjwvU/t6erb9fOVpcF+B3CCAC0gYpqt37253XaesSlhM5OvXNnhnp24dRdoCFc+hEAWpnHY/TAu18pZ0+hOjtrxogQRIDGEUYAoBUZY/TEwi3659dHFOqwae6/j9awFMaIAE0hjABAK3r10z1647N9kqTnbhqpS/onWFsQEAAIIwDQSlbtPKbsf22TJD0ycbCuH5VicUVAYCCMAEAryCsu173vbJAx0s0Xper28X2sLgkIGIQRADhPVW6P7vm/9SosrdSQ7tH6zXVDrS4JCCiEEQA4T899uF1r9p1QZ2eI/nDrhVz0DvARYQQAzsPSLfl6ZcUeSdKzN45QWkIniysCAg9hBABa6ODxMj3w7kZJ0vSL03TN8O7WFgQEKMIIALSAMUb/+f7XcpVXa1RqrGZeM9jqkoCARRgBgBaYv+agcvYUKjzUrt/fPEphIXydAi3FpwcAfJRXXK7fLtwqSfrl1QPVqwvjRIDzQRgBAB8YY/TIgk0qqajWyNRYTb+4t9UlAQGPMAIAPlj4zRF9tDVfoQ6bnvnhCDnsNqtLAgIeYQQAmulEaaVm/22zJOk/vttPA5OiLK4ICA6EEQBopif+uUWFpZUakNhZ/3F5X6vLAYIGYQQAmmHNvuP6YEOubDbp6R+OkDOEWVaB1kIYAYBzMMbov2rPnrn5olRd0DPO4oqA4EIYAYBz+MfXR/TVwSJ1CnPo/qsGWF0OEHQIIwDQhPIqt57+1zZJ0l2X9VW3qHCLKwKCD2EEAJowb/U+5RadUlJ0uG4f38fqcoCgRBgBgEYUnqzQnI93SZJ+NWGgIsIYtAq0BcIIADTi98t2qqSiWkOTo/VvF6RYXQ4QtAgjANCAXQUn9b9fHJAk/XriYNmZaRVoM4QRAGjA80u3y+0xyhzcTeP6JlhdDhDUCCMA8C0780v0r015kqRfTRhkcTVA8COMAMC3/GH5bhkjTRiayPVngHZAGAGAM+wvLNXfNuZKkn5+eX+LqwE6BsIIAJzhj8t3y2Ok7w7squE9YqwuB+gQCCMAUCu36JT+sv6QJOmeK/pZXA3QcRBGAKDWKyt2q8ptNK5vF43uFW91OUCHQRgBAEkFrnK9s+agJOnnHBUB2hVhBAAkvfrpHlVWezS6V5wy+nSxuhygQyGMAOjwTpRW6q3Pa2Zb/fkV/WSzMdsq0J4IIwA6vHfWHNSpKreGdI/Wdwd0tbocoMMhjADo0KrdHr31+X5J0k8uTuOoCGCBFoWROXPmKC0tTeHh4UpPT9eXX37ZZPsXX3xRAwcOVEREhFJTU3X//fervLy8RQUDQGv6aGuBcotOKS4yVNeNTLa6HKBD8jmMzJ8/X1lZWZo9e7bWr1+vkSNHasKECSooKGiw/dtvv62HHnpIs2fP1tatW/Xaa69p/vz5evjhh8+7eAA4X2+u3idJmjK2p8JDHdYWA3RQPoeR559/XnfccYemT5+uIUOGaO7cuYqMjNTrr7/eYPvVq1fr4osv1i233KK0tDRdffXVmjJlyjmPpgBAW9uW51LOnkI57Db9+3d6WV0O0GH5FEYqKyu1bt06ZWZmnn4Bu12ZmZnKyclpcJ1x48Zp3bp13vCxZ88eLVq0SNdee+15lA0A5+/N1TVjRa4ekqjk2AiLqwE6rhBfGh87dkxut1uJiYn1licmJmrbtm0NrnPLLbfo2LFjuuSSS2SMUXV1te66664mu2kqKipUUVHhfexyuXwpEwDOqbisSgs21FwQb9q4NGuLATq4Nj+bZvny5XryySf1hz/8QevXr9cHH3yghQsX6oknnmh0nezsbMXExHhvqampbV0mgA7m3bU1p/MOSopSem+mfges5NORkYSEBDkcDuXn59dbnp+fr6SkpAbXefTRR/XjH/9Yt99+uyRp+PDhKi0t1Z133qlf//rXstvPzkMzZ85UVlaW97HL5SKQAGg1bo/Rnz/fJ0n6yThO5wWs5tORkbCwMI0ePVrLli3zLvN4PFq2bJkyMjIaXKesrOyswOFw1IxYN8Y0uI7T6VR0dHS9GwC0lk+2Fejg8VOKiQjV9aNSrC4H6PB8OjIiSVlZWZo2bZrGjBmjsWPH6sUXX1RpaammT58uSZo6dapSUlKUnZ0tSZo0aZKef/55XXDBBUpPT9euXbv06KOPatKkSd5QAgDt6X9qJzm7+aJURYTxPQRYzecwMnnyZB09elSzZs1SXl6eRo0apcWLF3sHtR44cKDekZBHHnlENptNjzzyiHJzc9W1a1dNmjRJv/3tb1tvKwCgmQ4XndLKnUclSbek97S4GgCSZDON9ZX4EZfLpZiYGBUXF9NlA+C8vPzxTj334Q6N7R2vd3/WcPcygNbR3N/fXJsGQIdhjNH76w5Jkm4a3cPiagDUIYwA6DDW7DuhfYVl6hTm0LXDu1tdDoBahBEAHcZ7aw9KkiaO6K5OTp+HzAFoI4QRAB1CaUW1Fn5zRJJ00xjmLQL8CWEEQIew6JsjKqt0q3dCJ43pFWd1OQDOQBgB0CG8t7Zm4OqNo3sw4yrgZwgjAILevmOl+nLfcdlt0g8uZMZVwN8QRgAEvbrTeS/p31XdYyIsrgbAtxFGAAQ1t8foL+trwsiPxjC3COCPCCMAglrO7kIdKS5XTESoMgcnWl0OgAYQRgAEtX98dViSdO3w7goP5aJ4gD8ijAAIWpXVHv1rU83cIteNTLa4GgCNIYwACFqf7jwqV3m1ukU5NbZ3vNXlAGgEYQRA0Dqzi8ZhZ24RwF8RRgAEpfIqt5ZuyZckTaKLBvBrhBEAQemTbQUqrXQrJTZCF/aMtbocAE0gjAAISv/4uqaL5vsjuzP9O+DnCCMAgs7Jimot21ogSZo0gi4awN8RRgAEnY+25Kui2qPeCZ00NDna6nIAnANhBEDQqTuLZtIIumiAQEAYARBUisuqtHLnUUmcRQMECsIIgKCyZHOeqtxGg5Ki1D8xyupyADQDYQRAUKk7i4ajIkDgIIwACBrFZVXK2V0oqWbWVQCBgTACIGh8vD1f1R6jAYmd1Tuhk9XlAGgmwgiAoLFkU8307xOGJllcCQBfEEYABIXyKrdW7Kg5i4YwAgQWwgiAoPDpzmM6VVVzLRomOgMCC2EEQFBYsjlPknTVkEQmOgMCDGEEQMCrdnu0bGvNeJGrhyZaXA0AXxFGAAS8NftO6ERZleIiQzU2Ld7qcgD4iDACIOB9uKWmi+bKwYkKcfC1BgQaPrUAApoxRh9uru2iGUIXDRCICCMAAtrmwy7lFp1SRKhDlw7oanU5AFqAMAIgoH1YexbNpQMSFB7qsLgaAC1BGAEQ0D7cwqyrQKAjjAAIWPsLS7Utr0QOu01XDOpmdTkAWogwAiBgLa09KpLeO16xkWEWVwOgpQgjAALWx9sKJNWc0gsgcBFGAASkkvIqfbn3uCTpSrpogIBGGAEQkFbtPKZqj1GfhE5KS+hkdTkAzgNhBEBAWlbbRXM5R0WAgEcYARBwPB6j5dtrx4sQRoCARxgBEHC+yS3WsZOV6uwM0RgujAcEPMIIgIBTdxbN+P4JCgvhawwIdHyKAQScjxkvAgQVwgiAgFLgKtc3ucWSpMsHEkaAYEAYARBQlm8/Kkka2SNGXaOcFlcDoDUQRgAEFLpogOBDGAEQMCqq3fp0Z82RkSsHMQU8ECwIIwACxpq9J1Ra6VbXKKeGJkdbXQ6AVkIYARAwvF00A7vKbrdZXA2A1kIYARAwPqmddfUKxosAQYUwAiAg7C8s1d5jpQqx23RxvwSrywHQiggjAALCyh01A1dH94pTVHioxdUAaE2EEQABYcWOY5KkywZ2tbgSAK2NMALA71VWe7R6d00YubQ/YQQINi0KI3PmzFFaWprCw8OVnp6uL7/8ssn2RUVFmjFjhrp37y6n06kBAwZo0aJFLSoYQMezdv9xlVW6ldDZqSHdOaUXCDYhvq4wf/58ZWVlae7cuUpPT9eLL76oCRMmaPv27erW7ewR7pWVlbrqqqvUrVs3vf/++0pJSdH+/fsVGxvbGvUD6ABW1nbRXDoggVN6gSDkcxh5/vnndccdd2j69OmSpLlz52rhwoV6/fXX9dBDD53V/vXXX9fx48e1evVqhYbWDDpLS0s7v6oBdCgragevXjaALhogGPnUTVNZWal169YpMzPz9AvY7crMzFROTk6D6/z9739XRkaGZsyYocTERA0bNkxPPvmk3G53o+9TUVEhl8tV7wagYypwlWvrEZdsNukSTukFgpJPYeTYsWNyu91KTKx/TYjExETl5eU1uM6ePXv0/vvvy+12a9GiRXr00Uf1u9/9Tv/1X//V6PtkZ2crJibGe0tNTfWlTABBZOXOmi6a4Skx6tKZq/QCwajNz6bxeDzq1q2b/vSnP2n06NGaPHmyfv3rX2vu3LmNrjNz5kwVFxd7bwcPHmzrMgH4KbpogODn05iRhIQEORwO5efn11uen5+vpKSkBtfp3r27QkND5XA4vMsGDx6svLw8VVZWKiws7Kx1nE6nnE7+AgI6OrfHaFXtVXovJYwAQcunIyNhYWEaPXq0li1b5l3m8Xi0bNkyZWRkNLjOxRdfrF27dsnj8XiX7dixQ927d28wiABAnW9yi3WirEpR4SG6IDXW6nIAtBGfu2mysrL06quv6s0339TWrVt19913q7S01Ht2zdSpUzVz5kxv+7vvvlvHjx/Xvffeqx07dmjhwoV68sknNWPGjNbbCgBBacX2mqMil/RLUIiDORqBYOXzqb2TJ0/W0aNHNWvWLOXl5WnUqFFavHixd1DrgQMHZLef/tJITU3VkiVLdP/992vEiBFKSUnRvffeqwcffLD1tgJAUFpJFw3QIdiMMcbqIs7F5XIpJiZGxcXFio5m9kWgIyguq9IFT3woj5E+e+gKpcRGWF0SAB819/c3xz0B+KXPdh+Tx0j9unUmiABBjjACwC+trD2llwvjAcGPMALA7xhj9GntZGfjBzDrKhDsCCMA/M7eY6XKLTqlMIdd6b3jrS4HQBsjjADwO6t21RwVGd0rTpFhPp/0ByDAEEYA+J2VO+iiAToSwggAv1Ll9ujzPYWSpPH9GLwKdASEEQB+ZePBIp2sqFZcZKiGJjOvENAREEYA+JW6s2gu7pcgu91mcTUA2gNhBIBf+XQn84sAHQ1hBIDfKD5Vpa8OFkmSLunP4FWgoyCMAPAbObVTwPft2knJTAEPdBiEEQB+wzvrKl00QIdCGAHgN06HEbpogI6EMALAL+wvLNWB42UKsduU3qeL1eUAaEeEEQB+oe6oyIW94tTZyRTwQEdCGAHgF1bVddH0o4sG6GgIIwAsV+326LPdddejYfAq0NEQRgBY7uvcYpWUVys6PETDU2KsLgdAOyOMALDcqjOmgHcwBTzQ4RBGAFjuzDACoOMhjACwVGlFtdYfOCGJ+UWAjoowAsBSX+wtVLXHKDU+Qr26dLK6HAAWIIwAsFTd/CKX9OMsGqCjIowAsNQqpoAHOjzCCADL5LvKtbPgpGw2KYMp4IEOizACwDJ1R0WGp8QorlOYxdUAsAphBIBlVu2qGy9CFw3QkRFGAFjCGHM6jDBeBOjQCCMALLEj/6SOllQoPNSu0b3irC4HgIUIIwAs8enOo5Kksb27yBnisLgaAFYijACwRF0XzXjGiwAdHmEEQLurqHbriz3HJTFeBABhBIAFNhwo0qkqtxI6h2lgYpTV5QCwGGEEQLs78yq9drvN4moAWI0wAqDdfcr8IgDOQBgB0K6Kyir19aEiSdL4/lwcDwBhBEA7W727UMZI/bt1VlJMuNXlAPADhBEA7apufhGOigCoQxgB0G6MMVq5o3Z+EU7pBVCLMAKg3ewrLFNu0SmFOmxK7xNvdTkA/ARhBEC7WVXbRTO6V5wiw0IsrgaAvyCMAGg3n+6s66JhvAiA0wgjANpFtdujnN2FkhgvAqA+wgiAdvHVoSKVVFQrNjJUQ5NjrC4HgB8hjABoF3Vn0VzcL0EOpoAHcAbCCIB2sap2CvjxTAEP4FsIIwDanKu8ShsPFkmSLmG8CIBvIYwAaHM5uwvl9hj1SeikHnGRVpcDwM8QRgC0uVW1p/RyVARAQwgjANoc16MB0BTCCIA2dfB4mfYVlslht+k7TAEPoAGEEQBtasWOmqMiF/aMVVR4qMXVAPBHhBEAbWplbRi5bABdNAAaRhgB0GYqqz1aXTsF/GUDullcDQB/RRgB0GbWHzihkxXV6tIpTEOTo60uB4CfalEYmTNnjtLS0hQeHq709HR9+eWXzVrvnXfekc1m0w033NCStwUQYOq6aMb3T5CdKeABNMLnMDJ//nxlZWVp9uzZWr9+vUaOHKkJEyaooKCgyfX27dunX/7ylxo/fnyLiwUQWOoGr142kPEiABrncxh5/vnndccdd2j69OkaMmSI5s6dq8jISL3++uuNruN2u3XrrbfqscceU58+fc6rYACB4WhJhTYfdklifhEATfMpjFRWVmrdunXKzMw8/QJ2uzIzM5WTk9Poeo8//ri6deum2267rVnvU1FRIZfLVe8GILDUTXQ2LCVaCZ2dFlcDwJ/5FEaOHTsmt9utxMTEessTExOVl5fX4DqrVq3Sa6+9pldffbXZ75Odna2YmBjvLTU11ZcyAfiBFZzSC6CZ2vRsmpKSEv34xz/Wq6++qoSE5l+TYubMmSouLvbeDh482IZVAmhtHo/Rp7XXo7mULhoA5xDiS+OEhAQ5HA7l5+fXW56fn6+kpKSz2u/evVv79u3TpEmTvMs8Hk/NG4eEaPv27erbt+9Z6zmdTjmdHNYFAtWmw8U6Xlqpzs4QXdgrzupyAPg5n46MhIWFafTo0Vq2bJl3mcfj0bJly5SRkXFW+0GDBumbb77Rxo0bvbfrrrtOl19+uTZu3Ej3CxCk6k7pvbhfF4U6mM4IQNN8OjIiSVlZWZo2bZrGjBmjsWPH6sUXX1RpaammT58uSZo6dapSUlKUnZ2t8PBwDRs2rN76sbGxknTWcgDBo268yKWMFwHQDD6HkcmTJ+vo0aOaNWuW8vLyNGrUKC1evNg7qPXAgQOy2/lLCOioXOVVWn+gSBLjRQA0j80YY6wu4lxcLpdiYmJUXFys6GimlAb82eJNR3TXW+vVp2snffzAd60uB4CFmvv7m0MYAFoVp/QC8BVhBECrMcZo+XbCCADfEEYAtJqtR0p0pLhcEaEOfadPF6vLARAgCCMAWs3H22rmILq4X4LCQx0WVwMgUBBGALSaj7fVXL37ikHdLK4EQCAhjABoFcdLK7XhYJEk6fJBjBcB0HyEEQCtYsWOAhkjDe4ere4xEVaXAyCAEEYAtIplW2u6aK6kiwaAjwgjAM5btdvjvR7N5YQRAD4ijAA4b+v2n5CrvFrxncI0KjXW6nIABBjCCIDzVncWzWUDuspht1lcDYBAQxgBcN44pRfA+SCMADgvB4+XaWfBSTnsNl3KFPAAWoAwAuC8fLK95qjI6F5xiokItbgaAIGIMALgvNSd0ksXDYCWIowAaLGyymrl7CmURBgB0HKEEQAt9tmuQlVWe5QSG6H+3TpbXQ6AAEUYAdBiSzbnSZKuHpoom41TegG0DGEEQItUuz1atjVfknT1kCSLqwEQyAgjAFpkzb4TOlFWpbjIUF2UFmd1OQACGGEEQIvUddFcOThRIQ6+SgC0HN8gAHxmjNHSLTVdNBOG0kUD4PwQRgD4bPNhl3KLTiki1KHx/ROsLgdAgCOMAPBZXRfNZQO6KjzUYXE1AAIdYQSAzz7cXNtFMyzR4koABAPCCACf7DtWqu35JQqx23TFQMIIgPNHGAHgkw+31HTRfKdPF8VEcmE8AOePMALAJ0vqumiGclQEQOsgjABotoKScq0/cEKSdBWzrgJoJYQRAM320ZYCGSONTI1VUky41eUACBKEEQDN5r0w3hC6aAC0HsIIgGY5UVqpz3YdkyR9bxhdNABaD2EEQLMs3pynao/RkO7R6tu1s9XlAAgihBEAzfKPrw5LkiaNTLa4EgDBhjAC4JwKSsr1+Z5CSdL3R3S3uBoAwYYwAuCc/vVNnjxGuqBnrFLjI60uB0CQIYwAOKe6Lprvj6CLBkDrI4wAaFJu0Smt3X9CNps0cThdNABaH2EEQJMWfl1zVGRsWjwTnQFoE4QRAE36x1dHJHEWDYC2QxgB0Kh9x0r1TW6xHHabrmGiMwBthDACoFH/rO2iGde3i7p0dlpcDYBgRRgB0Ci6aAC0B8IIgAbtyC/R9vwShTpsmjCULhoAbYcwAqBBH6zPlSRdNqCbYiJCLa4GQDAjjAA4S7Xbo7+sPyRJunF0isXVAAh2hBEAZ1m586iOllQovlOYrhiUaHU5AIIcYQTAWd5bW3NU5IZRKQoL4WsCQNviWwZAPcdLK/XR1nxJ0k1jelhcDYCOgDACoJ6/bcxVldtoWEq0BnePtrocAB0AYQRAPXVdNDeNTrW4EgAdBWEEgNem3GJtOeJSmMOu60cx0RmA9kEYAeD1/rqaoyJXDU1UbGSYxdUA6CgIIwAkSRXVbi3YWDPR2U2jGbgKoP0QRgBIkpZtLVBRWZWSosM1vn9Xq8sB0IEQRgBIkt5be1CS9MPRKXLYbRZXA6AjaVEYmTNnjtLS0hQeHq709HR9+eWXjbZ99dVXNX78eMXFxSkuLk6ZmZlNtgfQ/g4eL9PyHUclSTdyFg2AduZzGJk/f76ysrI0e/ZsrV+/XiNHjtSECRNUUFDQYPvly5drypQp+uSTT5STk6PU1FRdffXVys3NPe/iAbSO//l8v4yRxvdPUO+ETlaXA6CDsRljjC8rpKen66KLLtLLL78sSfJ4PEpNTdU999yjhx566Jzru91uxcXF6eWXX9bUqVOb9Z4ul0sxMTEqLi5WdDSTMAGt6VSlW9/JXqbiU1V6bdoYXTmYa9EAaB3N/f3t05GRyspKrVu3TpmZmadfwG5XZmamcnJymvUaZWVlqqqqUnx8fKNtKioq5HK56t0AtI0FG3NVfKpKPeMj9d2B3awuB0AH5FMYOXbsmNxutxIT6//llJiYqLy8vGa9xoMPPqjk5OR6gebbsrOzFRMT472lptKHDbQFY4zmfbZPkjQ1oxcDVwFYol3Ppnnqqaf0zjvv6K9//avCw8MbbTdz5kwVFxd7bwcPHmzHKoGO4/M9x7U9v0QRoQ7dNIbQD8AaIb40TkhIkMPhUH5+fr3l+fn5SkpKanLd5557Tk899ZQ++ugjjRgxosm2TqdTTqfTl9IAtMCbq/dJkn5wYYpiIkKtLQZAh+XTkZGwsDCNHj1ay5Yt8y7zeDxatmyZMjIyGl3vmWee0RNPPKHFixdrzJgxLa8WQKvJLTqlD7fUdK9OG5dmbTEAOjSfjoxIUlZWlqZNm6YxY8Zo7NixevHFF1VaWqrp06dLkqZOnaqUlBRlZ2dLkp5++mnNmjVLb7/9ttLS0rxjSzp37qzOnTu34qYA8MX/5OyXx0jj+nbRgMQoq8sB0IH5HEYmT56so0ePatasWcrLy9OoUaO0ePFi76DWAwcOyG4/fcDlj3/8oyorK3XjjTfWe53Zs2frN7/5zflVD6BFyqvcemfNAUkcFQFgPZ/nGbEC84wAreudLw/ooQ++UUpshFb+5+WcRQOgTbTJPCMAAl+126O5K3ZLkn4yLo0gAsByhBGgg1n4zRHtKyxTXGSobknvaXU5AEAYAToSj8fo5Y93SZJuu6S3Ojl9HjYGAK2OMAJ0IB9uydPOgpOKCg/RVAauAvAThBGggzDG6KXaoyI/GZem6HAmOQPgHwgjQAfxyfYCbT7sUmSYQ9Mv7m11OQDgRRgBOgBjjP57Wc1RkX//Ti/FdwqzuCIAOI0wAnQAq3cXauPBIjlD7Lp9PEdFAPgXwgjQAfz3sp2SpClje6pbVONXzAYAKxBGgCC3fHuBvth7XKEOm+68tI/V5QDAWQgjQBCrdnv05KKtkqSpGWlKjo2wuCIAOBthBAhi7649pB35JxUTEap7ruhndTkA0CDCCBCkTlZU6/ml2yVJv7iyv2IjOYMGgH8ijABB6pUVu3XsZKXSukTqx9/pZXU5ANAowggQhI4Un9Krn+6RJD10zSCFhfBRB+C/+IYCgtCzS7arvMqjsWnxmjA0yepyAKBJhBEgyGzKLdZfN+RKkn49cbBsNpvFFQFA0wgjQBBxe4x+vWCTjJGuH5WskamxVpcEAOdEGAGCyBuf7dVXB4sU5QzRzGsGW10OADQLYQQIEgcKy/TchzWn8j48cbCSYpj2HUBgIIwAQcAYo4c++FrlVR5l9Omimy9KtbokAGg2wggQBN5de1CrdxcqPNSu7B8MZ9AqgIBCGAECXL6rXP+1sOb6Mw9cNVBpCZ0srggAfEMYAQKYMUaPLtikkvJqjewRo+kXp1ldEgD4jDACBLB31hzUh1vyFWK36ekbRyjEwUcaQODhmwsIUJtyizX775slSQ9cPVCDkqItrggAWoYwAgQgV3mVZry9XpXVHl05qJt+dmkfq0sCgBYjjAABxhijX733lfYXliklNkK/+9FI2e2cPQMgcBFGgADz+mf7tGRzvkIdNv3h1gsVGxlmdUkAcF4II0AAWbf/hLIX1ZzG+8jEIVx7BkBQIIwAAWLfsVLd+ee1qvYYTRzRXVMzelldEgC0CsIIEACOnazQtDe+VGFppYYmR+vpH45gllUAQYMwAvi50opqTX9jjfYXlik1PkJvTL9InZ0hVpcFAK2GMAL4sSq3R3f/73p9k1us+E5henP6WHWL4mq8AIILYQTwUx6P0YN/+VordxxVRKhDr//kIvXp2tnqsgCg1RFGAD9U7fbol+9/pQ/W58phrzmFdxRnzgAIUnQ8A36motqte97eoA+35Mtht+n5H43U5YO6WV0WALQZwgjgR8oqq3Xnn9dp1a5jCnPY9fItF+jqoUlWlwUAbYowAviJ4rIqTZ/3pdYfKFJkmEOvTh2ji/slWF0WALQ5wgjgB3YVnNRdb63TroKTiokI1bzpF+mCnnFWlwUA7YIwAlhs8aY8/fK9r3SyolqJ0U7Nmz5Wg7tHW10WALQbwghgEbfH6LkPt+uPy3dLktJ7x+vlWy5U1yinxZUBQPsijAAWKHCVK+vdr7Rq1zFJ0u2X9NaD1wxSqIOz7QF0PIQRoB0ZY/SX9bl6/B+b5SqvVkSoQ8/cOEKTRiZbXRoAWIYwArST3KJTeviDb7Rix1FJ0vCUGP3uRyM1IDHK4soAwFqEEaCNVbk9evuLA3p2yXadrKhWWIhdWVcN0O2X9FYI3TIAQBgB2ooxRos35enZJdu151ipJGl0rzg9c+MI9eUaMwDgRRgB2sCafcf15KKt2nCgSJKU0DlM92YO0C1je8pht1lbHAD4GcII0Eo8HqPlOwr0p5V79Pme45KkiFCH7ri0j+68tI86O/m4AUBD+HYEzlN5lVsLNuTq1U/3aPfRmu6YELtNN41J1f2Z/dUtOtziCgHAvxFGgBYwxmhTrkvvrzuov391WCfKqiRJUc4QTUnvqZ+MS1NybITFVQJAYCCMAD44dKJMi745or+sy9X2/BLv8uSYcP30kt6afFGqosJDLawQAAIPYQRoQt0RkKVb8rR0a4G2HnF5n3OG2HX10CTdOLqHLumXwMBUAGghwghwBmOMDhwvU87uQuXsKVTO7kIVlFR4n7fbpDFp8bphVIomjuiumAiOggDA+SKMoEMrq6zWplyXvjpYpK8OFWn9/hM6XFxer01kmEOX9u+qzCGJumJQN8V3CrOoWgAIToQRdAhuj9HB42Xanl+iHXkl2lFwUtvzXNpVcFIeU79tqMOmkT1ildG3izL6dNGFveIUHuqwpnAA6ABaFEbmzJmjZ599Vnl5eRo5cqReeukljR07ttH27733nh599FHt27dP/fv319NPP61rr722xUUD32aM0YmyKh0uOqXcolM6XHRK+wvLtL+wVPsLy3TwRJmq3KbBdZOiwzWiR4xGpsZqZI9YXdgrVpFh5HQAaC8+f+POnz9fWVlZmjt3rtLT0/Xiiy9qwoQJ2r59u7p163ZW+9WrV2vKlCnKzs7W97//fb399tu64YYbtH79eg0bNqxVNgLBxxij8iqPXOVVKiqr0vHSSp0oq9Tx0prbsZMVOlpSeztZoXxXucqrPE2+pjPErn7dOmtgYpQGJEVpQGJnDU2OUSLzgACApWzGmIb/XGxEenq6LrroIr388suSJI/Ho9TUVN1zzz166KGHzmo/efJklZaW6p///Kd32Xe+8x2NGjVKc+fObdZ7ulwuxcTEqLi4WNHR0b6Ui1ZmjFG1x6jabVTp9qiq9lZZXXOrqL1VVntUXu1WRZVb5VUelVe5VV7lVlmVW+WVbpVV1twvq6jWyQq3TlZUqbTCrdKKarnKq1R8qqrRIxlN6RrlVHJMuLrHRKhXl0j16tJJaV0i1Suhk5KiwznjBQDaUXN/f/t0ZKSyslLr1q3TzJkzvcvsdrsyMzOVk5PT4Do5OTnKysqqt2zChAlasGBBo+9TUVGhiorTZzC4XK5G256P11bt1cHjZedsV5fXjPdxA21qnz3zOVPvsfHeN6Z++zNf16hmgal9X3NGG2OMt40xp9t76u4bI4+peeypbes5Y5kxRm7P6ec9xsjtkdwej3d5tccjj6fmp9tzOnhUuU8/bk8Ou00xEaGKiwxVfKcwxUaGKT4yTF2jnPVuiVHhSoxxyhnC2A4ACDQ+hZFjx47J7XYrMTGx3vLExERt27atwXXy8vIabJ+Xl9fo+2RnZ+uxxx7zpbQWWfj1Ya2vvZAZWi7MYVeow6awEPvpm8Ou8FBH7c2u8JCa+xFhDkWGORRRe79TWIg6h4eokzNEnZ0OdXaGKjoiRNHhoYqJCFVkmEM2G0czACCY+eUovZkzZ9Y7muJyuZSamtrq7/PD0T00rm/CWcsb+t131qIzGtm+tch2RuvTy7712Fa/jU222p/1H9ex28583iZ7bUP7GW3ttrrnaipw2OuW22pvkt1ec99hP708xG6T3V7/p8NuU4jdXvvTphBHzeMQh02hdrtCQ2wKddgVYrcRFgAA58WnMJKQkCCHw6H8/Px6y/Pz85WUlNTgOklJST61lySn0ymn0+lLaS1ya3qvNn8PAADQNLsvjcPCwjR69GgtW7bMu8zj8WjZsmXKyMhocJ2MjIx67SVp6dKljbYHAAAdi8/dNFlZWZo2bZrGjBmjsWPH6sUXX1RpaammT58uSZo6dapSUlKUnZ0tSbr33nt12WWX6Xe/+50mTpyod955R2vXrtWf/vSn1t0SAAAQkHwOI5MnT9bRo0c1a9Ys5eXladSoUVq8eLF3kOqBAwdkt58+4DJu3Di9/fbbeuSRR/Twww+rf//+WrBgAXOMAAAASS2YZ8QKzDMCAEDgae7vb5/GjAAAALQ2wggAALAUYQQAAFiKMAIAACxFGAEAAJYijAAAAEsRRgAAgKUIIwAAwFKEEQAAYCmfp4O3Qt0ksS6Xy+JKAABAc9X93j7XZO8BEUZKSkokSampqRZXAgAAfFVSUqKYmJhGnw+Ia9N4PB4dPnxYUVFRstlsrfa6LpdLqampOnjwYNBe8ybYt5HtC3zBvo1sX+AL9m1sy+0zxqikpETJycn1LqL7bQFxZMRut6tHjx5t9vrR0dFB+R/sTMG+jWxf4Av2bWT7Al+wb2NbbV9TR0TqMIAVAABYijACAAAs1aHDiNPp1OzZs+V0Oq0upc0E+zayfYEv2LeR7Qt8wb6N/rB9ATGAFQAABK8OfWQEAABYjzACAAAsRRgBAACWIowAAABLBX0Y+e1vf6tx48YpMjJSsbGxDbY5cOCAJk6cqMjISHXr1k2/+tWvVF1d3eTrHj9+XLfeequio6MVGxur2267TSdPnmyDLWi+5cuXy2azNXhbs2ZNo+t997vfPav9XXfd1Y6V+yYtLe2sep966qkm1ykvL9eMGTPUpUsXde7cWT/84Q+Vn5/fThU33759+3Tbbbepd+/eioiIUN++fTV79mxVVlY2uZ6/78M5c+YoLS1N4eHhSk9P15dfftlk+/fee0+DBg1SeHi4hg8frkWLFrVTpb7Jzs7WRRddpKioKHXr1k033HCDtm/f3uQ68+bNO2tfhYeHt1PFvvvNb35zVr2DBg1qcp1A2X9Sw98nNptNM2bMaLC9v++/lStXatKkSUpOTpbNZtOCBQvqPW+M0axZs9S9e3dFREQoMzNTO3fuPOfr+voZ9lXQh5HKykrddNNNuvvuuxt83u12a+LEiaqsrNTq1av15ptvat68eZo1a1aTr3vrrbdq8+bNWrp0qf75z39q5cqVuvPOO9tiE5pt3LhxOnLkSL3b7bffrt69e2vMmDFNrnvHHXfUW++ZZ55pp6pb5vHHH69X7z333NNk+/vvv1//+Mc/9N5772nFihU6fPiwfvCDH7RTtc23bds2eTwevfLKK9q8ebNeeOEFzZ07Vw8//PA51/XXfTh//nxlZWVp9uzZWr9+vUaOHKkJEyaooKCgwfarV6/WlClTdNttt2nDhg264YYbdMMNN2jTpk3tXPm5rVixQjNmzNDnn3+upUuXqqqqSldffbVKS0ubXC86Orrevtq/f387VdwyQ4cOrVfvqlWrGm0bSPtPktasWVNv25YuXSpJuummmxpdx5/3X2lpqUaOHKk5c+Y0+Pwzzzyj//7v/9bcuXP1xRdfqFOnTpowYYLKy8sbfU1fP8MtYjqIN954w8TExJy1fNGiRcZut5u8vDzvsj/+8Y8mOjraVFRUNPhaW7ZsMZLMmjVrvMv+9a9/GZvNZnJzc1u99paqrKw0Xbt2NY8//niT7S677DJz7733tk9RraBXr17mhRdeaHb7oqIiExoaat577z3vsq1btxpJJicnpw0qbF3PPPOM6d27d5Nt/Hkfjh071syYMcP72O12m+TkZJOdnd1g+x/96Edm4sSJ9Zalp6ebn/3sZ21aZ2soKCgwksyKFSsabdPYd5G/mj17thk5cmSz2wfy/jPGmHvvvdf07dvXeDyeBp8PpP0nyfz1r3/1PvZ4PCYpKck8++yz3mVFRUXG6XSa//u//2v0dXz9DLdE0B8ZOZecnBwNHz5ciYmJ3mUTJkyQy+XS5s2bG10nNja23tGGzMxM2e12ffHFF21ec3P9/e9/V2FhoaZPn37Otv/7v/+rhIQEDRs2TDNnzlRZWVk7VNhyTz31lLp06aILLrhAzz77bJPdauvWrVNVVZUyMzO9ywYNGqSePXsqJyenPco9L8XFxYqPjz9nO3/ch5WVlVq3bl29f3u73a7MzMxG/+1zcnLqtZdqPpOBsq8knXN/nTx5Ur169VJqaqquv/76Rr9r/MXOnTuVnJysPn366NZbb9WBAwcabRvI+6+yslJvvfWWfvrTnzZ5UdZA23919u7dq7y8vHr7JyYmRunp6Y3un5Z8hlsiIC6U15by8vLqBRFJ3sd5eXmNrtOtW7d6y0JCQhQfH9/oOlZ47bXXNGHChHNeZPCWW25Rr169lJycrK+//loPPvigtm/frg8++KCdKvXNL37xC1144YWKj4/X6tWrNXPmTB05ckTPP/98g+3z8vIUFhZ21pihxMREv9pfDdm1a5deeuklPffcc02289d9eOzYMbnd7gY/Y9u2bWtwncY+k/6+rzwej+677z5dfPHFGjZsWKPtBg4cqNdff10jRoxQcXGxnnvuOY0bN06bN29u0wuCtlR6errmzZungQMH6siRI3rsscc0fvx4bdq0SVFRUWe1D9T9J0kLFixQUVGRfvKTnzTaJtD235nq9oEv+6cln+GWCMgw8tBDD+npp59uss3WrVvPOcgqULRkew8dOqQlS5bo3XffPefrnznWZfjw4erevbuuvPJK7d69W3379m154T7wZRuzsrK8y0aMGKGwsDD97Gc/U3Z2tt9O19ySfZibm6vvfe97uummm3THHXc0ua4/7MOObsaMGdq0aVOT4ykkKSMjQxkZGd7H48aN0+DBg/XKK6/oiSeeaOsyfXbNNdd4748YMULp6enq1auX3n33Xd12220WVtb6XnvtNV1zzTVKTk5utE2g7b9AEZBh5IEHHmgyuUpSnz59mvVaSUlJZ40KrjvLIikpqdF1vj1wp7q6WsePH290nfPRku1944031KVLF1133XU+v196erqkmr/K2+sX2fns0/T0dFVXV2vfvn0aOHDgWc8nJSWpsrJSRUVF9Y6O5Ofnt8n+aoiv23f48GFdfvnlGjdunP70pz/5/H5W7MOGJCQkyOFwnHXmUlP/9klJST619wc///nPvQPZff3rODQ0VBdccIF27drVRtW1rtjYWA0YMKDRegNx/0nS/v379dFHH/l8NDGQ9l/dPsjPz1f37t29y/Pz8zVq1KgG12nJZ7hFWm30iZ871wDW/Px877JXXnnFREdHm/Ly8gZfq24A69q1a73LlixZ4jcDWD0ej+ndu7d54IEHWrT+qlWrjCTz1VdftXJlbeOtt94ydrvdHD9+vMHn6wawvv/++95l27Zt89sBrIcOHTL9+/c3N998s6murm7Ra/jTPhw7dqz5+c9/7n3sdrtNSkpKkwNYv//979dblpGR4ZcDID0ej5kxY4ZJTk42O3bsaNFrVFdXm4EDB5r777+/latrGyUlJSYuLs78/ve/b/D5QNp/Z5o9e7ZJSkoyVVVVPq3nz/tPjQxgfe6557zLiouLmzWA1ZfPcItqbbVX8lP79+83GzZsMI899pjp3Lmz2bBhg9mwYYMpKSkxxtT8Rxo2bJi5+uqrzcaNG83ixYtN165dzcyZM72v8cUXX5iBAweaQ4cOeZd973vfMxdccIH54osvzKpVq0z//v3NlClT2n37GvLRRx8ZSWbr1q1nPXfo0CEzcOBA88UXXxhjjNm1a5d5/PHHzdq1a83evXvN3/72N9OnTx9z6aWXtnfZzbJ69WrzwgsvmI0bN5rdu3ebt956y3Tt2tVMnTrV2+bb22iMMXfddZfp2bOn+fjjj83atWtNRkaGycjIsGITmnTo0CHTr18/c+WVV5pDhw6ZI0eOeG9ntgmkffjOO+8Yp9Np5s2bZ7Zs2WLuvPNOExsb6z2D7cc//rF56KGHvO0/++wzExISYp577jmzdetWM3v2bBMaGmq++eYbqzahUXfffbeJiYkxy5cvr7evysrKvG2+vX2PPfaYWbJkidm9e7dZt26dufnmm014eLjZvHmzFZtwTg888IBZvny52bt3r/nss89MZmamSUhIMAUFBcaYwN5/ddxut+nZs6d58MEHz3ou0PZfSUmJ9/ecJPP888+bDRs2mP379xtjjHnqqadMbGys+dvf/ma+/vprc/3115vevXubU6dOeV/jiiuuMC+99JL38bk+w60h6MPItGnTjKSzbp988om3zb59+8w111xjIiIiTEJCgnnggQfqpeNPPvnESDJ79+71LissLDRTpkwxnTt3NtHR0Wb69OnegGO1KVOmmHHjxjX43N69e+tt/4EDB8yll15q4uPjjdPpNP369TO/+tWvTHFxcTtW3Hzr1q0z6enpJiYmxoSHh5vBgwebJ598st5RrG9vozHGnDp1yvzHf/yHiYuLM5GRkebf/u3f6v2C9xdvvPFGg/9fzzyIGYj78KWXXjI9e/Y0YWFhZuzYsebzzz/3PnfZZZeZadOm1Wv/7rvvmgEDBpiwsDAzdOhQs3DhwnauuHka21dvvPGGt823t+++++7z/lskJiaaa6+91qxfv779i2+myZMnm+7du5uwsDCTkpJiJk+ebHbt2uV9PpD3X50lS5YYSWb79u1nPRdo+6/u99W3b3Xb4PF4zKOPPmoSExON0+k0V1555Vnb3atXLzN79ux6y5r6DLcGmzHGtF6nDwAAgG86/DwjAADAWoQRAABgKcIIAACwFGEEAABYijACAAAsRRgBAACWIowAAABLEUYAAIClCCMAAMBShBEAAGApwggAALAUYQQAAFjq/wN3maJGQFqtcwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.plot(torch.linspace(-10, 10, 100), torch.sigmoid(torch.linspace(-10, 10, 100)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.7311, 0.8808, 0.9526],\n",
       "        [0.9820, 0.9933, 0.0000]])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2744864"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from modules import Decoder\n",
    "\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "\n",
    "gpt = Decoder(\n",
    "    num_layers = 2, \n",
    "    context = 256, \n",
    "    attn_heads = 4, \n",
    "    hidden_dim = 256, \n",
    "    mlp_hidden = 512, \n",
    "    reduced_dim = 32\n",
    ").to(device)\n",
    "\n",
    "sum([p.numel() for p in gpt.parameters()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "class TextGeneration(Dataset):\n",
    "    def __init__(self, path, name, max_length, split = None, T = 32, **kwargs):\n",
    "        self.T = T\n",
    "        \n",
    "        dset = load_dataset(path, name, split=split, **kwargs)\n",
    "        self.len = dset.info.splits[split].num_examples\n",
    "        self.dset = iter(dset)\n",
    "        self.max_length = max_length\n",
    "    \n",
    "    def __len__(self):\n",
    "        return self.len\n",
    "    \n",
    "    def __getitem__(self, id):\n",
    "        \n",
    "        tokenized = tokenizer(\n",
    "            next(self.dset)['text'],\n",
    "            padding = 'max_length',\n",
    "            truncation=True,\n",
    "            max_length=self.max_length\n",
    "            )['input_ids']\n",
    "        \n",
    "        if id + self.T >= self.len:\n",
    "            id = self.len - self.T - 1\n",
    "        \n",
    "        buf = tokenized[:self.T + 1]\n",
    "        inputs = torch.tensor(buf[:-1])\n",
    "        target = torch.tensor(buf[1:])\n",
    "        return inputs, target\n",
    "    \n",
    "train_set = TextGeneration(\n",
    "    'wikitext', 'wikitext-2-raw-v1',\n",
    "    max_length = gpt.config.context,\n",
    "    split='train', streaming = True,\n",
    "    T = 64\n",
    "    )\n",
    "\n",
    "train_loader = DataLoader(train_set, batch_size=64, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e701c37d30d947389737b84562b4052b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/574 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32mc:\\Программы\\github\\transformers\\prototyping.ipynb Cell 5\u001b[0m line \u001b[0;36m2\n\u001b[0;32m     <a href='vscode-notebook-cell:/c%3A/%D0%9F%D1%80%D0%BE%D0%B3%D1%80%D0%B0%D0%BC%D0%BC%D1%8B/github/transformers/prototyping.ipynb#W6sZmlsZQ%3D%3D?line=17'>18</a>\u001b[0m logits \u001b[39m=\u001b[39m gpt(inputs)\n\u001b[0;32m     <a href='vscode-notebook-cell:/c%3A/%D0%9F%D1%80%D0%BE%D0%B3%D1%80%D0%B0%D0%BC%D0%BC%D1%8B/github/transformers/prototyping.ipynb#W6sZmlsZQ%3D%3D?line=18'>19</a>\u001b[0m loss \u001b[39m=\u001b[39m lossfn(logits\u001b[39m.\u001b[39mview(\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m, logits\u001b[39m.\u001b[39msize(\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m)), target\u001b[39m.\u001b[39mview(\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m))\n\u001b[1;32m---> <a href='vscode-notebook-cell:/c%3A/%D0%9F%D1%80%D0%BE%D0%B3%D1%80%D0%B0%D0%BC%D0%BC%D1%8B/github/transformers/prototyping.ipynb#W6sZmlsZQ%3D%3D?line=20'>21</a>\u001b[0m loss\u001b[39m.\u001b[39;49mbackward()\n\u001b[0;32m     <a href='vscode-notebook-cell:/c%3A/%D0%9F%D1%80%D0%BE%D0%B3%D1%80%D0%B0%D0%BC%D0%BC%D1%8B/github/transformers/prototyping.ipynb#W6sZmlsZQ%3D%3D?line=21'>22</a>\u001b[0m optimizer\u001b[39m.\u001b[39mstep()\n\u001b[0;32m     <a href='vscode-notebook-cell:/c%3A/%D0%9F%D1%80%D0%BE%D0%B3%D1%80%D0%B0%D0%BC%D0%BC%D1%8B/github/transformers/prototyping.ipynb#W6sZmlsZQ%3D%3D?line=23'>24</a>\u001b[0m loss_history\u001b[39m.\u001b[39mappend(loss\u001b[39m.\u001b[39mitem())\n",
      "File \u001b[1;32mc:\\Users\\dlaptev\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\_tensor.py:525\u001b[0m, in \u001b[0;36mTensor.backward\u001b[1;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[0;32m    515\u001b[0m \u001b[39mif\u001b[39;00m has_torch_function_unary(\u001b[39mself\u001b[39m):\n\u001b[0;32m    516\u001b[0m     \u001b[39mreturn\u001b[39;00m handle_torch_function(\n\u001b[0;32m    517\u001b[0m         Tensor\u001b[39m.\u001b[39mbackward,\n\u001b[0;32m    518\u001b[0m         (\u001b[39mself\u001b[39m,),\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    523\u001b[0m         inputs\u001b[39m=\u001b[39minputs,\n\u001b[0;32m    524\u001b[0m     )\n\u001b[1;32m--> 525\u001b[0m torch\u001b[39m.\u001b[39;49mautograd\u001b[39m.\u001b[39;49mbackward(\n\u001b[0;32m    526\u001b[0m     \u001b[39mself\u001b[39;49m, gradient, retain_graph, create_graph, inputs\u001b[39m=\u001b[39;49minputs\n\u001b[0;32m    527\u001b[0m )\n",
      "File \u001b[1;32mc:\\Users\\dlaptev\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\autograd\\__init__.py:267\u001b[0m, in \u001b[0;36mbackward\u001b[1;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[0;32m    262\u001b[0m     retain_graph \u001b[39m=\u001b[39m create_graph\n\u001b[0;32m    264\u001b[0m \u001b[39m# The reason we repeat the same comment below is that\u001b[39;00m\n\u001b[0;32m    265\u001b[0m \u001b[39m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[0;32m    266\u001b[0m \u001b[39m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[1;32m--> 267\u001b[0m _engine_run_backward(\n\u001b[0;32m    268\u001b[0m     tensors,\n\u001b[0;32m    269\u001b[0m     grad_tensors_,\n\u001b[0;32m    270\u001b[0m     retain_graph,\n\u001b[0;32m    271\u001b[0m     create_graph,\n\u001b[0;32m    272\u001b[0m     inputs,\n\u001b[0;32m    273\u001b[0m     allow_unreachable\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m,\n\u001b[0;32m    274\u001b[0m     accumulate_grad\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m,\n\u001b[0;32m    275\u001b[0m )\n",
      "File \u001b[1;32mc:\\Users\\dlaptev\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\autograd\\graph.py:744\u001b[0m, in \u001b[0;36m_engine_run_backward\u001b[1;34m(t_outputs, *args, **kwargs)\u001b[0m\n\u001b[0;32m    742\u001b[0m     unregister_hooks \u001b[39m=\u001b[39m _register_logging_hooks_on_whole_graph(t_outputs)\n\u001b[0;32m    743\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m--> 744\u001b[0m     \u001b[39mreturn\u001b[39;00m Variable\u001b[39m.\u001b[39;49m_execution_engine\u001b[39m.\u001b[39;49mrun_backward(  \u001b[39m# Calls into the C++ engine to run the backward pass\u001b[39;49;00m\n\u001b[0;32m    745\u001b[0m         t_outputs, \u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs\n\u001b[0;32m    746\u001b[0m     )  \u001b[39m# Calls into the C++ engine to run the backward pass\u001b[39;00m\n\u001b[0;32m    747\u001b[0m \u001b[39mfinally\u001b[39;00m:\n\u001b[0;32m    748\u001b[0m     \u001b[39mif\u001b[39;00m attach_logging_hooks:\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import torch.nn as nn\n",
    "from torch.optim import Adam\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm.notebook import tqdm_notebook as tqdm\n",
    "\n",
    "lossfn = nn.CrossEntropyLoss()\n",
    "optimizer = Adam(gpt.parameters(), lr=3e-4)\n",
    "\n",
    "loss_history = []\n",
    "epochs = 1\n",
    "\n",
    "pbar = tqdm(range(epochs * len(train_loader)))\n",
    "for epoch in range(epochs):\n",
    "    for batch in train_loader:\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        inputs, target = [v.to(device) for v in batch]\n",
    "        logits = gpt(inputs)\n",
    "        loss = lossfn(logits.view(-1, logits.size(-1)), target.view(-1))\n",
    "        \n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        loss_history.append(loss.item())\n",
    "        pbar.set_description(f'Epoch: {epoch}, loss: {loss.item():.5f}')\n",
    "        pbar.update(1)\n",
    "    \n",
    "plt.plot(loss_history)\n",
    "plt.yscale('log')\n",
    "plt.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(gpt, './gpt_wikitext.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_set = TextGeneration(\n",
    "    'wikitext', 'wikitext-2-raw-v1',\n",
    "    max_length = gpt.config.context,\n",
    "    split='test', streaming = True,\n",
    "    T = 64\n",
    "    )\n",
    "\n",
    "test_loader = DataLoader(test_set, batch_size=1, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 5199,   347,  2852,   353,   318,   281,  3594,  2646,   837,  5581,\n",
      "           290, 21421,  8674,   764,   679,   550,   257,  8319,  2488,    12,\n",
      "            31, 20495,  2597,   319,   262,  5581,  2168,   383,  3941,   287,\n",
      "          4751,   764,   770,   373,  3940,   416,   257, 20495,  2597,   287,\n",
      "           262,   711,  2332,   684,  3194,   416, 11288, 37072,   837,   543,\n",
      "           373,  6157,   287,  5878,   379,   262,  8111,  3078, 15752,   764,\n",
      "           679,   550,   257,  8319]])\n",
      "tensor([  347,  2852,   353,   318,   281,  3594,  2646,   837,  5581,   290,\n",
      "        21421,  8674,   764,   679,   550,   257,  8319,  2488,    12,    31,\n",
      "        20495,  2597,   319,   262,  5581,  2168,   383,  3941,   287,  4751,\n",
      "          764,   770,   373,  3940,   416,   257, 20495,  2597,   287,   262,\n",
      "          711,  2332,   684,  3194,   416, 11288, 37072,   837,   543,   373,\n",
      "         6157,   287,  5878,   379,   262,  8111,  3078, 15752,   764,   679,\n",
      "          550,   257,  8319,  2597])\n"
     ]
    }
   ],
   "source": [
    "inputs, targets = test_set[25]\n",
    "inputs = inputs.unsqueeze(0)\n",
    "print(inputs)\n",
    "print(targets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Robert Boulter is an English film, television and theatre actor. He had a guest @-@ starring role on the television series The Bill in 2000. This was followed by a starring role in the play Herons written by Simon Stephens, which was performed in 2001 at the Royal Court Theatre. He had a guest\n",
      " of\n",
      "\n"
     ]
    }
   ],
   "source": [
    "generated = gpt.generate(inputs.to(device), max_new_tokens=25)\n",
    "\n",
    "decoded = (\n",
    "    tokenizer.batch_decode(inputs, skip_special_tokens=True),\n",
    "    tokenizer.batch_decode(generated['answer'], skip_special_tokens=True)\n",
    ")\n",
    "\n",
    "print(decoded[0][0])\n",
    "print(decoded[1][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " had a wide long or the north @-@ lane of the state of the south of the west from Europe to the same\n"
     ]
    }
   ],
   "source": [
    "text = 'Hello, ML&DS Community! What'\n",
    "\n",
    "print(tokenizer.batch_decode(gpt.generate(\n",
    "    tokenizer(text, return_tensors='pt')['input_ids'].to(device),\n",
    "    max_new_tokens = 25\n",
    ")['answer'])[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}