Bert/02-Bert-Embedding.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bert Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABAAAAAFUCAMAAACN5L73AAADAFBMVEX///8iIiJdXV3p6en/8swAAADZ2dn0zM3z8/O216j29vY+Pj78/Px4eHjU1NTm5uY4PTZAQEB9k3O3rZI7ODLWy6tRUE6nxprhvL3X19dCNzfv7+/9/f0BAQGIcnP5+fnc3NzY2NhkZGRZWVkkJCP19fUJCQnw8PAFBQX+/v6IcnJoaGjo6OgSEhJVVVV6enqdnZ3yysvNzc26urqrq6vt7e3k5OS2trYmJibi4uL77skwMDDe3t4MDAxwcG8qKikPDw7Q0NAoKCey0qRzc3OTk5PvyMgzMjFyhGqiiIm11qdMTEyIiIgtLSw6OjrDw8OwsLBbW1vx8fE8OjU9PT0YGBg/PTjT09Ozs7P+8ctBQUGpoYg7Ozv468a9vb1lV1dJSUmAgH/MwaNRUVHkv7+sy5/CwsJ2dnYeHh40NTRBOjqz1KXCo6M6NTVsbGwxLixea1jg4OA4OTjy8vKFnXvrxMWkpKTZtrZXVEtKU0bBt5wbGxr6+vpbaFYzOTHLqquqyJ2RrIahoaG6sZXeurthYWE2OzRGRkZ+fX3IyMhEQTioqKienp7j17bSsLGQkJBqZVhOS0PAwMDw5MDq3rs9NzdPRUXGxsbt4b5fWk6fl4FDQ0OYgIB6j3CQqIaYmJhiclu7np43NjZWS0vLwKQ2NDFPT0+MjIyYkX34+PgyNTB4Z2cVFRWdhYVxYWFsfmSCgoI8RTeCb2+mxJmbm5vMzMywz6JOWUmXsYyPj4+qkJCPh3SwlJVqW1yiv5WTfX19a2tFUEBFQz3nwsKcuJHTyKmQenq1mZmIgm+FhYXPxadXV1fGp6iWlpbm27mAemt3jG5neV/f1LNeUlL06MN0ZGTHvaFKRkIuMS1aTk5IQEBxbF5gWlrLy8tFPj5VUUW0q5JSXk5aV00/Pz/St65TSEiBlnhETEF8dme+tJp4c2Ofmo6upo2VjnqLd3fVy6xXZFKLo4F1iG1jX1M9RDt1cGFugmY5PjaZi386MTGSi3jaz69BSD6Ff22GgG+CfWy2kVifAABW00lEQVR42uycX2gT2RrAM1vsSNc/7Mtl2mtjrXG11u3u0W2bYm006JSh62VZFEUuThcD1w0zLauM82Bh4BKposwFodMH8zKWtkg2QcL1YcMMDgQapLlPIUgCNy+XvvY9b3cmTdo0SdfMmaz/+v0opaX06znfnPP7vjmdxOUCAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAOAToW+w0w6DO4TZN2onyujhxlHabI3leOMge9vtBBnqcxqhPKV9DYYy2InFyKFG8zp+xE6MtvoAHT1Yo+k5VhfpoP0oR9oaXqzBI50Oab/bIOyhIbxgg7tPAD2TfFfzFCd7GkaZeey1EaVrermhAS58ZSdIcXWkUZCO3mkbQfjxp/UR+u1EKON5PFOf24liFw7Tww1sMjJgJ1jxxoG6ab304Y1mrKP2Oj23P6/is18bXKwhzAxVD+9BfSE4NDyOFas4277b6v/IRDZENU/ImGhkgJknPGcjCsUF/9NRP5aeG3E7QUL6wMEGJeUfOclOEKZwpqYHGLyYkynbcMUntbt2ZDISonBI8PUGOLiqi3bm5Z69VtPWvHkrYY1GWnv407ZI8zfC9uclGn85UG/I1axIOURm+2uXwb6nPIMVK2TsUOA+W0YnDYG0gxBpYICZe0XVVhRSCi7XFcxrN9btjSUQO1rXA7zoVwL2hhLl57cXyimFIjFQizU9QM+EkSSxQKYBau4ChgZi9oLRJ2e3bbiONwUZbzRkSBmrntmFKysIZ1LZ2dqrdWpSp0nHJBdqeoC+YT6B8GIJxuRu6gH6RpciSfspqjXAzHc8ZzfhXKGmB+hrfxa3uxoE/dapuuot2gxCJ64MV0cYsx2hvMA5/kn1PhmajQi4axrV9gBDR2M2vYZo97OqHmDvy7cywh2OvPZwa4vNfxXG27VCfGl7D9A+EEmSLSC0vQew6j/2VGljdhf1AL9OYqxRIbLUU1P/edV+wqW55W1F7oDd+l/uAYacV+/w4qOtCGNKCHfXqnxVD4Bf/xv0AKfs1v/SharqATp+wK7/pS2WH6v46PT/VnCrNspOVp8DtE20ov6XlsFCb1UPcOZ5AuHHEoyJ3dID9PUsYRm4RpI49d8iXbi0dbzc1/nsJM5qEPSlturqncGp3nTib8NO639tDzAyG3G2uhPF4U2bjKzarf8b83JXTgKPPcyLyMlo5IX9f92o/1+G8eclxCe2mpL2W0aSbBEU21vpAWaeLjKOpkobz3ZJDzA0idmjJo2qHgCv/pd6AH55M8qRG268sVT3APjVO8w/qkQQHaydrR5gdMLp6kYaP1w+nWwb0PGCmT1AZ+lgZDkoOdxiYv7ivtL/aVaceI3e6gHallpV/0sGWOgvH1SeGU8gZ7GSu+UkcL+Ou0ZpfWzz5Hx0UcVNOHd5tBKlF7teCrk3lSCP89jdOzPZ5ixCpQf4ZWPtXHS+uhHzvHxK8kMugF3NHlhCusmLyOlwxKDZJd1dXXE2LzpWOU/8lhXIFhJQ/r0R93nU8VTp2NSuEMDYCnaKor2bN6g9Qew9Q3V1VqJ8wWCPxf1tJch3LP7y/vKgwwjlKQWPlIbSH3a+ptNzZQH8HMeOoQ1YD1xcV1qwxRZ+dLmOT3AOozC95SdApl6TLSX2+0ZcnnMei/kCBAACAAGAAEAAIAAQAAigNWfuIAAQAAhg1wpgpP9i27sU0dYJAgABgAA+SwF0+n8Zeddjub4xEAAIAASwWwXw9AQIAAQAAviMBdDz6KfT3/9oPZ114dXg03vXj1uPwM/vdbn2dc7fPbh8YmD+OAgABAAC+FwFsH98ynP1G9/jGdfEnaXbV38b6HEdnvKZQtjbP37h+6vd55+/AgFsIatCSwSABAF9bgKg02kaWwAh1bxmtEA3m+JmBICktPCnCODdF+/TEUD/nnP77/Wf8w257uy5/a+bY92rMy+OfnPBFMDR2/8d/fnE7PVBEEDVNSW4lgggrbuTrRNASP2Dl9XtJIBQStNE64sk1XjfipqmhWwIgJrLB7AFECaiJB1mUyQZIdQWCUDI8NKfIQA6posOBIAkTVOtq492yrysaSnqvQlg6rBr75XuA6473ZdmXMf+/vW1LQF84DMAs0xaBheoAG19U17jKBkwvzR/Rn/CAlDnWKp1AtB4t30BaHxeSZBIZQuFvK7RZFgpkUKGoqzpmkAmlPx46r0KwFhkPjYBJNHmwjMbFOvDjJufkxwIgM7yCiuTdKqU+RSNcqXE66KcUZSFuIRIRimMc+9LAN3XrcW+57Trznnrn4K/n7j30QiAiq3FzTxlugpZkZTZjaQn19e6cgwts2vMJyyAJCfTLRSA7w/et2AnASQKadOh6uW38ZVIYSFAGr6IYZJGuTnDWPCumys/Pae9TwEgmQt9ZAIQYrJolMs9ikbJqPVYP0o3dWuxowB01rqH0HyKmflgLomm81biV6g0n4tneT5tVjeOV9+XAL623oCuf88r1x3CenXlve6XZQGsfnABiL6chpjbXQZL5EX1fqka0ZH7ejz/20ooejvbtAAQx8iqOxsNoXQ4FudoUmRSJa2rjNisAFCIyWajmZIAQolINrzRdQc0Q19JMbJNAVAJlSaRyMSz6ynKlgBQQFuPrHBJIZWwtpvIpEk5681FGS1p7iBmc2BNCMDcHUgPmjfutJgSSGORok0QymUoWsxYOZTtCkDSjCxDbeQlHlvnaHsCSFuXY0MAEmP2IGaazCjWyQIKJNxZtya+WwC0lpATRsRKTkkASArHY5GomRQqoZUab4mRmhYAl5fVYPlmLZBh0FrUiqAlzF4goYmbsxXUuB5PMWqzArCCZApSKfM0mo7Qpcyn58JIUMcjVqLfnwDObQrgqnW3f6n7n6YATpuJvfXhBTAdJiUiZy4E9Xy4LACqmDGTV+iiyWLzAqCzfsVHEAS7Pm5+XuRI6WzQWksC729aANKah/Cav81Z70bj8fuIguUQivV6/ISHeG33DIBnBVJWPF4f4d2hfd9BAHLG6/ES/jC1ELSskyIMMkpY5GVSLXgWxz15zoYA2Ll0+XtjcWOhmwIIkMhtFU+7AjireAjCy5rzETP+ad57NkrbEsBJX6IsACnojQrWXM0o42bVFVmP57LHw75bAIH8dN4ahS5sCEBgCcLv9QTTpKictXaVwFp/pUkBvGaF17lK5udkKmj9MVqZDplpLc+Wtl7heJnwezwEa0MAdK7yemhTAKic3bD1ZmeZDyWA7psu18zkiWsvHpwbNn/qKwlg6gMLYP2+lT2UTpYFkFR8mixQIWRPAJ5FN5dSPJd1TYoRutnZ+a1eLuHRUZMCEDLemCSqb00BJFl/VhTD0wWRFAzvQjqU1rEEgGIetyxy+g4vZ20sgGRpHKncSpUAAq/9Men/7J3Pi9voGcdHXVqFkrBXbdKaHMRmMnShb3bTaJkGDaaREa1IKO0trCgIFoxd0RShwy4IFh2yFFQoSLkYijCmFNWm6LbCxgKDRVnvSZhBhvqmq/+Gvu8r2zPjGU8leZKZ0vdLDkM8fubVo1ef53mf94cMS7CO7YAxW1Ks5AYAHfDNUJxAd9Ihl7ahXAQAoCezEhkA67SMfo2dQP/yvsm49ZpVDgCNY7Ul0NqAr0Ardk2nF2zPMCdeLw8AWKdrBDYbZAAQul3LtEKIJTBkUZcxqrNRPgBYnlefo38w7IOxJ/M9+XDg6ysAsMfwaptsgwYiN490q1IIAHSXt8NgwgAIgA5yfM9EABAi1b8uAHz/N8/f//N+7e+3/7n/yzdff/tjCICv91/95d51AgD0qGVvXgKAdtX9agqjQzEA8Oh5H7JoS7zOzWAvlNoafJKlKG8NIFJTZlkDcGttEx0mBbusZWOMi4UB0IAAEBwWxxOlSAYAG47aoeinALCqAQy5FkChhdfzA0DotqsU3+4DOuR7HlRWA0j5Ll0cADZqkUeJdKMjo4dsSImlAGDNWHSH3doAjbUXVEAPsCVg5gEAHviLrAxOagBwTH3M0Cavws7kb9ysSwAQDvipV5VDDIDQdsL5LKysAYCNh7BhShv3I6MYALRWCj2fRACo8yl0/ALXAHpNfFvfDQD4UwB4tPdqnzs6+vDlG3QW3v7To8OXEACP+f2jJ9cKAJ/CNx2sAQAU17v7UVxwCKCiL/fZEPWQmg2DC3r4XDVW8gJgyOFzZxEARNYH+NGb0i6HUwix3BBgStmVyNCKFAHBmBtmvLgAADIro0e4frZQeTkAaKAxjUqzqZ+qAahOMnWFEgDARcAWdIbIHqOWxJRfBgBjR8WHfHVZB1lJqAU95mpeZIxAHgDM0d03OqmSAUARZVuVWJiv0T22Bax53coJACBETW0yswSApwDSLhOL0EErAMwRibvUmLbqqVkcANjzBx2bOakBNO12XMEHpbwTADz45MWtvTdfoW725qeP914dPf/uF8/uoy2C9//6+XeP/v2n9/b2Hn3zqx9c6xBAzHrztLsEgO5BX4+8D80SAIgyADQhAGCHmwKP6uadBRD8qriaBRizuKNMqJgOKI8unQHQVsJRnN2zCgBACKvLZl0AAIdt1rGKAAB3xQU1OVUDSPRsUAJKAQA6A4wpKWvJuAwAahS+NFCh1MzKkNZ7LMV25EluAFi2Y2IAjAZVqT0NmwgAkZQwYnXzpKdLagDDGLRWxwTpTsPK2HEaACIEQKODA0lRAGAX+5R1qgYwXE05vhMAbOjV0eOHt5c7hB/+aLVV+OHD660BWBRMvYXgaLgCwD9CDcbBp8yOANDrdnTY1PICAFS4Lr3OAA7woIDq0S67SwZAC0a3V2flAjUAsOBaawAYGwCIOVfIYkluAIz8CI5ClR4cZoc1czULoKzLXuUAAGO3f0FL8gKgnrAopIIWt1hbEaxg6rBzLXcGUEs0DIA+m+oC0B0EACXmA0ea5JwGtOR2p5k0O21/RAuL9kxKHS5tG+cBYJTJAJQwgmM/ZdBBGYC2mgU46R/XAoAbdh4AAgA4eDr3Bl/Udfe3M1mWx8zgI2caf5EIOwJA8Lh6lsrnGwJ0ORw4EAAiHg9wKzAzNdSZjvLyUhkAQH8duJI9KlADEKt4XEyDUdKZnAAANy5kF6DIOgCUH7dReEwkCJCQi6GD5cYJAEpnANBFsVJiHcASACLTptCZw/0TK9hVVke18gJAZHtZDcCnUJ8yMQBg2+ZcW8gJAKa7OPT8mt/qa7QQDdO05aTDlnkeAKZziNrgFgIA46gJ8vxCAKqNHB+a1wyA7718cAMBAF2f1B3IYEtOoXwNdJN62ipaBDwHAHoiUfYk/0Igy5ZEa7KoQgAwSXVhMv1ax6K1HpuKYsyVygCUeGgwZiA5WgEA6MfcwjKNcCxMKc8yujYCgKHWJ7olTGx1DD8L4gJFQH08SOIemtEMBlgNejjUds0AlBhV7S23F5SbBTBjCkKWSVgPWol6fVoODdNs1JpMDgB0AssKOryRzQK0qBRNteAaAK11qGo//0Ig19Hd5pIXSoJWAYCLhgD0kLK74pQvBABgYc+7gAZT7Hjf1MPoOgHw6f3bNw4AQwEHu7NPCBihlSHCjgAQ2tQFx+RuBQDwOWnerPKoJOEe8k5ak1DjLIdjuZpDtUpkAIpTradO7VAstA4gqFVnjs1VgMtx8zovIQBobbbuxLrQleBndV7NC4CsuqWMLm4AEIyyAKAndc525odcuVkANEJjfY1u2MiKygV0zNqO06wu8tQAqMPZXJIOhAwAVpOrz2pVDgMAhNT5Q9a3A2AoC+tVALptmLPsqcQAUE4BgJGrHMfDDpUXAMJ/8/x1AOAmaBMAXwy2JZLG3SMv/0rAIES5ozFFAUnzceYvtNlGkaXAgii3e5E41fH8UJz0sjs0ilqBPqb6BQGg+10BGBUYA7wJKLYU2PLleIAWoPUH7YE4wdekw/+rMPizRPbPLsDbCgBplvYvXbObzqpFAKAd4BcpNabIM0prEMvTYJQXAO60QYO+B3OyYIrvVejBBIUZQiseTMGX4VLIA4COP5Cn6GB4oeXrNDCm7SRsjBcKxjg7BLkBAMQ+La48oB+MrHE2IhHG4YjWKosRvloUWoRGV7TczTmPrQBQ08t3EwXQ8wQAyK3R1rXcIzeydtoL0Lgw9b5sL4B2sncLKMuf0QZWwMQbhaW85wEAbVsMuGwvAPyWsJxG0k6yoqUhTdncqroNAHoQBJe+m8uCv1BkN+CGtsW3YucBrK0IiiLk2QugzOa6cMYHArP+5qij6oX2AuQT0AQAhJAVcwEANIIgunQvqAE9zxAAFFNBAGjT6kWvayh6IEiQhK1hm9t4TR45EOSyxOLtHgiyLAJerBbl0W8BAPrMG7YG1WMmFwBKiQDgigFgzZ2Lwl9RAPQ7Ei/VNhM6AoBrBICz/WVsStx03woAahLPq8nm+ygJAG4wAJRoQl8BAAQrCqJzO3sJAK4PAKDR2Lo1Wms0tLcBAKC7Qf/8kk4CgBsMgG1PCzkT8H8cACWeLfJmIAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACgLes3zuVsmqfAoDklbUSUmsAfJuUNXIwOwFAs/QFeUsANA92UygtAZBWdlZPWgLg9ay0jTgDgHqwu1QMgMGOVpI1AJyDK9XdJQD4we6eT/4/APDZ53dK6+P10SX3/lbeyutbKysvdmjL85WRD16XN/IVftg++N2dHfVNdqrjx3euQM++zK7rzQ7OeYFI/fN/XUFrXv9hb+/LT3Y282TZcz67c7V6/Siz++wqjD3ZIyIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIlrq4Q9vbeqPZ/Tr03pwRvdO63FevZdbPzmt97frZ5fo0636D3vnE9NE2sfxIlNj2OpmA5qQhcACKqQiRGDBqQg0mB6AFcSiYx2CNXkph1eiJoa0nGwI+KfhzzSh1MMmpkVIOOxBD4QbSROosSZNMx6IKcGLEAwHwsUG952/nT8FOp0Z0BefjzE1rfPNM89MP/N7npnp3N+dS3uQL5GjQv6gyN6VE3twY3capHJBxJs3bz4yPGEY53OV4T8EYxydfCo4qkXc5NMqpFLEMz6PRDzm8ZyhvLz8NZ+/Gd5xXOEzzDBL84pPs4hFjn/43GEY4lHGwF6k283RyOeyiHs89CIKRAxwHDkyvg8CqBpqK+TxluN8AndFTCbQJKIrgcHBQfJvnBcct3j8RvKJ4heS6xS3efxF8oDjJZ+eOCYOQwItIuoSMHLUcBTxSOOw7YyDxMwwI6I0gXoRtQmcEVG8M7kCtID/ex7tgwD+7AH9CgBw5CalOBXOJKU2Keyx4NlhFUCyDt6rxwRHS+GxVHisNQtw8Ek8aseP6vEjffzwz9UEglpBUEoICw2i+ughyhK2VqFqF7aQoSobqsihyh2y8OHqILY24pdMg6KSSlBxCesxQbkmqOW4Oo8t/driHBEwwKcgAX0C9wRcTkBQHHcLYWvpsjhDAtg6XFCcLwppTkBQ8r+anR2+kox3ifzN43Ui5STP+TxO5FEiz4g/j54lULkzF/dBAP3ZxGiQHRgKRo3MKJMdiLKDVWosy45seWNedozMjqPZoTY7GI+P2Inx/A7j/aoD4feknE3KtaQ83IFjSjidlAwVmE5KXlL6VUCXlJyklOwBmPgDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAODj+HDilgEk9E3NOQUxTOtuY2Sb5IZVsyNW78ptS9pAJSb8nP+TuH2xTXjedUsyknnmGRmWX/M6Z1TEtyj8ivyUF93n7zT35qzb5hJfTMHlKLZpm++OxehV6vundzyCABv96pny+9rI7lt8iN8PSG38yUOGa7JZsck8GGpUd4qWfDESuz0vZ65OZVRHfDZcyFfPFzz4abEJ2xvb5PNaOWfJbYrrA7TYl/jnZOU8reftfddZypkpsF8afMKMxfVWet/ZTPBmooQOF5GNtZ78wT91yMzxPuWcDWmS3ZI4TwIjsEBcngOuY7JRRTgBeSDGBDlYAC7IznJwA5uW3pJ0vgA6r3BhkQiCAKAyphJMngHar8jwLEAAQABAAEICKnI3/WkQD11bNxQsPgQCAAIAADr8Abp5hb8st5f3kyOXiMSAAIAAggMMvAPLBtd3a2uPHj//zKxAAEAAQwE84B9Cf6xC9AwQABAAE8NMJQHej8bapsPo0LYCM8sLmjJz87t9MbZ0ZmpzHA1cbX/5y5z4QABAAEMDhFEBDXbHhU/3MlRxSALrXpcYxTXZPseFFvfmxRqcvNs68LM0d0AEBAAEAARxGAZweLH6VkffRVptNCKD67zM9N0ryzhcPZ+Q12GbSdfrcW0fzGtJsDUAAQABAAIdRADdsD84RL0PaK4QABmZaxomdzmYgL4pt1LYSFcC7Es2xtzOdQABAAEAAh1EArfUF5IatyNVrLmu1WtNRjaaztJj81dx6bbNOf4b46k/rgQCAAIAADqkASvXTlAAGCAH0dOVeztFUl9roX4qu1OlriYpg+h4QABAAEMDhFMBHR1cV8TKsHSKGABW/GmsrcxrMn8i3TrTeBwIAAgACONwCqHpQezNPd85QXEGeBch5Xf/g0sMHZyrz+qtu1Xd+JwGgbhZYsQDCdga3fAFYmQgrqlgAHrY5YRkCgIPs0ogCAaSYIkEAGM4EBmQJANlgG4QqFICLDQqpIIAYk4VjagkAk7TxD1gAmpulpfoyQ21jP30dwIBW3189U1/Q3ZM72P+dBLAef4BQQLEAvrAP/FmWL4BRJmI+plgAqzVM1jckdQFgK+zKwAoE4O5jQupglQQQzGISNxE5AoBH2NUKKxSAlw2yqCCANSar166WAHzsYyXXfggB2Hqo1+nW22Zz0eJZjWYx7YlGk20wPZmuvu4wOxrzNbrjxjcaTd6QYfxABWDR2lZoXCoIYGWKIqxEAN+oCItHBQHM083xQXIEULdGLexFFAmg5XMKKZIE0D5HJToheQIoWqI7BVMsgD46KKaKAOhWeV3qCeAp3Tz7jyAAjpz0+8dEb5VUXbr2/S4FtmhTHT7vJYApxXMAo2nSa7akApCyajsLAMFW/D7lcwDuvvagunMAwaw+j7Svxi4CMK6qMgeAeNPm1JsDWEteNKYsgIkfaA7gB/49ACCA3SoAIIAkFQAQABCAWACffRQeJQJYpiICERUEsEI3xyVLACYvvbSyIYDfQoVY1RsCfFilEmMyBVAzRa+WwklAogLYooPcqgjgK5UVxNQTwCjdvDAQQBIB1ExQTKEqCKDOROFUIgADFRFVYxLQSDfnq6xJwKJeemllk4BMSrt6k4A1dLOW5E4CttDLu5QJgKgAmI29rIoA6FZ14OoJgOmnbSCAJALQ5lJMRFQQQPQ9yb8nlQhgi8r44lZBAB1U1PtVeZOAI9TC28oqACZlQb0KwL9GJVpkCqBok95GHsUVwCgdFFBFAHSrFmLqCaCd3vjLQABJBDDhovAgKgjgC0yBKBFATGKGlDkACVG7zwHg9NIK5wB8KaRImwMISUrcdQ7AKbl/k80BSA+SMAeQSqukzQHAMnoezAGASUBwFgCcBVCVqgqGznTxOcEbY7v9NOg4u0PpToyd7b9w9dhPKgAUPixnARA0gu56fBN9Jk0AMKpMAHTf7p2SylmAPZK+qwBS6fn9EEBnLnOpXW2r6JPTbebdbv7vKc6h/5ExYB6v6jJmH4AAIlbySkwksByAIZiZj4Vjy3gEggO467sIAJlzqiKAgBNWqwKAI/IEEBkZWQpBSNgytb6BQCeXCTJd6Opypg+F0KWRrVjKAghuKRPAexzB7UhwDVGpAlhdUF0A7syQcgF4iJ53QzDu9TqJl1Wy51fRCNHzQaLnR0a2wvsugJZXFMP5YgG8lSaAh0MFF/dPAI6nNBarw2+HnL11WY6oO5SWSR0Ho0VZxl67Z9OxJkkAHX0UuBIBjFIRS5RxkL4pJQIw0c2xIJYtOOUKoIZuyIro+2E3pCQAI50ygWAmCx5BvO3Rzah/HVowraysTNhdWaMr8x82YBw34VIF0BulEhcQp0nWJOAHulNCUIcFmVuAM/2IzAqgnQ6KX2rnHVUggHkqazNANDEUcZNNCkVC0MaST7YAeunmTUEh0zqOYiMdE5sfDCjaMU/0/BIWexklej4M46um4L4L4O0un0gVgKY/r2QfzwIwLFhb7Gi4aCLg8jq2ww5Kyetp6yFrVhRx921JEkD8XgA35iaKBgwPwMT2JLcmJlkA8XsBkJD9ZJQQgAcPUO3dWHWF0JQEwGQtIC7ioBsJ4u5UKgBm4SIYQlx4GKLWA3MjFmM4jJJhmCQBxO8FwP67QazK/PsQGnJaoW9RDwHq8js9vtHPxP/0SxZA/F4ApwkN+MjOjfh8mPTTgPF7ATosUCwGkQJAXRgEb1C39USCgqs49qoAGNYht8eDx2gBYEH7SRiKhIhvOMrbWKncC+CZwskzHMiczwtFAhjijoTsYcoI9pMhTLoAmMjPUCiLOMYvdyxHIrEpGG33Eh2PITGT3YN/2CZW2H+AAshoPlJ51/aiPL3ZYCtsIARQutjl+DRMTgTkl5kct66cJe8XGDT/9aqOEIBubNJm6G4yj18rG7ioed7W2mY0Fhwt0fRXDzpMd5oHf9ekLxrMJv2JHHVuB7b2BqE5P3nktawzAvAWrUegoBdCJ6QIgMdI1v+ou4PQtrE0AMDq5rmUbLqhtDmUJmQ2pK5DJilMp51UHsdu6OLDbtppvFWqqoJsCtP0MA1toQjbpzElU++KkWPD2p7DQIg7DuTQQ+dQdBMYMi6VQQjpYIqM5yIbGR+MLjFx9sluWidtd2zFG3beKbFjWf719OX9f9574WR3UZbdm1Emi4JoItciAE19Ne4Oyc4kmvMonEKDiO6hMHu6DQCaaaJAmgspK4GWRwBNDf7yULAaT8ET3YyJ8nBISc/LcghrXuH328uB6wDwWKZxe9UUHjYIgBegah5tHYDmzj+hyCcoEfdyITmYag2A5gYBULMGAGFdqdIxTvHE8VVPSPFIrYwAmlo2wcn2JAkBELMyhSnhqr0A8KQcbhmAJljK8WxmWwApIqZu0OXQajTGcfJiDqCEWwk5N1sFoKnVASCwXP1NIQAw8BEIQBqg2Qp6IACMD9abH3k4bvUdumkdm1u/OTEwN3L65YDDOe6zfteDzKxY77485Dh7BPlq3TV3xWWFAFydsh77weV4UwO4bPX5lny28fPIkwXX3N8dY5au/qNW+zWn7eanHQMAjVGNa0M3AIhgC7KaRk0AQEkCgx0XWErUuD5AhOi2AShzSVoIJSMhlk9RG4BIzEdynHkANkJV/kNrQ38TAJzgCny6EGkAQGpTNB2pZKsCkxXbBgBV7XlVi+KgNm1kF146USBTcgaYAQAPOHOCFpSiVEYox4rmARBroXmwoaRpFhNVSuAloV0AFElgOQECgFarfADzolQRiBQLTAFQqzEEDcoqm881AGDSVYbBy8E4LyimAaA5jNmQYNLvMdJUJlx1r5LzHHswI4CxlXobggAMjD9EXlh855DzE65eCMDTLuSk3XfVf9Y22YP0zjnOdbsdN5C1n2y2fv9t24/+kUfWHQAsd7uQ+3brpSMP4E/4n9ssXYOH1s8jg39wvu4YAKQSw5sBAGGt4nEVTQCwDe+6ShhI7igZ3IxQJbxtALQEatQAytPw8pXyYhaeQUo2D4BmVzUBNwEAWskYL4s2AEC9TngewYqqUqFI2wAAsqBWPLCn1uRCoeDl6QTGeZgIMDcCcMOzD+WqE4yqhijSLAD2Iky3UDmkqswwH0+oOQG0CwDUh59KGwCkVUqe1oD2De31VM0AQEpqPCahpMQQTJqsAxCvX//cIuwOMdMAgEg8Sy3GYAqwBSMvkVV3EEY+fDAAWK/X21MIwNhXCLJsnetG/OOOP51+6YLf9jyzver9dWH51KlT/7Kc/bNvCeYBZ3y2/rWFQ/DDnb/5FoDvjdvF8u9Z31OYM3Q7LV3+u9ajP8/cP3mkcymAGjRugkK8WgcADcARKp/x8aYASEIAVt1RwHLHv6FBuwDgBIYbAKQX4WvjVJRR9wdAmKhgoVUTAJCVGtgDQBrbIAgiQLYNAA4/Uhgm/Xgt33jzRMmbqr+rmREABIBXcqmpEjyZAmoWAGcCK+Mil4UHIcRwvOIJlU0DIMjZQEDWoEulrC6aAUAkqEw2h4rJWEYPiDsAJPOi5sbNAYDXAcCNvW/i8FecR6s/WnXHG5E/0BrAw3Hfz/CCOB4jiP+aY/b0S99h+Ojn1juzTtu6z+cbs4zfcN0zLtwxW3/vwA/GFgKjOwDYbiDG9sHLn7vu+OFX45aukX8uWAamRr/2dw4A7/Uk7JZY7A0AWRg7nFig9wVAGcN00DYAIJAQQDSUFBIBIOoMynIRksDMAyCIIBzcMFMDyMSiAKWjMRaQGQMA0ejfKHwIbb8GcDwKP3xFfwfATg3d5AgAAsCvEDhABfMpQFWXy6ACExpcQAUUVOVk+wDgWtBIAaQJGr4e3mKbiRMSMAMAKJfixkVOxwl4gCYAJLeEVkOmRwCSBN/TuyK9A+BNVzpgAIxd/686ht4C8EkdgLOXpq4PXTba8xtjdQCWbP0X6gAMTr6tAXyxA8CkccfftnQh/ku//Lpu853rHABocSFfPDE1T/uCsVhMSzsTDDNVwfcFAJm1F0wAQFPUVmwxKWbkkurxglRIyXPulGkAapUSix03A0Cfh0lmWbGGbTMn8mgfl2XLcay4yajt1wD6ZEXdoiBjtYSu68XV/QCwMwIgWU5lmcw+ioB8JU97MaakK+EMk1SDbRcBsW2VY40iYJ+nUsvDFAAIToU0B4AYqf/1IByJiLsACGe5vLJoegSQDFZqRTlGkh4FRr4m/p8AMLaMIP3PbD9efDAxC5//8tvXhxfmYAowOLyTApx5uheAJ9eNJOH0hKVr5tENpP/wqGWocwDAIRgT2+4D4S0YJj0AqhsVhgi3XwPQAhBdggT0pgjQ7UrYBACgmikSSQmIcT1jFNxpQtM42hwAEgHSrK4GSDMA4Cm2yKbwCKsnNQJHvWpGQANb+obU/giAlNiivlnGQWEDNjYV3hTMjwCq8G4Ix/sAmcvobLp9AJIpEMiBagkHdKmAplh9S0Pna3om0HYNQKmpBApW4wAtFOE1g0pHVzRgDoDdE4HiNEnAvG0VHl7MERIVNzsCiGpbukrAoVzJiHxSjG7yBwfA095Gm3kPgIHHnw5+5j70ZG3SdvnUYO9jx6vBP1q/6j7yrVEEvGb9y8PuL1x7Aehesj7vuvA3m6Wra33hZPfaI8u9DgHgzIn1GZ343sm4aFVpE4BdHVXWgBkAduaW4vU5q5FcKpXNo+YAMFJA/MPTilvZFRivh2QnMPX1JXui1AIAiYBAGof6wLRUXBBMANAcKBy0D8Cec2jEZ0+UWkwB3i25IclGAWf3n306sCswGpCqSS5lZgSQ8MLM5sPXHxfKBwCAa7rRfpp5DwDb3D2n7VkPcnF44K/3Htiu3Ee+Xlh/fG1swNaPzLpdL0d9tr0AIMvD1lt3HRCAniGbc2h0YX25MwCUTyQ+soFaNLbImgfgOCWaA2D3jcpgHJMGJgH4L8c9oG3BxXy+8rE1+GQlX+nbBwCtzwRst7UEwMZ7y4JEPYl2GgA2iOUJ0D4AIAIj/7GeSsLnqv9jAJ647W/aiyN35g7D++/WLwjSc/bWl91Hl76/MvHglfEZZkenJ6ZHPxlBes7dnLDfGbo7goy8nptwDn23NDszefs+8uKBsVvoC/clxP/Z5LHxR+6BbqT3H9PDzqfnOlQERHn+I8s58EjTFKy2ARDfX75mAgA0zPMi/vsFAI9Gwx/r93uf+50BEH4vbmh0t/md+L8AIs9HUTMAGNHFW4z8wS8HXruwtlMguHimMeF37f7OJqH+C+f/w975hkTV5XH8jsMdpmnWNjFCmqFpRu0O0x1DndGh+Z+hqJNmOztiIUqUI4E1KUI7GiWYuQqyvhFWWiWwHmWgHrIyHlhoaV+4EtSbB2HxhRs8sbAvFnm2500Le8515s5of7ydc6yd/H1fxBG8336/37nn473n3HPv+88AH14cQKeHI57ADwBq3Y7Pf0oYPgzyFQDwOYIPgyjOMjs+DMJUumrPQGTYLJiJ9wcAAAAAAICsBQA3IAphP383xgEAAAAAgJ0HAP2hMW/JSDfFbsDTf9tHrjMzqQFz+nekHj/mywA49QNxJP9JA+AfxCb/TAPg5I/ELr9PA+C/+6j1r/wUAP5O7PFdGgC/JY9kJhMAp88Q+/xhAwD+8td9jPTHDADMnKH3+2FHvBLMdCWfQpd2J21id2ZIPWYK5RccdPmIIwmOpUwayU1m7I6kiWuIOJ/89H6Mdl8+tS69TgLgRCW5R1dqbjjQSR7J2UMZ583kJWKfysyH1cYr81npkndWtn19id4v1LETAGCd1dFoloWNPIHRR2EiL3/oKUzkdy1Q5WNgkc97JWZSHCtNIJkTTUzqg2TQsVMfm/A+5AcCgUAgEAgEAoFAIBAIBAKBQCAQCAQCgUAgEAgEAoFAIBAIBAKBQCAQCJQFcgUCgcgspUkTMgmoKU0MBcjERLs7w4nz0VKaaCPIZQ+jCutykRn1555NyKSAtp8O4H5y0KfUh31cLL9gy7TiqOYBpnbfsPRmURQLI5QujWFRTNRTmhSFUChtRyhdvMikspXSpKwQubDaKfoImSXMtC51KKLOXEqT3oQohqfoU3Kj/hZtWobnYWsUFYnd3twVfEJ26GF8b83yOp7n/RO0J5aR54203acVkUm1k9LFgvLBH2SjUrkfhTLIqMQNHmRWR+tyDOUVLaU0GcP9tEifUgH+vry9ieF52CwwrDjHjeMABwEAAAAAAAAA9JEB5zaZTIcmUaU85ahlIrpl6nPhQ5/iE8uMW2T38A58ZASPutAEbhLdpuqk/x8DTZzCLaeB5JbIiQ99gcesXfKjue027MEOI8XIbFIyKyJx6ZYO9aG84o24pSa59+6L4UNv4X6qwq0Y6VyLVJ8yPL7aAqjlpr4PWK/4vCBXXEdlZ5DsenGA5zQ4QB2M9I9ooBN/vACNOl6oxC3LYQITTRs+NIpMjGHcqh4munGXvqNgxCwK4dYDktOzWcpHxPmocMtOMvHmtuNDVXigiFJQZRQVdliwQyU280vFGSFxOSHFUYzzCuKWjaSfAi340DgOJY5bLSuEOUn9jWnES/Wp7qWdCnSdwz6FcsWrK6jsmoakmqcDrICR/hG14wGXIR/J/FtpeKMJ30wSytCmUCwkf3enhE35FBCYRHwbTYyjFBV2hjblRTRP6t1k8ppkuWVc3GiSIB0Xm3z4egPlaahRbXIcpbJz+DfZzcNIBwAAAAAAIAAAAAAAAJL1Ai/minjUGP241UJyb1lwDB/qwSbFuJVoIAllEB8q4hNdkFpekjmA8nQ+HtzaTfIpFdNuKR85lEQzRYXVk9jCj29v182Izu36xMZ+sh8gAXWlnJfUT0HShR+pv6VBtp7SFO0cQOykXKT1M6iRyu5AaFOA5TDSPyY9kk5aBmzATTKUG/ChA9LsMm6RrbxY8ZGHpWXAGG5aifPRL0nLgJT5SMuAOeT5bDAbl5YBKYujl5YBV8iLI4UirdYskhcnFU1EWgZ0UNcnHVmjtAyoZ+Ao2bXiAHNm2QT4TQueA/jgxQQ8B/Dp6z54DgAAAAAAAAAAsh8Ag+FwOBigdGmMIxf6vQDIpI12B4cX50O7F6AiiFyqGJV4pRKZUZ/bbciEei/AKDKJv6BPyYR8wmz3AqgYVpzjhnGAVQAABXKWlpZGaLfgaSPIhXaXmVWDTNy0nabG+dA+/VWE81EzqrAuF5nRXthwMWSiOUjbT8gkwOAPtyGAU2K5G7Aol2HFUc0jTO1AIBAIBAKBQCAQCAQCgbJB1j5iZT52YqCwYRyLntzF8H+TzwdiovBgXBxW9aEM6ONZ0oT3Qb9vWOU55BqRS6SuIncZlJcI5ilikZ9fXzGTm9xLrV452slNzKZUKCM5DFSSXIapoPAYTQ26mJfc5FeZq7C3KPo7c2G5wJzDTKPpBasSFn69OwIAXVfzSLVqlv8omJZrSF3WCuWNOZZV4lielaRMRn8hNnn7OnWOmzrXiF2W5dcE2J7kUevVcnLdsv0ZsUe/PbVG2LBMHslyxqNG1rlpYp+5zCfwK0J5rNRvST98EJqm91vN2REA8N7fRaqXGQC4+ZDU5fo1GQCnbhPH8koGwNEnxCZ/3i0D4Mp1Ypfv0wCo2UWthWspAEwTe/SkAfALeSSdmQC4eYPUpvbZBgB8f3EXI/VkAGDuBr3fbQAAAAAAAABgqz6dZtgxSzO9YLUCAAAAAIDsBIDLnOB5wddL/jyq48UEAAAAAADISgAcWRKCHe32hPiU+M/4GOO3GAAAAAAAgC8FgDJhCL9bq9UTJf7ITYkfAAAAAABkJwDKjRbcFUX2zgjHqXttS/URPIacY0vm8vH2GDcwH7lnuZWra+yynOhGZXYctVkemPSctazeNJ9jHjnMNZ4Ulo5qAQAAAABAFgKgNB5+kDvLcXjguOuMwU5PvILjDkwag6FiFd/AXYlHxUIh5PUE/UKVjnMPCZXVnsJmTm8T7vh9HqPXUJUwxnMcAAAAAAAgCwGgu+cXomdLytQGbraq2KZx9kavuPtKBLPb/cCDAcDvnTANGj0D7sb4MZfeK3hNrnLRp9Xb+FCFuzGhKlB3eaaOGAAAAAAAQDauAlhHcqJG3ni2mXOpgk6O66sqLt9zJaThuO69EgDGOa7Z2MJx+tfxQ7GoD5+QXXwFugJ4auW67YlxmAMAAAAAshcAnFUbu1BSLURLCwTP5NDQkMpY/0h1F0dehQEgqDlu3NjF4dc8FVxI+PeiXynk7+ltxehWYdaWaM0iAFxMq5YcAApdPg2ADJOLxACoVWaiAABKnbYEgMLifBoASqPZCgAKi6wYALUKs/scAHxO5bdlEnBM+qqFq864OMyLQ5Lmx8N23Me3MAD8DgyAqnUANPvDk9KvHNXbPBMcd7DriwBg/8u3smquEwNg//3LKU0vEANg/6u0y3NiAKzJJpd/IgbAn9Imbx9TAaAn7VRzkQIA59MlvrxADIC8tMltGgC8VFZkxQBYmJb97j9kA4DajMqvfQ0ATArSpjXrCaFD4+lUNzU1FTQ4c4O/wTcDS+8BYCXc4sK/csH5RQFwIyzI8i8QA+D5nCqlYA8xADJcfD3EAHgjm6guEwOgplA2ebefCgDT6XDeHKcAwMObsk9hHjEAVtPRrNIA4KqyIisGwE8+2e/mc0ZXAP3pGK9+DQC087sf4WW/oeL5I52JCivXNBgt7z7nGeCsF+LvAcAR8l9AZLCHKzIAcMszst0AEPlwYVK+5zQAeNe/rsvPSQFQi1zeJF0orgDeqOaSJv09NAD4d9Kk5jElAIIppzWaK4CH11Q/p0q8QAGA5VRx8ugAoKjIygEQVK0m/e7vZwYAX6ryL78GAFwWQWzzWlTFdoehV4wvHr0rTMa4ibD/1GBceA8AXK8nujh/VzhWlAGAp8Kde47tBsDb8ynV0gCg5nhStTRXAGtJE4o5gDeqq48VhLIlAM6nQtlFCYA5RU4KAPBKSYm3AsDPiqLZGgBykS8yAsCC0iopB8BNxZXflklAtVfF88ZoBzrRiqZURl6YDKBqNteJYXMLX5oCQEcSANqpKPrtuytcBgA0Ib4ysN0AWFNScAUAUDJ3s+UVwEslsWwFgCfHFZhsCYDHbFYBEAAUOSkAwH1F02NbAOCdkuIoAYASn88CAOtVgH7VrxVXfntWAfQ6V0NMJ72gxaozDTtRzxxsbTis02nbPBpOp7NynKEIv9lkVodGl7XIHXDinw7in9b/1brVhh0DgF0AAADANwWAD6jof+zdT0gb2xoA8PGGSCNabTYiJrTVMR3pje/R1jbFGyedaanM7TxtuUGRIj4krymBEm9DISSRpKBW+qDgxoWYG7qwGALxkb6Z3E1AmE2RgtnN5i1CIdkmEswmi3cm/9SYVGfO+Ofem29jqeOXOWcyvzlzzpfJovkeormjf9yOnEOcKgDQtwDUgVsAEuoW4CQjv2MB8F68W4CTjLrP7BbAckq3ACQlGwDnfAtQe3HQpEWtSnS5/8IAwO6UIgoFQGMV4NhVgC0SDoCLuQrgkQmAP90qQJ2HhNyZ//UfPedz/tcGQKktRRoKANRQDNoBMwI4UZbjACgnMQRgACgnyXthAShnckICUG4XkYQAoNI5kABU8thlAqCcMCcjAJWevzgAnGvUBMBFF4MfgQLA7i+GzwszAgicJMtxAOQzpSwxGADKSaI2SAD4SiYKDoB0KY+/EwKAjvLecHAAnKiTxQCwVu4lRjYAgidq618dgHisFExjFaAxCSh+EpBqTAL+sQForAI0AGisApzGx4FnS3G/+qGA/YP3632/c/vs/jYbGsVd918IgMYIoAHAnwmAKUxZDONYNQ1PzfXKe15ipfOsZd68MjhNdDVGAA0AGgD8MQHQWnsK8ar6bO/bPhkA3T2L104fgHClFLjQXVTtRdNjAaikKfw5xVhsUkYAI4eyMBZGCgDOzoMNIi0WSQBEykmo4q6QkgGIHOrfOplOAEDAW+kckAXkIyUAkD2wN8x++0QDsHWgk8nCTxscAL79ivT6rRMHwEJVW+unPB0AtutVA50QAGSjpf/0AbDmylF4C8cSdosUABacxWB9xSxsRsoIoKOcRbgeUAF2zyYBAL6UxFl4iSSbYKQAsFVKEhC6wxJnR6QCYCjvTlxoTKedVUsEoNzFTr+QhWUT9oxNNAB4OcmeyrYLkrDpXUYKAHQ5Dxi0ZQp5wjEoAPKlfJ6YiknXa504AIhKz6vIYkoHeS4AXH27udS2+Grq9Z3txc1ZAEDo38OrPRPC8wIUb3snb94Dx91969Xk/NgQAEDT/GJ1+/OAeeXS180fkYl3S1/W158o+pHWB8OTT9+8eXUJufR++0rv7+39sgCwH4VCgLDWlJQCwOFCoB0UTZASRgCHCoE4Hq25KHziQiDhNPKyKOqQAkClEEj4rY838IzUOoBDhUAO2pAlpQFwoBAolguy7FaQT1JiAThQCESmCSfr7CDsNgkAHKy22iXyCTaLOyMwABwoBPLyfIJNBb9TDCK6EIgJ0gnQYXTyXAB4PWBEQzTm6tEPmbW9AABM7/rFZHrUj7Ss69BvIdOHDWS5SfeSMBkBAA9eYvSQUV+aA3howk1DJqznBnKZxh7jRpdSoXlitK66sJlBeAA6w4H9EK7ZJE+Yat3efR8Ay9pIOXaF64AtlcvTEbEjAO/OoSzUCM/iatEAJCtJRvzCSUU7g2lSLADR/SRJ4cQPo+majBwPgP9AJvD+t9nxBO2XAACz38UjHAAgzahsDtxOigPAsZ8kAwAIgmPEZbMR0QAc7uRdHvSNpUa7TgwAt1vJp/YCAFjhgR6EBwqAzP4+OgAATpAyQyTOdA5guxDvAADab9cVL5Shvysuu/SDfdva5+Pdt1HDg9YP2MDd7v/+Glq5Ool9Hbz/UItpWuexd+2KJ1gZACX6YHDUYBrf+Fn3dfDuO61S4cb1XTdmp63/gwfgaM/qA1smTiwA1cc9ysfV6I5YAKpxcqY43mkTC0D1xgZfolZ1qahnAjK5lG8uzUgBoJpKns3wAZt4AKr2XgBA5afDNnEAHIoiAJFUyiIagMNRAID0LEQlA1D1NhEAUPmIMBQAh4+gAIAqigfOEoBSXAEAmN73I1PY6mvEPWPs6ts2vulH3G3YZ8WqdQls+la5+ZP1Obig/xjCNFcNLvAvxccKAE/AFtvKlWbrajeC3GhSKq7ixt/uI13Lg/IDQKb1mSQWhwTAFuYzXIeTgQPAh4dVLBGBA4DJ5Rm1YQ0SAD8+YmE7OBkAcKBJSz4bgQdgy+FY28pzKigAeLUj6QkmVbAA0PGMIx4MM7IBkM+A1mU5OQHIgpTOXPQsAZgZLMQzAIALXK1HjeBV3E+N433brstgg/e6NjCqJ4aGhlBl7239TWFK4DmmUWi/CZOAPWUAMGEVcVj5rwnTQ6EqYB3cAjwKYforn65r5AcgZp2zxfAaZ50oADrzKYa01zgwogAIEJzKYRiBAyBDj1AWOkVBAUDawcUtWWtIIxYAJpGNqfZQBzwAxNxc0LCghhsBoMG5II7HO2EBMPBzczSa4GQDwFBoXZKUEYBCyrk15nzmAKwPBAB6KgD8BP53TPfiFqpfnBFic8w0LBy4aUwzq1wUPjQ0XAZAeEYwGAcsvzcWABhQKpCWiXmrUkufwi2AWmvnOKdRDQeAzxCmwC3c0XVrMQBYclsWiss5LTAAkIE5H0UlDDEoAGJZJ0N5iRQ8ANEFD0lxRBoeAGfGn1HncD/cCCDp9zsSRNwGOwII+/2ZOJqyyTYCAPnWsnxGzhEASKnO1plYPGsAQiugXz9hX2c/4uNut/vu7euXXeuvwW9xTLPhIjaEP6kGYDQkbLFBKxUto7fcfUsDyleyA0DmtDqjUaf9DwkFgKfJQBCGpg4SBoAkjhIEgfJ+GABiHYUkNWpoxACQRIUsTQQHDcBIMVMwJsscgMp/9AOPoucAVMxcjpFhDkBFJVBOzjkAVQavN1kidQ5A5aft5EUAAHu3gbRPhqb6enUfEKT1b/pPfbge9PmyDtO4p3UTCHLZXA1A92PTFKK5gykV7SH8BuhO5bzsAETNC2tqtZrQxWAA6MxlwyCcaBQCANLOB0ASOxqHAcCBpkGS+NyWFwaABB0HWdLEHjQAeV7omQS+Jg8AUTkAoBY6ZAFA5ZEZAJ/8AET5swQg9LwYj55VA6DFfvj0UQuG9M1mbc/v0zq6Gblndm2+MGsxDbJEhB5+acKqAUAmzMb1VSMAwD2Dfdx8QpjuyQ2AzV4c/O9iaQoCADW9U9LWJh2AWJAV9sG2ECSlA0CxfOEtHsAzEABwfGEdMZZNReAAoLhCaYKKW2AtsAAkIt5IlDU44ADgvJ1cuEYhgFgAkt7OmAPvoGQDwOuN+Nj6j4WRAMAWSBlN4MkzuwVYNupK0fbsqWEcQVZCYMzunjd39f1gvRnS6YeFrw2/N6nDdJOjrcjGZytmXP9m0iCtY6hO98tA09LgzFAz0qYX3mNtuhXk6ti0mf4nrexDunrBHxG/9ckNgAV3FeaDOCsRkw4AA95Xws9IKhuTDkCy9NybAOqXDkAsWKwC9DcFIADYodVFHo/ecYsEIFyEyMbSHCwAfCqVX8A9DBQAhnwqn8NTHOwkIJFLpbL0gkO2SUAatG6OqFmTKhUAIeUC4bGcGQDfj75r5c8IDjaXHvzZ0lVe19u4VqPK79KbKXBonzWZhK27x9slPC30OAAiW8VBLrnHRqUDYInvFa8EySPfESACALW9eFpydodkAKiovXjKUva4dADItXixwVFPBg4AZjdcfCm/xwcHgDcufD/AXo1HjIgBQC0kCdcqjxUHgN8OEsV3jp5bEgFgAvVaJxkAWzGlz6a6IACIj0u4a/SG4gv2s/TPBzS+HFTaHED9aHw56PcF/j975/PSSNLG8TZNh5DJmN3TEJJgJmrSwV/DaCcrGmMk4sFEoygTRCRDYCZe5I1ZAi/JOwfBUZmA4MXDsPjiwUERXl/WgZ37XmTw8N7m4mn/gr3sXt+qnvSP+Dpjz1P1uqPzfA9LsdLfeeqprk+6qrqrvu3DQTlrOy53HEu/MXwhjABAACAAbi0AOmNd+cJJgOEDwZF/PoDqXyYAvPgF6vLrBx0A7X+AY/mbAYC/g01+MQHgV7DLnwYA/vOAWb8faQD4B9jjJwMAL+GRHJkB8OJHqM3P/24CwJ8/P+Ckn8wA+JHd749vY0uwk8x3UH3Y1MFTWnwJdXk5NqjD6AM4lsy2ZpKFm7xo12ZcBvMv4aHoL2M9hYdixLTQAMB4AuxxNKLtIpVchUfyvek5M7JxBPZJ9Jtuv2eJ73jpKGVss7VwxO734f43AQDFDpfx4UFtGO4ScGkuIkMsXn1eJAA3iWqPNLVBhvoc8sitrlKDsmEGD1EjdRtDO5XMe9Gx5Me8UuUM2LlJjPAIz+QnoFAoFAqFQqFQKBQKhUKhUCgUCoVCoVAoFAqFQqFQKBQKhUKhUCgUCoVC3SY56BEGLkaTGjVxMppEwsSkzHosaicNpcajPp2cMuzyEjMHq0uZmIRZ26mN1uuQvUo+6uPgeYIt14xzt7vDctlCoVD1CaNLticUiqdZu+4YMXndzegyQupTXGI06S+SUFKcUvyWmq2zumyQemVaGU3W4qHQ6B57lUqjpEpBnh1saZ5jxgVhiwaY8mH/vh6Vk5Ikje4yulT8kuR/zvos0kNMJqKMLu2kPj1ZRpOBURLKOqcUT8eJ2SSryxCpV/Exo8k2bacz9ip56KHSvbMc78OczDHjgrBDA1x3Yf9GACAAEACoK+UUiaIbFAAxWvRCxnQuhV56Tm+sFVoSQcNvB71ymPa6oSQtgkbMbeq/X6AAWKMlBfIE6FPrU6d9th1eH21Ww0sdctRsQzU7BLeTmCH1qi7R0iy8nc5oO/WpyYHWS/WZpv3rIECLrDM/jYxXZC4ZJzlX7bI0wPduWmzDnv4Jvbq3uLg4RX52JTlDzzFZLwNMAgV66Ri9sarqaSigH6k+euWqTFm0TItpyE3QP0cvPab1SdCSDfIwMWijlybo7Tiv1meHIcNKkDpkaHJ6VDPQk8mpeimhiBQaoqUU5Je3NU8v7aChFGlpMgmsUy+9eoL4SMerpDQ3wDoVGG1XM+7XM842gVM+0HIuqXZz/djTP6GnH08xp6xUCwnI/Nvj+WYTKQcJZUM3UV2kdshP5Z58qT6QfZSfJJpN/HWWeztzKTmgedKR5uT4f4NsY7XT0xzKcQxYp+PLVWKdabO3XHKsM9kpo5fsTrGnfxIAUpOAAGg2gQKg2QQIgEv1gQGg2YQVAM1uQAA0mwAB0GwCBsAlHx4AuOTICoBLdggABAACAAGA+h+Nj9IDDOnd5VePMlz0QsaWY/RS2vX8Mi3FQWPmmVBTKPEVyBzAq9HmUPaHIXMa+6oJDeVjfbIMGRbn1HoZyQHdjCfNyQkdKACTi2JzvYrTwDoltCo1muqcdQ5geOLSHZRlA8APml2jAQewp39Ch/Rtqe68ugpAi7DZZfqam7JHp3BGFPBLeOrriIPqKsAT8Dtc6mtuiroKUKGlMGgVgL6OqKypqwAK60uFkVnqEKNmeQX8Ep5TvZSuAhR3wG/guVSTNG2nTVryQuul5mdXXQUocXmv8GPGX8lcMk5XXqhHTn2OFGkRVwE+K3wP4CrhewCfF74HgABAACAAEAB3AAAHFABbHAAgsZ6y5KBL+MusALBRAOQYTbJ0JinIKcW71OyA1WWZAiDJaDJO2klOs1fJrr5nU+Z4H8ZCHDMuCM9ogEEEgAV1J5NJD+tMrrOVuIQZTSIBYlJiDUUhJq2s4z6nh7gonDLcZidmzEdQRYlJgPUrx05i8sTBXiUfbe9unrch14wLwiFfOxQKhUKhUCgUCoVCoVAoFAqFQqFQKBQKhUKhUCgUCoVCoVAoFAqFQqFQqJtUpzsKldv4dMQnRuE2+iYWDgYTfQvqGrxCUX33c5/CEIqLQ24NKY2YnDwaysXQTqL5WyyG/HSbP8eqdUd5yV3mEt5Vt/dd1vk9uMb1ruvuhbssuDWXM4ZY1jSTpQ24SUrb+cwdhJts6Iepjd/joJlGf6kweJxpnwp6CgyBmLdRszG0t3mHibcL97gpbZBlhoffu28CACMndqjeBPXfhMDcLtRla1HfoHNmGxxLX59mUp8Bm+zc01gUWLwAu2zou813pe3Myk01nm2epsAej3q1rjH9GzySfdOWI5GpAajNw4J5J75Y3mPnpHq78ZO9OsDut237NgCw7YCqbgLAwjDUpbSgA8C2Bo4lrQPgURBs8tAAwFwU7DJpAGDPwaytOQ0AJ2CPrAGA1/BIpswAmNuB2sy2NwGgEHZwUs4EgMVn7H4VBAACAAGAAOApl+rsdF69e0vksJP0KB8ZWLc5XAgABAAC4K4BIKYeRyDPB6/cadN70PJEEMcfC66u+DQCAAGAALh7AKgWCgf5oj8TuOKvs7YJj5A+jgmu59UtBAACAAFw9wCg7m4cfe1/etUJDk5HREiNxugQwIcAQAAgAO4oACJ7sq0z4j5db08nyWC/FhsprFREwflqWxxYlmfWHLGzkhAR68H3Z8maIOy885yvrz8qIwAQAAiAOwGAcTl46Fn0/zAUmh+IROpydarq7y3TOYCVY38xqNA5APuCv2MiNF+hDwVD8cSo3FVGACAAEAC3GwDv3e5oKZYI7c3OhFbsg6fzQx5loRhzJ18XdygAxGC8LtYIADrXQ12ewUo1kRRSUnVpOFc8bkUAIAAQALcbAD2ZTKYqy73l4eMhkQz2R/yn4uL8uehL5tzqKgCdA6CrAO75MXI31u7L25GURE9s6pV2EQAIAATALQfAxMTERrCiCDv+Gfp/BuSU70083lGoD0eaALDlL9C/Z0MrrpScpT1Vena7AFBWdHnLYAB4DZdZMADChkkYDABzfdgAMGvR6VoAhK0l5/MAmLWWnOsBYK2prAPAamDWAFA2hVf+i+cAqPqlLvX/yEGhVm+v+qX9t00A2JXUf38pbqulQjFSSt0YALbGDWUVMACm+55rejMIBcBwWjfZvAADYE83eT4ABsDSfd3kXTcTAHJGOOdhBgC4jeTcnwYDoGJEU2cBwPCZ0VRveQDAFNgjdgC0nuhuJ61fAQBa5XyNJPJc3uz02MvR2IacEs0AGJa/p38/lVOumwZAOmQoXwID4HSsRdPGQygAHu/rJpksGAAF3aSlDwyA8Q7dZLLEBIBNI5xehQEAgUXdp6MCBkCXEU2QBQBvTU2V4wEAq4FZA0D/hO62v/MVACCaqJIxvZgPDSSX50r01OmgWwVAPKsCQMkc7wiCchBai9w0AM4keSzTUDDKAoCu1Ee9i0IBQO6qQsOkL8kAgKmRhksWDIDtjhbNZFxkBEBCc9oLswEg3/B5vgUGQLBlQoumwgKAC2tN9SUAGNICW+MAgKGW9obbiecrAIDrdLTl3dqCvOL1vve/3qsvj1bUIUDavzpOlwF9az3FdCUvBxXh5gFQnHY3JJZZAPDQwkj3WgDsWRm5XQcAW7eFsfK1AIjymQPYbNmPWh3cXgOAMyvJuQ4Ak24ecwAX1prqSwCQd/ObAyAAyFqYkvo/A8BvHHNe3iYD//i6IggPF2TJP58uhwsdrYKnQ0q0rsR3Bcde0S+FbN2CBgD/9I0BoOUxh0lAAoAA8yoAAcAp+yoAAYBiweRaAIh8VgE2W1YtOVkAQHqWeRWAAMBKcqwAwMpQ/YsAoDis63oA5P7yVQBXp2k7JJ/TfiGq3wW2uXftTp8QOST/ERwB0ddGCz5nYMtN/15z0m8Da50+BAACAAFwmwFwO/YD4AcAj4VHQutDgFkWAHQzLwMSAAxqoXAbAngZAXAmMi8D/pe9O2hJJdoDAD41KEOl0lJUmtR0orRHOqVMNipFC52riaKIiBfhmptAB9zkKhANg8CNCxcXXCiBCxe26COE9BFc9UXeGXXKyu5t5syz7sP/IgLH4/HMzM+Zc878z8QtgEaqW4CQRACoPttMnwLA/uXDgP8KAB3/5SgKOSgA4EcB7r7PKMDN/+kogF6iUYDBNx8FoL7BKMC/AoBMphuFqQYFgMM7ipQcAgC+ECoGAYBxXIi3AgMAX5UWNAB8dfqQAPDfi4YB4LlxtiAB4NvnRCIAnismDQB89X7MAfgLACYqP4pVDxQA9uQo6iQEAJVxITUnBAAt27gUJQwA++NCrkhIAE74kgYsHACH43KSBxAAxPnaeCAB+MyuEgLA6qcq9mkA2uPSAqo5AH8ZBtznU6OG5p2A807AeSfgvBNwDsAcgDkA0LGUG4dz6c0rm6Rz7YM3qXIv29xq0b3iHIA5AHMA/kkAkoRsFK7gWxpOI54P3pQlxufZUilSJ1Ne9RyAOQBzAP5NAHRn+mGcvz3bN/qfA0CjL+Mz6AOoPa+QBvZPCE8m181iAPDwy8END9PoNS4GgOPnlfswsv6oODreDQkHwHDAr3CHYdu29tFhTykCADn/fUZdd2jQrhIHQBN/qQ4a7HFR2BMDQNfJLyE4bEvbzbTnFP4GQOqlNubksDK9S1wMABO7Ctver2x1bSQUAOWJZjIrj48ekigMAFV+1b/hZw4eh5Vj8cuMf5+cFQD9D175LADI7dLmDEYB8qlxtHYx1pYlTAuFkAgA0qejUOQwFr07z9fEABAfF6KvY54m3Vg2ThvG/RsAJ/1xKdcYVqWZVMq4qhQOgHVcSGV03t8wKbU4ALx8dboaLJc2MhRFxT1iAEgYRuUsJsGhPDh1hFNWp2AAGL429xh5RHHhzSbFADCxq0gF7dMv048kDADPFWuDo7BxYl1m2iQEAOVxaVugqffu+vRwH6uttC9MH5OzBmDtsnulKJ8nWXu/3M0BADrXhw197Ba8hF4a4lv7YL8Xd8/jpSAFANDKM41+zxqpY4/dbSRWuXpotfzoJrLiOYyfFgrnGIJV+6uGe9WmBAD4ZS/hSmK5sDdWK9N1EQBMTARSZVbppigAJiYCqbsBHL+k9KhgACYnAsVuPAcH916/cABeTQTSBLO0WAAmJwLl0taBWq2Wk2IAmJwIhKfCBfkBHhIMwMR8GzYHqqLeLT0NxAAwMRHIdpLBt6OGhgcGgImK7aWWA071IhOAAGByIlB61TsC4IG+znlK4fqsAWCtLmOHIS70FiqiMwAACMtF2O0+3kSWWibjU8d9c4vUFkxZr9sFAPBkCYZyWcZ9AEdu2k25Cb0ZUTLECe26kKFav+uscUGkSQkACHRfwo9jNt0Dhu1bHkICARg8PqcVKagwMlndEnEFkOs9F/J7fLLh6RYuEIDqS4YT/petdqIQCED9pZAqJ1AyfNgXB0DypSR7CACwuCeuD2D7+qUgcKI90lUxfQCxl0L4m+QapdAIBuD1rqoy9yGMzSSuxAPwqmKDhZ9cy9E7YgGQv1SvB46fYNUwBECVSoNLphj9yM6oD6A/jAoAQPe0jmZknf+gygsLudHXJaIam9HhWbkhrHuau3ynvhYnHknnkY7QrpSIigr1EzwAMqOHDDjc0Vuf6ZHcq+hkaJG2qM251NlAAgDenls6oCMZSZMCAXgX9yIAmBKexqJKIABvQ6P+pc8GBALwOlhPw5DbEgfAmwZIp21XUVQEAG84SPts+/Zpc3CE5gQkS1ml8E7A16GMZ6v4ru99PgmROQGTC/dcqcsGViQA7yIzBCCaOAIHtpxRhGYDwDhWAQDu6iaSJBosUky71Bt9V2GTywHaQxtn3LJAl7LuwVkC/KBvdwjtmuMC/Ic2nwHwgy36srr8rKFBEPOCDF2jXb+diLpGSg9Axs1dxnmf0G8CgN/xXmuBAMh9dP4SgwIAP/XtmiUBwJl20PRyBocFQF6mssYF+hyHBiBGdTWwALD2PJ1u+XYxiQCQe/uAfXX54+3FAVBv/gTfVUXpzbMBID1+WgoAcAF+rQMu8CnFU1d0o3/BZf2umhTgqt5LUZRRZrBZtrgugQShRXVPXCegngeAzxIacx9xswJa4BbguENYVtvrWukBOOxEwV/qx/cAgLxkTlEMEgBnOxNfvoIBgNwKD1hpAECr9v2C3qs3QwJwlfAFceUOvUNCApDrl6MYLADb/h8/S9mTICkRAOQO3apk4o4SJi0AtWwX1AClSpqZ9wGceTgA9M8AHCBcEuDMrtFSTnPRDboPuR2XIrQ5WZnLCHDIAzBOEVSruoYAWGUoshQrncl0zP/gFuDn8ArA0fgetwCFbNqDwQIAYj9/SEIAsJ4tK3YUTyeHdmgARiRZf+CQAAzCGfCN9sI+FSQAVUebhQWALVBtMxlLZWsSAYDljltP6SNaITEAg3wFnPpO6tz81QB06qBd28RjrklHi8Xinm1dedFiwas0ob298N5yb3kLQKDDbXHLyNClwG5x48oqO5cegJ4O7ERV5/Q7AIA+UotTP0kwAPJWPwcBwIGe68o5YVrH0gBAHjXvIAE4SJW4SvveV10YAHg5NbWJBQFAZriudrZn/C0VAMMs4QVjVWIA8KctcEE5YLqhrwaAqNwiqngnuWEw3SDIyo6lvUFbQJvXTIS2mDLFEEQZeQuA5sSdRLR2QoaqOrT5Vc5B6QAImMChF3T72a8HgPQzW9OfLBYAAItzV3u/mnqYK4AQlzDRqU8NUGgAtrkc485UwgkJALmYB4bgeQMKBUDokZmeq1wQAOYuxaVv6DkKkgHAna2+ck5iAEjDkxxjfzPVGQ0DdhKjOA69BUBHLLabOnBJL4/o9PcpEyNH9iMX3UxER2iRK2/n6GGBeAsAEou4Wg0XAKCYJppdv5dbWlhqALYNnXaPoTwYFADcMCD1UI1CAVCgs10/iAEMAOh52V84ztM2qE7A4VEuSR+A/aRyeZ32HsN2AmJ1KlWoWrNBFgqAaCI9/bsL6wP4lV8u1HoJq1oqADy9qj+xHGMlAsBeNXiv7eDXJMgYCu1mH58JADUXn29fETp1RBGk3gHX7MVSRL2xeLbVMVkOt7lGiZsIUzywgtz2zghX68mtRVaCRpMpbF24ItOUHFFYuGNMYaoja8FUhPnJyDYQtQG8yft7Q3oA2Luy2+0NhuAA2CvRXPSgALikhoVMyX4hAADyepX20vHrEDQAofP0OjwA9RaoTvajzOkCACALYYZJFEi4UYCbj3KLCAMgZC9TCcrwPt+BWAD2mzR1mvzD1sIAyHPHEWUHvwc3WZr21dmZAPDn2MD5ZwRJOTo6t5bU/LjeLT5llh9WSIJdG1pwc1troioRaUM/szQYm1ufNk3tn10bMJS7m/Zsw5etDej0KD+c4SpobUBSrZ6WPe9r1gbUqK9yU57aEAsAGY3++dEg0WsDksqD/7Z39yoNQ2EYx6FBKVIindtB/KCVFrfi4sfiZqgfCIUgEhDs3sHF7hUidOzQQeigi4NDHbyEIr0Kb8SlOTlCEXnfAyX0/7uAh5cmfRoazjnzniuysB9A/vjhrOJNqqfy9QEcDrr4Avjz5R6Hg/5TFg4Hda5/WN1+W/lWrBCmACgACiCzBbCzX4tu3zcUCwSPHn2pwCqA6ztpyqBjCuDiVTxL2xTA05Y45CYtgOZAnDJMC2Diq513kgI4EWeM0wK4kk8ysgugGUtjWi+/CqDX8h0ZWwUwivV5n8uxJdhzUSycmuJZ/5LH9OpJSqCY5d68aYnkKe3KLKRek4dEfjLKNCzq5WZf3liRESS7SK3uKQY5sG6bTXnOcGDlfPSKroRBuqdWzkVgfykKYM2TS185lBuKGFMjeUWI2WOtqwi5TEYplxQpBfN85jlQ2tVfKPPTWFBcp4b9F7Pm87H3tOt67uTdjDfn9gYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIvxA2vpz35XAE/FAAAAAElFTkSuQmCC",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython.display import Image\n",
    "Image(filename='./images/02-01.png') "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Embedding 实际上是一种查找表\n",
    "- 查表\n",
    "  - token embedding: 30522 * 768\n",
    "  - segment embedding: 2 * 768\n",
    "  - position embedding: 512 * 768\n",
    "- 后处理\n",
    "  - LayerNorm\n",
    "  - Dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import BertTokenizer, BertModel\n",
    "import torch \n",
    "from torch import nn\n",
    "\n",
    "model_name = 'bert-base-uncased'\n",
    "tokenizer = BertTokenizer.from_pretrained(model_name)\n",
    "model = BertModel.from_pretrained(model_name)\n",
    "test_sent = 'this is a test sentence'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': tensor([[ 101, 2023, 2003, 1037, 3231, 6251,  102]]), 'token_type_ids': tensor([[0, 0, 0, 0, 0, 0, 0]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1]])}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input = tokenizer(test_sent, return_tensors='pt')\n",
    "input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_ids = input['input_ids']\n",
    "token_type_ids = input['token_type_ids']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 7])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_ids.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos_ids = torch.arange(input_ids.shape[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0, 1, 2, 3, 4, 5, 6])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pos_ids"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Token Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 7, 768])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "token_embed = model.embeddings.word_embeddings(input_ids)\n",
    "token_embed.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Segment Type Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 7, 768])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seg_embed = model.embeddings.token_type_embeddings(token_type_ids)\n",
    "seg_embed.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 0.0004,  0.0110,  0.0037,  ..., -0.0066, -0.0034, -0.0086],\n",
       "         [ 0.0004,  0.0110,  0.0037,  ..., -0.0066, -0.0034, -0.0086],\n",
       "         [ 0.0004,  0.0110,  0.0037,  ..., -0.0066, -0.0034, -0.0086],\n",
       "         ...,\n",
       "         [ 0.0004,  0.0110,  0.0037,  ..., -0.0066, -0.0034, -0.0086],\n",
       "         [ 0.0004,  0.0110,  0.0037,  ..., -0.0066, -0.0034, -0.0086],\n",
       "         [ 0.0004,  0.0110,  0.0037,  ..., -0.0066, -0.0034, -0.0086]]],\n",
       "       grad_fn=<EmbeddingBackward0>)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seg_embed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Position Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([7, 768])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pos_embed = model.embeddings.position_embeddings(pos_ids)\n",
    "pos_embed.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 1.7505e-02, -2.5631e-02, -3.6642e-02,  ...,  3.3437e-05,\n",
       "          6.8312e-04,  1.5441e-02],\n",
       "        [ 7.7580e-03,  2.2613e-03, -1.9444e-02,  ...,  2.8910e-02,\n",
       "          2.9753e-02, -5.3247e-03],\n",
       "        [-1.1287e-02, -1.9644e-03, -1.1573e-02,  ...,  1.4908e-02,\n",
       "          1.8741e-02, -7.3140e-03],\n",
       "        ...,\n",
       "        [-5.6087e-03, -1.0445e-02, -7.2288e-03,  ...,  2.0837e-02,\n",
       "          3.5402e-03,  4.7708e-03],\n",
       "        [-3.0871e-03, -1.8956e-02, -1.8930e-02,  ...,  7.4045e-03,\n",
       "          2.0183e-02,  3.4077e-03],\n",
       "        [ 6.4257e-03, -1.7664e-02, -2.2067e-02,  ...,  6.7531e-04,\n",
       "          1.1108e-02,  3.7521e-03]], grad_fn=<EmbeddingBackward0>)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pos_embed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Input Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 0.0316, -0.0411, -0.0564,  ...,  0.0021,  0.0044,  0.0219],\n",
       "         [-0.0489,  0.0285, -0.0204,  ...,  0.0190,  0.0265, -0.0045],\n",
       "         [-0.0469, -0.0156, -0.0336,  ...,  0.0117,  0.0135,  0.0109],\n",
       "         ...,\n",
       "         [ 0.0185, -0.0608, -0.0331,  ...,  0.0040, -0.0071, -0.1213],\n",
       "         [ 0.0294, -0.0017, -0.0217,  ..., -0.0004, -0.0913, -0.0948],\n",
       "         [-0.0077, -0.0166, -0.0123,  ..., -0.0310,  0.0124, -0.0064]]],\n",
       "       grad_fn=<AddBackward0>)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_embed = token_embed + seg_embed + pos_embed.unsqueeze(0)\n",
    "input_embed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Post-Processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 0.1686, -0.2858, -0.3261,  ..., -0.0276,  0.0383,  0.1640],\n",
       "         [-0.6485,  0.6739, -0.0932,  ...,  0.4475,  0.6696,  0.1820],\n",
       "         [-0.6270, -0.0633, -0.3143,  ...,  0.3427,  0.4636,  0.4594],\n",
       "         ...,\n",
       "         [ 0.6010, -0.6970, -0.2001,  ...,  0.2960,  0.2060, -1.7181],\n",
       "         [ 0.8323,  0.2878,  0.0021,  ...,  0.2628, -1.1310, -1.2708],\n",
       "         [-0.1481, -0.2948, -0.1690,  ..., -0.5009,  0.2544, -0.0700]]],\n",
       "       grad_fn=<NativeLayerNormBackward0>)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embed = model.embeddings.LayerNorm(input_embed)\n",
    "embed = model.embeddings.dropout(embed)\n",
    "embed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 7, 768])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embed.shape"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "lightning",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.17"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}