aura-text-attention-maps.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install ecco","metadata":{"execution":{"iopub.status.busy":"2023-05-31T16:44:00.075742Z","iopub.execute_input":"2023-05-31T16:44:00.076132Z","iopub.status.idle":"2023-05-31T16:51:17.698562Z","shell.execute_reply.started":"2023-05-31T16:44:00.076103Z","shell.execute_reply":"2023-05-31T16:51:17.697022Z"},"_kg_hide-output":true,"_kg_hide-input":true,"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":1,"outputs":[{"name":"stdout","text":"Collecting ecco\n  Downloading ecco-0.1.2-py2.py3-none-any.whl (70 kB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m70.7/70.7 kB\u001b[0m \u001b[31m2.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hRequirement already satisfied: transformers~=4.2 in /opt/conda/lib/python3.10/site-packages (from ecco) (4.29.2)\nRequirement already satisfied: seaborn~=0.11 in /opt/conda/lib/python3.10/site-packages (from ecco) (0.12.2)\nCollecting scikit-learn~=0.23 (from ecco)\n  Downloading scikit-learn-0.24.2.tar.gz (7.5 MB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.5/7.5 MB\u001b[0m \u001b[31m50.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[?25h  Installing build dependencies ... \u001b[?25ldone\n\u001b[?25h  Getting requirements to build wheel ... \u001b[?25ldone\n\u001b[?25h  Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n\u001b[?25hRequirement already satisfied: PyYAML~=5.4 in /opt/conda/lib/python3.10/site-packages (from ecco) (5.4.1)\nCollecting captum~=0.4 (from ecco)\n  Downloading captum-0.6.0-py3-none-any.whl (1.3 MB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m48.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hRequirement already satisfied: matplotlib in /opt/conda/lib/python3.10/site-packages (from captum~=0.4->ecco) (3.6.3)\nRequirement already satisfied: numpy in /opt/conda/lib/python3.10/site-packages (from captum~=0.4->ecco) (1.23.5)\nRequirement already satisfied: torch>=1.6 in /opt/conda/lib/python3.10/site-packages (from captum~=0.4->ecco) (2.0.0+cpu)\nRequirement already satisfied: scipy>=0.19.1 in /opt/conda/lib/python3.10/site-packages (from scikit-learn~=0.23->ecco) (1.10.1)\nRequirement already satisfied: joblib>=0.11 in /opt/conda/lib/python3.10/site-packages (from scikit-learn~=0.23->ecco) (1.2.0)\nRequirement already satisfied: threadpoolctl>=2.0.0 in /opt/conda/lib/python3.10/site-packages (from scikit-learn~=0.23->ecco) (3.1.0)\nRequirement already satisfied: pandas>=0.25 in /opt/conda/lib/python3.10/site-packages (from seaborn~=0.11->ecco) (1.5.3)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from transformers~=4.2->ecco) (3.12.0)\nRequirement already satisfied: huggingface-hub<1.0,>=0.14.1 in /opt/conda/lib/python3.10/site-packages (from transformers~=4.2->ecco) (0.14.1)\nRequirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from transformers~=4.2->ecco) (21.3)\nRequirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.10/site-packages (from transformers~=4.2->ecco) (2023.5.5)\nRequirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from transformers~=4.2->ecco) (2.28.2)\nRequirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /opt/conda/lib/python3.10/site-packages (from transformers~=4.2->ecco) (0.13.3)\nRequirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.10/site-packages (from transformers~=4.2->ecco) (4.64.1)\nRequirement already satisfied: fsspec in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.14.1->transformers~=4.2->ecco) (2023.5.0)\nRequirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.14.1->transformers~=4.2->ecco) (4.5.0)\nRequirement already satisfied: contourpy>=1.0.1 in /opt/conda/lib/python3.10/site-packages (from matplotlib->captum~=0.4->ecco) (1.0.7)\nRequirement already satisfied: cycler>=0.10 in /opt/conda/lib/python3.10/site-packages (from matplotlib->captum~=0.4->ecco) (0.11.0)\nRequirement already satisfied: fonttools>=4.22.0 in /opt/conda/lib/python3.10/site-packages (from matplotlib->captum~=0.4->ecco) (4.39.3)\nRequirement already satisfied: kiwisolver>=1.0.1 in /opt/conda/lib/python3.10/site-packages (from matplotlib->captum~=0.4->ecco) (1.4.4)\nRequirement already satisfied: pillow>=6.2.0 in /opt/conda/lib/python3.10/site-packages (from matplotlib->captum~=0.4->ecco) (9.5.0)\nRequirement already satisfied: pyparsing>=2.2.1 in /opt/conda/lib/python3.10/site-packages (from matplotlib->captum~=0.4->ecco) (3.0.9)\nRequirement already satisfied: python-dateutil>=2.7 in /opt/conda/lib/python3.10/site-packages (from matplotlib->captum~=0.4->ecco) (2.8.2)\nRequirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.10/site-packages (from pandas>=0.25->seaborn~=0.11->ecco) (2023.3)\nRequirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch>=1.6->captum~=0.4->ecco) (1.12)\nRequirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch>=1.6->captum~=0.4->ecco) (3.1)\nRequirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch>=1.6->captum~=0.4->ecco) (3.1.2)\nRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->transformers~=4.2->ecco) (2.1.1)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->transformers~=4.2->ecco) (3.4)\nRequirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->transformers~=4.2->ecco) (1.26.15)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->transformers~=4.2->ecco) (2023.5.7)\nRequirement already satisfied: six>=1.5 in /opt/conda/lib/python3.10/site-packages (from python-dateutil>=2.7->matplotlib->captum~=0.4->ecco) (1.16.0)\nRequirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch>=1.6->captum~=0.4->ecco) (2.1.2)\nRequirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch>=1.6->captum~=0.4->ecco) (1.3.0)\nBuilding wheels for collected packages: scikit-learn\n  Building wheel for scikit-learn (pyproject.toml) ... \u001b[?25ldone\n\u001b[?25h  Created wheel for scikit-learn: filename=scikit_learn-0.24.2-cp310-cp310-linux_x86_64.whl size=7680713 sha256=d412e8a881479cc928b56adc3a9d41029e640f7cd9c3a24c88fa0f91f9e8fe65\n  Stored in directory: /root/.cache/pip/wheels/13/a4/68/4e78865652fa14db4a162b491e5138565f97646f9e1f2ab8cc\nSuccessfully built scikit-learn\nInstalling collected packages: scikit-learn, captum, ecco\n  Attempting uninstall: scikit-learn\n    Found existing installation: scikit-learn 1.2.2\n    Uninstalling scikit-learn-1.2.2:\n      Successfully uninstalled scikit-learn-1.2.2\n\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\ngplearn 0.4.2 requires scikit-learn>=1.0.2, but you have scikit-learn 0.24.2 which is incompatible.\nimbalanced-learn 0.10.1 requires scikit-learn>=1.0.2, but you have scikit-learn 0.24.2 which is incompatible.\nmlxtend 0.22.0 requires scikit-learn>=1.0.2, but you have scikit-learn 0.24.2 which is incompatible.\nnilearn 0.10.1 requires scikit-learn>=1.0.0, but you have scikit-learn 0.24.2 which is incompatible.\nyellowbrick 1.5 requires scikit-learn>=1.0.0, but you have scikit-learn 0.24.2 which is incompatible.\u001b[0m\u001b[31m\n\u001b[0mSuccessfully installed captum-0.6.0 ecco-0.1.2 scikit-learn-0.24.2\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0m","output_type":"stream"}]},{"cell_type":"code","source":"!git clone https://github.com/karthik0899/EmoSense","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-31T22:19:52.509263Z","iopub.execute_input":"2023-05-31T22:19:52.509642Z","iopub.status.idle":"2023-05-31T22:19:54.808945Z","shell.execute_reply.started":"2023-05-31T22:19:52.509609Z","shell.execute_reply":"2023-05-31T22:19:54.807727Z"},"trusted":true},"execution_count":1,"outputs":[{"name":"stdout","text":"Cloning into 'EmoSense'...\nremote: Enumerating objects: 292, done.\u001b[K\nremote: Counting objects: 100% (217/217), done.\u001b[K\nremote: Compressing objects: 100% (152/152), done.\u001b[K\nremote: Total 292 (delta 136), reused 106 (delta 65), pack-reused 75\u001b[K\nReceiving objects: 100% (292/292), 2.92 MiB | 11.69 MiB/s, done.\nResolving deltas: 100% (163/163), done.\n","output_type":"stream"}]},{"cell_type":"code","source":"!pip install -r 'https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt'","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-31T22:19:56.125400Z","iopub.execute_input":"2023-05-31T22:19:56.125798Z","iopub.status.idle":"2023-05-31T22:20:10.481205Z","shell.execute_reply.started":"2023-05-31T22:19:56.125765Z","shell.execute_reply":"2023-05-31T22:20:10.480071Z"},"trusted":true},"execution_count":2,"outputs":[{"name":"stdout","text":"Requirement already satisfied: transformers==4.29.2 in /opt/conda/lib/python3.10/site-packages (from -r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 1)) (4.29.2)\nCollecting accelerate==0.19.0 (from -r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 2))\n  Downloading accelerate-0.19.0-py3-none-any.whl (219 kB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m219.1/219.1 kB\u001b[0m \u001b[31m7.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hCollecting datasets==2.12.0 (from -r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 3))\n  Downloading datasets-2.12.0-py3-none-any.whl (474 kB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m474.6/474.6 kB\u001b[0m \u001b[31m27.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hRequirement already satisfied: scipy in /opt/conda/lib/python3.10/site-packages (from -r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 4)) (1.10.1)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from transformers==4.29.2->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 1)) (3.12.0)\nRequirement already satisfied: huggingface-hub<1.0,>=0.14.1 in /opt/conda/lib/python3.10/site-packages (from transformers==4.29.2->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 1)) (0.14.1)\nRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from transformers==4.29.2->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 1)) (1.23.5)\nRequirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from transformers==4.29.2->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 1)) (21.3)\nRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from transformers==4.29.2->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 1)) (5.4.1)\nRequirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.10/site-packages (from transformers==4.29.2->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 1)) (2023.5.5)\nRequirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from transformers==4.29.2->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 1)) (2.28.2)\nRequirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /opt/conda/lib/python3.10/site-packages (from transformers==4.29.2->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 1)) (0.13.3)\nRequirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.10/site-packages (from transformers==4.29.2->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 1)) (4.64.1)\nRequirement already satisfied: psutil in /opt/conda/lib/python3.10/site-packages (from accelerate==0.19.0->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 2)) (5.9.3)\nRequirement already satisfied: torch>=1.6.0 in /opt/conda/lib/python3.10/site-packages (from accelerate==0.19.0->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 2)) (2.0.0)\nRequirement already satisfied: pyarrow>=8.0.0 in /opt/conda/lib/python3.10/site-packages (from datasets==2.12.0->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 3)) (10.0.1)\nRequirement already satisfied: dill<0.3.7,>=0.3.0 in /opt/conda/lib/python3.10/site-packages (from datasets==2.12.0->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 3)) (0.3.6)\nRequirement already satisfied: pandas in /opt/conda/lib/python3.10/site-packages (from datasets==2.12.0->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 3)) (1.5.3)\nRequirement already satisfied: xxhash in /opt/conda/lib/python3.10/site-packages (from datasets==2.12.0->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 3)) (3.2.0)\nRequirement already satisfied: multiprocess in /opt/conda/lib/python3.10/site-packages (from datasets==2.12.0->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 3)) (0.70.14)\nRequirement already satisfied: fsspec[http]>=2021.11.1 in /opt/conda/lib/python3.10/site-packages (from datasets==2.12.0->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 3)) (2023.5.0)\nRequirement already satisfied: aiohttp in /opt/conda/lib/python3.10/site-packages (from datasets==2.12.0->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 3)) (3.8.4)\nRequirement already satisfied: responses<0.19 in /opt/conda/lib/python3.10/site-packages (from datasets==2.12.0->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 3)) (0.18.0)\nRequirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets==2.12.0->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 3)) (23.1.0)\nRequirement already satisfied: charset-normalizer<4.0,>=2.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets==2.12.0->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 3)) (2.1.1)\nRequirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets==2.12.0->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 3)) (6.0.4)\nRequirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets==2.12.0->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 3)) (4.0.2)\nRequirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets==2.12.0->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 3)) (1.9.1)\nRequirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets==2.12.0->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 3)) (1.3.3)\nRequirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets==2.12.0->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 3)) (1.3.1)\nRequirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.14.1->transformers==4.29.2->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 1)) (4.5.0)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging>=20.0->transformers==4.29.2->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 1)) (3.0.9)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->transformers==4.29.2->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 1)) (3.4)\nRequirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->transformers==4.29.2->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 1)) (1.26.15)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->transformers==4.29.2->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 1)) (2023.5.7)\nRequirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch>=1.6.0->accelerate==0.19.0->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 2)) (1.12)\nRequirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch>=1.6.0->accelerate==0.19.0->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 2)) (3.1)\nRequirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch>=1.6.0->accelerate==0.19.0->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 2)) (3.1.2)\nRequirement already satisfied: python-dateutil>=2.8.1 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets==2.12.0->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 3)) (2.8.2)\nRequirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets==2.12.0->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 3)) (2023.3)\nRequirement already satisfied: six>=1.5 in /opt/conda/lib/python3.10/site-packages (from python-dateutil>=2.8.1->pandas->datasets==2.12.0->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 3)) (1.16.0)\nRequirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch>=1.6.0->accelerate==0.19.0->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 2)) (2.1.2)\nRequirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch>=1.6.0->accelerate==0.19.0->-r https://raw.githubusercontent.com/karthik0899/EmoSense/main/requirements.txt (line 2)) (1.3.0)\nInstalling collected packages: accelerate, datasets\n  Attempting uninstall: accelerate\n    Found existing installation: accelerate 0.12.0\n    Uninstalling accelerate-0.12.0:\n      Successfully uninstalled accelerate-0.12.0\n  Attempting uninstall: datasets\n    Found existing installation: datasets 2.1.0\n    Uninstalling datasets-2.1.0:\n      Successfully uninstalled datasets-2.1.0\nSuccessfully installed accelerate-0.19.0 datasets-2.12.0\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0m","output_type":"stream"}]},{"cell_type":"code","source":"from EmoSense.emo_function import *","metadata":{"execution":{"iopub.status.busy":"2023-05-31T22:20:10.484161Z","iopub.execute_input":"2023-05-31T22:20:10.484944Z","iopub.status.idle":"2023-05-31T22:20:24.541811Z","shell.execute_reply.started":"2023-05-31T22:20:10.484905Z","shell.execute_reply":"2023-05-31T22:20:24.540869Z"},"trusted":true},"execution_count":3,"outputs":[{"name":"stdout","text":"[nltk_data] Downloading package stopwords to /usr/share/nltk_data...\n[nltk_data]   Package stopwords is already up-to-date!\n[nltk_data] Downloading package punkt to /usr/share/nltk_data...\n[nltk_data]   Package punkt is already up-to-date!\n","output_type":"stream"}]},{"cell_type":"code","source":"!pip install bertviz","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-31T22:20:24.543224Z","iopub.execute_input":"2023-05-31T22:20:24.544019Z","iopub.status.idle":"2023-05-31T22:20:39.953297Z","shell.execute_reply.started":"2023-05-31T22:20:24.543983Z","shell.execute_reply":"2023-05-31T22:20:39.952144Z"},"trusted":true},"execution_count":4,"outputs":[{"name":"stdout","text":"Collecting bertviz\n  Downloading bertviz-1.4.0-py3-none-any.whl (157 kB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m157.6/157.6 kB\u001b[0m \u001b[31m6.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25hRequirement already satisfied: transformers>=2.0 in /opt/conda/lib/python3.10/site-packages (from bertviz) (4.29.2)\nRequirement already satisfied: torch>=1.0 in /opt/conda/lib/python3.10/site-packages (from bertviz) (2.0.0)\nRequirement already satisfied: tqdm in /opt/conda/lib/python3.10/site-packages (from bertviz) (4.64.1)\nRequirement already satisfied: boto3 in /opt/conda/lib/python3.10/site-packages (from bertviz) (1.26.100)\nRequirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from bertviz) (2.28.2)\nRequirement already satisfied: regex in /opt/conda/lib/python3.10/site-packages (from bertviz) (2023.5.5)\nRequirement already satisfied: sentencepiece in /opt/conda/lib/python3.10/site-packages (from bertviz) (0.1.99)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from torch>=1.0->bertviz) (3.12.0)\nRequirement already satisfied: typing-extensions in /opt/conda/lib/python3.10/site-packages (from torch>=1.0->bertviz) (4.5.0)\nRequirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch>=1.0->bertviz) (1.12)\nRequirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch>=1.0->bertviz) (3.1)\nRequirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch>=1.0->bertviz) (3.1.2)\nRequirement already satisfied: huggingface-hub<1.0,>=0.14.1 in /opt/conda/lib/python3.10/site-packages (from transformers>=2.0->bertviz) (0.14.1)\nRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from transformers>=2.0->bertviz) (1.23.5)\nRequirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from transformers>=2.0->bertviz) (21.3)\nRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from transformers>=2.0->bertviz) (5.4.1)\nRequirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /opt/conda/lib/python3.10/site-packages (from transformers>=2.0->bertviz) (0.13.3)\nCollecting botocore<1.30.0,>=1.29.100 (from boto3->bertviz)\n  Downloading botocore-1.29.144-py3-none-any.whl (10.8 MB)\n\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.8/10.8 MB\u001b[0m \u001b[31m79.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hRequirement already satisfied: jmespath<2.0.0,>=0.7.1 in /opt/conda/lib/python3.10/site-packages (from boto3->bertviz) (1.0.1)\nRequirement already satisfied: s3transfer<0.7.0,>=0.6.0 in /opt/conda/lib/python3.10/site-packages (from boto3->bertviz) (0.6.1)\nRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->bertviz) (2.1.1)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->bertviz) (3.4)\nRequirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->bertviz) (1.26.15)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->bertviz) (2023.5.7)\nRequirement already satisfied: python-dateutil<3.0.0,>=2.1 in /opt/conda/lib/python3.10/site-packages (from botocore<1.30.0,>=1.29.100->boto3->bertviz) (2.8.2)\nRequirement already satisfied: fsspec in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.14.1->transformers>=2.0->bertviz) (2023.5.0)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging>=20.0->transformers>=2.0->bertviz) (3.0.9)\nRequirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch>=1.0->bertviz) (2.1.2)\nRequirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch>=1.0->bertviz) (1.3.0)\nRequirement already satisfied: six>=1.5 in /opt/conda/lib/python3.10/site-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.30.0,>=1.29.100->boto3->bertviz) (1.16.0)\nInstalling collected packages: botocore, bertviz\n  Attempting uninstall: botocore\n    Found existing installation: botocore 1.29.76\n    Uninstalling botocore-1.29.76:\n      Successfully uninstalled botocore-1.29.76\n\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\naiobotocore 2.5.0 requires botocore<1.29.77,>=1.29.76, but you have botocore 1.29.144 which is incompatible.\u001b[0m\u001b[31m\n\u001b[0mSuccessfully installed bertviz-1.4.0 botocore-1.29.144\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0m","output_type":"stream"}]},{"cell_type":"code","source":"import ecco\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2023-05-31T18:45:08.402321Z","iopub.execute_input":"2023-05-31T18:45:08.402851Z","iopub.status.idle":"2023-05-31T18:45:08.409293Z","shell.execute_reply.started":"2023-05-31T18:45:08.402813Z","shell.execute_reply":"2023-05-31T18:45:08.408143Z"},"trusted":true},"execution_count":55,"outputs":[]},{"cell_type":"code","source":"import transformers\nfrom datasets import Dataset,load_dataset, load_from_disk\nfrom transformers import AutoTokenizer, AutoModelForSequenceClassification","metadata":{"execution":{"iopub.status.busy":"2023-05-31T22:20:39.956263Z","iopub.execute_input":"2023-05-31T22:20:39.956670Z","iopub.status.idle":"2023-05-31T22:20:41.221662Z","shell.execute_reply.started":"2023-05-31T22:20:39.956632Z","shell.execute_reply":"2023-05-31T22:20:41.220479Z"},"trusted":true},"execution_count":5,"outputs":[]},{"cell_type":"markdown","source":"# Factor Analysis (NMF) :","metadata":{}},{"cell_type":"markdown","source":"![Factor Analysis](https://jalammar.github.io/images/explaining/neuron-factors-decomposition.PNG)\n\n#### Advantages:\n1. The factors can be achiieved by decomposing the matrix holding the activations values of FFNN neurons using Non-negative Matrix Factorization(NMF)[1].\n2. Beyond dimensionality reduction, Non-negative Matrix Factorization can reveal underlying common behaviour of groups of neurons. It can be used to analyze the entire network, a single layer, or groups of layers[1].","metadata":{}},{"cell_type":"markdown","source":"## Pre-trained DIstilBERT Model Neuron Factorization:","metadata":{}},{"cell_type":"code","source":"lm = ecco.from_pretrained('distilbert-base-uncased', activations=True)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-31T18:45:10.557716Z","iopub.execute_input":"2023-05-31T18:45:10.558476Z","iopub.status.idle":"2023-05-31T18:45:11.592328Z","shell.execute_reply.started":"2023-05-31T18:45:10.558431Z","shell.execute_reply":"2023-05-31T18:45:11.591188Z"},"trusted":true},"execution_count":56,"outputs":[{"name":"stderr","text":"Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertModel: ['vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.bias', 'vocab_transform.weight']\n- This IS expected if you are initializing DistilBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n- This IS NOT expected if you are initializing DistilBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n","output_type":"stream"}]},{"cell_type":"code","source":"text = ''' The vast expanse of the ocean stretches out before me, its calm waters reflecting the sky above.\nIts tranquil surface, undisturbed by the chaos of the world, inspires a sense of serene tranquility.\nThe soft whisper of the waves kissing the shoreline is a peaceful symphony, a melodious song of\nnature that soothes the soul. The gentle, rhythmic motion of the ocean’s currents embodies a silent\nstrength, a sense of enduring serenity that is profoundly peaceful. The boundless beauty of the sea,\nwith its captivating calmness, offers a sanctuary from the bustling noise of everyday life, a haven\nwhere one can find peace and solitude. It’s a place where the mind can rest, allowing thoughts to\nflow as freely as the waters themselves. The ocean, in its infinite tranquility, serves as a reminder\nof the simple and quiet beauty that exists in our world.\n'''\n# Factorize the activations of all layers\ninputs = lm.tokenizer([text], return_tensors=\"pt\")\noutput = lm(inputs)\nnmf_1 = output.run_nmf(n_components=8)                         #non-negative matrix factorization on 8 components\nnmf_1.explore()","metadata":{"execution":{"iopub.status.busy":"2023-05-31T18:45:11.594531Z","iopub.execute_input":"2023-05-31T18:45:11.595692Z","iopub.status.idle":"2023-05-31T18:45:13.494225Z","shell.execute_reply.started":"2023-05-31T18:45:11.595642Z","shell.execute_reply":"2023-05-31T18:45:13.492687Z"},"trusted":true},"execution_count":57,"outputs":[{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"<html lang=\"en\">\n<script src=\"https://requirejs.org/docs/release/2.3.6/minified/require.js\"></script>\n<script>\n    var ecco_url = 'https://storage.googleapis.com/ml-intro/ecco/'\n    //var ecco_url = 'http://localhost:8000/'\n\n    if (window.ecco === undefined) window.ecco = {}\n\n    // Setup the paths of the script we'll be using\n    requirejs.config({\n        urlArgs: \"bust=\" + (new Date()).getTime(),\n        nodeRequire: require,\n        paths: {\n            d3: \"https://d3js.org/d3.v6.min\", // This is only for use in setup.html and basic.html\n            \"d3-array\": \"https://d3js.org/d3-array.v2.min\",\n            jquery: \"https://code.jquery.com/jquery-3.5.1.min\",\n            ecco: ecco_url + 'js/0.0.6/ecco-bundle.min',\n            xregexp: 'https://cdnjs.cloudflare.com/ajax/libs/xregexp/3.2.0/xregexp-all.min'\n        }\n    });\n\n    // Add the css file\n    //requirejs(['d3'],\n    //    function (d3) {\n    //        d3.select('#css').attr('href', ecco_url + 'html/styles.css')\n    //    })\n\n    console.log('Ecco initialize!!')\n\n    // returns a 'basic' object. basic.init() selects the html div we'll be\n    // rendering the html into, adds styles.css to the document.\n    define('basic', ['d3'],\n        function (d3) {\n            return {\n                init: function (viz_id = null) {\n                    if (viz_id == null) {\n                        viz_id = \"viz_\" + Math.round(Math.random() * 10000000)\n                    }\n                    // Select the div rendered below, change its id\n                    const div = d3.select('#basic').attr('id', viz_id),\n                        div_parent = d3.select('#' + viz_id).node().parentNode\n\n                    // Link to CSS file\n                    d3.select(div_parent).insert('link')\n                        .attr('rel', 'stylesheet')\n                        .attr('type', 'text/css')\n                        .attr('href', ecco_url + 'html/0.0.2/styles.css')\n\n                    return viz_id\n                }\n            }\n        }, function (err) {\n            console.log(err);\n        }\n    )\n</script>\n\n<head>\n    <link id='css' rel=\"stylesheet\" type=\"text/css\">\n</head>\n<div id=\"basic\"></div>\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.Javascript object>","application/javascript":"\n         requirejs(['basic', 'ecco'], function(basic, ecco){\n            const viz_id = basic.init()\n            \n            ecco.interactiveTokensAndFactorSparklines(viz_id, {'tokens': [{'token': '[CLS]', 'token_id': 101, 'type': 'input', 'position': 0}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 1}, {'token': 'vast', 'token_id': 6565, 'type': 'input', 'position': 2}, {'token': 'expanse', 'token_id': 22944, 'type': 'input', 'position': 3}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 4}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 5}, {'token': 'ocean', 'token_id': 4153, 'type': 'input', 'position': 6}, {'token': 'stretches', 'token_id': 14082, 'type': 'input', 'position': 7}, {'token': 'out', 'token_id': 2041, 'type': 'input', 'position': 8}, {'token': 'before', 'token_id': 2077, 'type': 'input', 'position': 9}, {'token': 'me', 'token_id': 2033, 'type': 'input', 'position': 10}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 11}, {'token': 'its', 'token_id': 2049, 'type': 'input', 'position': 12}, {'token': 'calm', 'token_id': 5475, 'type': 'input', 'position': 13}, {'token': 'waters', 'token_id': 5380, 'type': 'input', 'position': 14}, {'token': 'reflecting', 'token_id': 10842, 'type': 'input', 'position': 15}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 16}, {'token': 'sky', 'token_id': 3712, 'type': 'input', 'position': 17}, {'token': 'above', 'token_id': 2682, 'type': 'input', 'position': 18}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 19}, {'token': 'its', 'token_id': 2049, 'type': 'input', 'position': 20}, {'token': 'tran', 'token_id': 25283, 'type': 'input', 'position': 21}, {'token': '##quil', 'token_id': 26147, 'type': 'input', 'position': 22}, {'token': 'surface', 'token_id': 3302, 'type': 'input', 'position': 23}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 24}, {'token': 'und', 'token_id': 6151, 'type': 'input', 'position': 25}, {'token': '##ist', 'token_id': 2923, 'type': 'input', 'position': 26}, {'token': '##urbed', 'token_id': 29595, 'type': 'input', 'position': 27}, {'token': 'by', 'token_id': 2011, 'type': 'input', 'position': 28}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 29}, {'token': 'chaos', 'token_id': 8488, 'type': 'input', 'position': 30}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 31}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 32}, {'token': 'world', 'token_id': 2088, 'type': 'input', 'position': 33}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 34}, {'token': 'inspire', 'token_id': 18708, 'type': 'input', 'position': 35}, {'token': '##s', 'token_id': 2015, 'type': 'input', 'position': 36}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 37}, {'token': 'sense', 'token_id': 3168, 'type': 'input', 'position': 38}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 39}, {'token': 'serene', 'token_id': 25388, 'type': 'input', 'position': 40}, {'token': 'tran', 'token_id': 25283, 'type': 'input', 'position': 41}, {'token': '##quil', 'token_id': 26147, 'type': 'input', 'position': 42}, {'token': '##ity', 'token_id': 3012, 'type': 'input', 'position': 43}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 44}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 45}, {'token': 'soft', 'token_id': 3730, 'type': 'input', 'position': 46}, {'token': 'whisper', 'token_id': 7204, 'type': 'input', 'position': 47}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 48}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 49}, {'token': 'waves', 'token_id': 5975, 'type': 'input', 'position': 50}, {'token': 'kissing', 'token_id': 7618, 'type': 'input', 'position': 51}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 52}, {'token': 'shoreline', 'token_id': 17721, 'type': 'input', 'position': 53}, {'token': 'is', 'token_id': 2003, 'type': 'input', 'position': 54}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 55}, {'token': 'peaceful', 'token_id': 9379, 'type': 'input', 'position': 56}, {'token': 'symphony', 'token_id': 6189, 'type': 'input', 'position': 57}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 58}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 59}, {'token': 'mel', 'token_id': 11463, 'type': 'input', 'position': 60}, {'token': '##od', 'token_id': 7716, 'type': 'input', 'position': 61}, {'token': '##ious', 'token_id': 6313, 'type': 'input', 'position': 62}, {'token': 'song', 'token_id': 2299, 'type': 'input', 'position': 63}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 64}, {'token': 'nature', 'token_id': 3267, 'type': 'input', 'position': 65}, {'token': 'that', 'token_id': 2008, 'type': 'input', 'position': 66}, {'token': 'soothe', 'token_id': 28384, 'type': 'input', 'position': 67}, {'token': '##s', 'token_id': 2015, 'type': 'input', 'position': 68}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 69}, {'token': 'soul', 'token_id': 3969, 'type': 'input', 'position': 70}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 71}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 72}, {'token': 'gentle', 'token_id': 7132, 'type': 'input', 'position': 73}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 74}, {'token': 'rhythmic', 'token_id': 14797, 'type': 'input', 'position': 75}, {'token': 'motion', 'token_id': 4367, 'type': 'input', 'position': 76}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 77}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 78}, {'token': 'ocean', 'token_id': 4153, 'type': 'input', 'position': 79}, {'token': '’', 'token_id': 1521, 'type': 'input', 'position': 80}, {'token': 's', 'token_id': 1055, 'type': 'input', 'position': 81}, {'token': 'currents', 'token_id': 14731, 'type': 'input', 'position': 82}, {'token': 'em', 'token_id': 7861, 'type': 'input', 'position': 83}, {'token': '##bo', 'token_id': 5092, 'type': 'input', 'position': 84}, {'token': '##dies', 'token_id': 18389, 'type': 'input', 'position': 85}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 86}, {'token': 'silent', 'token_id': 4333, 'type': 'input', 'position': 87}, {'token': 'strength', 'token_id': 3997, 'type': 'input', 'position': 88}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 89}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 90}, {'token': 'sense', 'token_id': 3168, 'type': 'input', 'position': 91}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 92}, {'token': 'enduring', 'token_id': 16762, 'type': 'input', 'position': 93}, {'token': 'ser', 'token_id': 14262, 'type': 'input', 'position': 94}, {'token': '##enity', 'token_id': 20693, 'type': 'input', 'position': 95}, {'token': 'that', 'token_id': 2008, 'type': 'input', 'position': 96}, {'token': 'is', 'token_id': 2003, 'type': 'input', 'position': 97}, {'token': 'profoundly', 'token_id': 28089, 'type': 'input', 'position': 98}, {'token': 'peaceful', 'token_id': 9379, 'type': 'input', 'position': 99}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 100}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 101}, {'token': 'bound', 'token_id': 5391, 'type': 'input', 'position': 102}, {'token': '##less', 'token_id': 3238, 'type': 'input', 'position': 103}, {'token': 'beauty', 'token_id': 5053, 'type': 'input', 'position': 104}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 105}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 106}, {'token': 'sea', 'token_id': 2712, 'type': 'input', 'position': 107}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 108}, {'token': 'with', 'token_id': 2007, 'type': 'input', 'position': 109}, {'token': 'its', 'token_id': 2049, 'type': 'input', 'position': 110}, {'token': 'capt', 'token_id': 14408, 'type': 'input', 'position': 111}, {'token': '##ivating', 'token_id': 17441, 'type': 'input', 'position': 112}, {'token': 'calm', 'token_id': 5475, 'type': 'input', 'position': 113}, {'token': '##ness', 'token_id': 2791, 'type': 'input', 'position': 114}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 115}, {'token': 'offers', 'token_id': 4107, 'type': 'input', 'position': 116}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 117}, {'token': 'sanctuary', 'token_id': 8493, 'type': 'input', 'position': 118}, {'token': 'from', 'token_id': 2013, 'type': 'input', 'position': 119}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 120}, {'token': 'bust', 'token_id': 13950, 'type': 'input', 'position': 121}, {'token': '##ling', 'token_id': 2989, 'type': 'input', 'position': 122}, {'token': 'noise', 'token_id': 5005, 'type': 'input', 'position': 123}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 124}, {'token': 'everyday', 'token_id': 10126, 'type': 'input', 'position': 125}, {'token': 'life', 'token_id': 2166, 'type': 'input', 'position': 126}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 127}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 128}, {'token': 'haven', 'token_id': 4033, 'type': 'input', 'position': 129}, {'token': 'where', 'token_id': 2073, 'type': 'input', 'position': 130}, {'token': 'one', 'token_id': 2028, 'type': 'input', 'position': 131}, {'token': 'can', 'token_id': 2064, 'type': 'input', 'position': 132}, {'token': 'find', 'token_id': 2424, 'type': 'input', 'position': 133}, {'token': 'peace', 'token_id': 3521, 'type': 'input', 'position': 134}, {'token': 'and', 'token_id': 1998, 'type': 'input', 'position': 135}, {'token': 'solitude', 'token_id': 22560, 'type': 'input', 'position': 136}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 137}, {'token': 'it', 'token_id': 2009, 'type': 'input', 'position': 138}, {'token': '’', 'token_id': 1521, 'type': 'input', 'position': 139}, {'token': 's', 'token_id': 1055, 'type': 'input', 'position': 140}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 141}, {'token': 'place', 'token_id': 2173, 'type': 'input', 'position': 142}, {'token': 'where', 'token_id': 2073, 'type': 'input', 'position': 143}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 144}, {'token': 'mind', 'token_id': 2568, 'type': 'input', 'position': 145}, {'token': 'can', 'token_id': 2064, 'type': 'input', 'position': 146}, {'token': 'rest', 'token_id': 2717, 'type': 'input', 'position': 147}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 148}, {'token': 'allowing', 'token_id': 4352, 'type': 'input', 'position': 149}, {'token': 'thoughts', 'token_id': 4301, 'type': 'input', 'position': 150}, {'token': 'to', 'token_id': 2000, 'type': 'input', 'position': 151}, {'token': 'flow', 'token_id': 4834, 'type': 'input', 'position': 152}, {'token': 'as', 'token_id': 2004, 'type': 'input', 'position': 153}, {'token': 'freely', 'token_id': 10350, 'type': 'input', 'position': 154}, {'token': 'as', 'token_id': 2004, 'type': 'input', 'position': 155}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 156}, {'token': 'waters', 'token_id': 5380, 'type': 'input', 'position': 157}, {'token': 'themselves', 'token_id': 3209, 'type': 'input', 'position': 158}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 159}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 160}, {'token': 'ocean', 'token_id': 4153, 'type': 'input', 'position': 161}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 162}, {'token': 'in', 'token_id': 1999, 'type': 'input', 'position': 163}, {'token': 'its', 'token_id': 2049, 'type': 'input', 'position': 164}, {'token': 'infinite', 'token_id': 10709, 'type': 'input', 'position': 165}, {'token': 'tran', 'token_id': 25283, 'type': 'input', 'position': 166}, {'token': '##quil', 'token_id': 26147, 'type': 'input', 'position': 167}, {'token': '##ity', 'token_id': 3012, 'type': 'input', 'position': 168}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 169}, {'token': 'serves', 'token_id': 4240, 'type': 'input', 'position': 170}, {'token': 'as', 'token_id': 2004, 'type': 'input', 'position': 171}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 172}, {'token': 'reminder', 'token_id': 14764, 'type': 'input', 'position': 173}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 174}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 175}, {'token': 'simple', 'token_id': 3722, 'type': 'input', 'position': 176}, {'token': 'and', 'token_id': 1998, 'type': 'input', 'position': 177}, {'token': 'quiet', 'token_id': 4251, 'type': 'input', 'position': 178}, {'token': 'beauty', 'token_id': 5053, 'type': 'input', 'position': 179}, {'token': 'that', 'token_id': 2008, 'type': 'input', 'position': 180}, {'token': 'exists', 'token_id': 6526, 'type': 'input', 'position': 181}, {'token': 'in', 'token_id': 1999, 'type': 'input', 'position': 182}, {'token': 'our', 'token_id': 2256, 'type': 'input', 'position': 183}, {'token': 'world', 'token_id': 2088, 'type': 'input', 'position': 184}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 185}, {'token': '[SEP]', 'token_id': 102, 'type': 'input', 'position': 186}], 'factors': [[[0.014623881317675114, 0.006287406664341688, 0.0003293639747425914, 0.002645679749548435, 0.0, 0.0, 0.0, 0.0016514131566509604, 0.026667799800634384, 0.024900343269109726, 0.031998541206121445, 0.35782814025878906, 0.0012579639442265034, 0.0, 0.0, 0.0, 0.0024387200828641653, 0.002308123977854848, 0.019681012257933617, 0.05173186957836151, 0.003574082627892494, 0.0, 0.0, 0.0071913329884409904, 0.38898879289627075, 0.016594551503658295, 0.015336638316512108, 0.0, 0.03263752907514572, 0.0, 0.0, 0.0, 0.0, 0.0004015808517578989, 0.49075108766555786, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.005670634098351002, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0008149119676090777, 0.0, 0.0, 0.030779022723436356, 0.0, 0.0, 0.008414591662585735, 0.3632657825946808, 0.0, 0.00029654448735527694, 0.0, 0.0, 0.003243319923058152, 0.0, 0.0, 0.0637759193778038, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.3200206160545349, 0.0, 0.0, 0.0, 0.0, 0.0, 0.03993526101112366, 0.014554595574736595, 0.0, 0.004363699350506067, 0.007907154969871044, 0.0, 0.0, 0.00406954251229763, 0.0018439029809087515, 0.36856481432914734, 0.0, 0.0, 0.0, 0.0, 0.0008693718700669706, 0.0, 0.06139012798666954, 0.013820777647197247, 0.0, 0.0, 0.000432528555393219, 0.01684398017823696, 0.006738350726664066, 0.004645475652068853, 0.0053152828477323055, 0.0, 0.0, 0.0040883757174015045, 0.4061482846736908, 0.09059631824493408, 0.0078094881027936935, 0.0, 0.0, 0.0, 0.021954305469989777, 0.48600009083747864, 0.0, 0.0, 0.006096240133047104, 0.0382818765938282, 0.007703704293817282, 0.003452879376709461, 0.01078104879707098, 0.004847335163503885, 0.0, 0.0, 0.018408993259072304, 0.35395267605781555, 0.0, 0.001666273339651525, 0.020380603149533272, 0.022034214809536934, 0.0, 0.0, 0.0, 0.07811950892210007, 0.0, 0.05594994127750397, 0.029258765280246735, 0.04777424782514572, 0.010597205720841885, 0.0, 0.0030133358668535948, 0.01733429916203022, 0.0, 0.017947092652320862, 0.003319879760965705, 0.020736567676067352, 0.41810762882232666, 0.013028343208134174, 0.013589739799499512, 0.045861922204494476, 0.001222399529069662, 0.06397274136543274, 0.007686878088861704, 0.04507475346326828, 0.0, 0.0, 0.036130595952272415, 0.0, 0.0, 0.0, 0.3903784155845642, 0.08903027325868607, 0.012719052843749523, 0.007702217437326908, 0.0, 0.0, 0.014456658624112606, 0.4973468780517578, 0.001570279709994793, 0.03508271276950836, 0.0, 0.0, 0.0, 0.0, 0.0, 0.10272730141878128, 0.0, 0.004371135029941797, 0.06612938642501831, 0.0, 0.027122262865304947, 0.0, 0.009662485681474209, 0.09344368427991867, 0.0], [0.031082477420568466, 0.0, 0.36308351159095764, 0.4785359501838684, 0.0, 0.013165549375116825, 0.49199047684669495, 0.36065661907196045, 0.235941082239151, 0.15693265199661255, 0.1899072378873825, 0.015768354758620262, 0.0875597819685936, 0.5379178524017334, 0.506274938583374, 0.42561280727386475, 0.03698829561471939, 0.42501702904701233, 0.26869210600852966, 0.014064092189073563, 0.06281916797161102, 0.059169553220272064, 0.2655060589313507, 0.3862018287181854, 0.0, 0.0, 0.0, 0.33361151814460754, 0.08263377845287323, 0.0, 0.47356826066970825, 0.0, 0.0, 0.39990535378456116, 0.0, 0.38409343361854553, 0.020888060331344604, 0.0, 0.3181700110435486, 0.0, 0.502275288105011, 0.06453881412744522, 0.1300751119852066, 0.28994661569595337, 0.0, 0.0, 0.4134015738964081, 0.4991334080696106, 0.0, 0.005868001841008663, 0.5010458827018738, 0.3394092619419098, 0.0, 0.48722192645072937, 0.0, 0.0, 0.5115761756896973, 0.4248306453227997, 0.0, 0.0, 0.0, 0.0, 0.2644219696521759, 0.3964439332485199, 0.0, 0.4079570472240448, 0.03184739872813225, 0.45450741052627563, 0.04942212626338005, 0.0, 0.4463280439376831, 0.0, 0.0, 0.517935574054718, 0.010040762834250927, 0.49197497963905334, 0.455480694770813, 0.0, 0.0, 0.41394004225730896, 0.010994037613272667, 0.05527324229478836, 0.49946242570877075, 0.0, 0.0, 0.12695953249931335, 0.0, 0.4479743242263794, 0.44336438179016113, 0.0, 0.0, 0.3089088499546051, 0.0, 0.429474800825119, 0.0036787446588277817, 0.4146431088447571, 0.04256057366728783, 0.05925588309764862, 0.4070478677749634, 0.5580933690071106, 0.0, 0.0, 0.14976711571216583, 0.25206196308135986, 0.48970502614974976, 0.0, 0.0, 0.46096423268318176, 0.005762394051998854, 0.0, 0.06217813864350319, 0.10989703238010406, 0.33535319566726685, 0.46053269505500793, 0.31349077820777893, 0.0, 0.1667594611644745, 0.0, 0.4142949879169464, 0.06670792400836945, 0.0, 0.08618815988302231, 0.23282749950885773, 0.4400833547115326, 0.006098937708884478, 0.4044605791568756, 0.3706856071949005, 0.0, 0.0, 0.4042375087738037, 0.09741836786270142, 0.16666346788406372, 0.16046039760112762, 0.2007381021976471, 0.521391749382019, 0.033214014023542404, 0.520551860332489, 0.014105957932770252, 0.025192683562636375, 0.014558067545294762, 0.0104470020160079, 0.0, 0.2777857780456543, 0.08249299228191376, 0.0, 0.40470090508461, 0.15534283220767975, 0.3546859622001648, 0.01593990996479988, 0.28481876850128174, 0.47097185254096985, 0.05994487926363945, 0.4307083189487457, 0.040425099432468414, 0.475263774394989, 0.04927458241581917, 0.03239990025758743, 0.4757125675678253, 0.258258193731308, 0.0, 0.0, 0.45210930705070496, 0.011569695547223091, 0.0, 0.047974199056625366, 0.39644289016723633, 0.07681882381439209, 0.12618331611156464, 0.28027766942977905, 0.0, 0.11752104014158249, 0.0, 0.0, 0.39888694882392883, 0.0, 0.0, 0.44809257984161377, 0.0, 0.5095003247261047, 0.5068928599357605, 0.0498146191239357, 0.28940731287002563, 0.02997327595949173, 0.15076090395450592, 0.3272840082645416, 0.0, 0.0], [0.00808081217110157, 0.002674888353794813, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0003597788163460791, 0.003544532461091876, 0.0, 0.0, 0.0, 0.0015512541867792606, 0.0018727232236415148, 0.00028847731300629675, 0.0003371734928805381, 0.00012536939175333828, 1.1850098371505737, 1.780958700692281e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0002952718350570649, 8.24722446850501e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4720102548599243, 0.0, 0.0019009152892977, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0007967630517669022, 0.004793430678546429, 1.8698232173919678, 0.005111600738018751, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0007311752415262163, 0.0, 0.0, 0.005970566999167204, 0.0, 0.0, 0.00047411699779331684, 0.0, 0.0, 0.0010978938080370426, 0.0012271740706637502, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0023811403661966324, 0.0, 0.0014332019491121173, 1.8786649703979492, 0.0040635173209011555, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0004206308221910149, 0.0, 0.0, 8.251407416537404e-05, 0.003128662006929517, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0004072836018167436, 0.0008253088453784585, 0.0009328239248134196, 0.0, 0.0008975901873782277, 0.0, 1.6339799165725708, 0.0007323484751395881, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.49862635135650635, 0.0, 0.0, 0.0, 0.00025187921710312366, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0005098182009533048, 0.007984330877661705, 0.0, 0.00030415202490985394, 0.0, 0.0, 0.0, 0.0, 1.2038196325302124, 0.0030357346404343843, 0.0015485754702240229, 0.0, 0.0, 0.0, 0.00719926692545414, 0.0005833201576024294, 0.0, 0.0007091765874065459, 0.0, 0.0, 0.0006817126413807273, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0021472631487995386, 7.811094292264897e-06, 0.0018083025934174657, 0.0, 1.7891544103622437, 0.007311051711440086, 0.0035047847777605057, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0001164629720733501, 0.44063296914100647, 0.0012352826306596398, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0007969745201990008, 0.0032991021871566772, 0.002462863689288497, 0.15094870328903198, 0.0], [0.01915881596505642, 0.6527720093727112, 0.09944996982812881, 0.024905312806367874, 0.040045224130153656, 0.6670612096786499, 0.015600206330418587, 0.0, 0.04415765032172203, 0.05047076568007469, 0.08935099840164185, 0.04431704804301262, 0.26411718130111694, 0.00947966892272234, 0.0035787110682576895, 0.0, 0.6202811002731323, 0.046782031655311584, 0.05301457643508911, 0.03172381967306137, 0.308771550655365, 0.0, 0.00994186382740736, 0.019184071570634842, 0.04065478593111038, 0.013818026520311832, 0.0, 0.0, 0.03436426445841789, 0.5870037078857422, 0.012795609422028065, 0.0, 0.64382004737854, 0.06414218991994858, 0.0, 0.0, 0.0, 0.09540625661611557, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0042635113932192326, 0.0, 0.5889013409614563, 0.03740962967276573, 0.0, 0.0, 0.6094500422477722, 0.0, 0.0, 0.5819351077079773, 0.0, 0.0, 0.054325785487890244, 0.0, 0.0, 0.0, 0.06998787820339203, 0.0, 0.010076209902763367, 0.0, 0.0, 0.0, 0.020363125950098038, 0.0, 0.0, 0.0, 0.6274664998054504, 0.021742934361100197, 0.0, 0.633337676525116, 0.0, 0.05725909024477005, 0.0, 0.0, 0.0, 0.6225771307945251, 0.02031167969107628, 0.036375630646944046, 0.1699001044034958, 0.0, 0.0, 0.0, 0.0, 0.06895729154348373, 0.009727992117404938, 0.0, 0.0, 0.061127569526433945, 0.0, 0.0, 0.0, 0.005606365390121937, 0.0, 0.019955869764089584, 0.004431979730725288, 0.024104589596390724, 0.0, 0.004327484872192144, 0.5806239247322083, 0.008504433557391167, 0.037250060588121414, 0.015590800903737545, 0.0, 0.6354747414588928, 0.023819368332624435, 0.019930681213736534, 0.046115465462207794, 0.24854066967964172, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.06990684568881989, 0.0, 0.0, 0.5496608018875122, 0.0, 0.023847438395023346, 0.0, 0.0, 0.07799558341503143, 0.035985417664051056, 0.00194196542724967, 0.07286638766527176, 0.0, 0.0, 0.11143521219491959, 0.0, 0.0, 0.0, 0.06215427443385124, 0.0, 0.00026491761673241854, 0.12752372026443481, 0.0, 0.0, 0.05295024439692497, 0.0, 0.0, 0.5855161547660828, 0.045804765075445175, 0.005607610568404198, 0.0, 0.0, 0.0, 0.012111129239201546, 0.030167628079652786, 0.0, 0.015390696935355663, 0.0, 0.016651906073093414, 0.5846959352493286, 0.014761559665203094, 0.041145894676446915, 0.00659529771655798, 0.639553964138031, 0.036629047244787216, 0.06363903731107712, 0.07930944859981537, 0.25267675518989563, 0.026735102757811546, 0.0, 0.0, 0.0012598407920449972, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.5660907626152039, 0.012684609740972519, 0.0714997947216034, 0.0, 0.0, 0.03995295614004135, 0.0, 0.07115879654884338, 0.2097393274307251, 0.06012614443898201, 0.033118732273578644, 0.0], [0.005863938946276903, 0.0, 0.0, 0.0002454595232848078, 0.7712588906288147, 0.0, 0.0, 0.0, 0.13964083790779114, 0.21916958689689636, 0.0124995531514287, 0.037159524857997894, 0.0, 0.0, 0.003953371196985245, 0.04261864721775055, 0.04127444326877594, 0.0032774764113128185, 0.0916157141327858, 0.04815523326396942, 0.0, 0.0, 0.001337632886134088, 0.004141943529248238, 0.042741212993860245, 0.0, 0.0, 0.0, 0.25298967957496643, 0.024956289678812027, 0.0, 0.8043357729911804, 0.02844168245792389, 0.051391713321208954, 0.0, 0.0, 0.02443661540746689, 0.0, 0.024557815864682198, 0.6595035195350647, 0.0, 0.0, 0.0, 0.06189846247434616, 0.0017530410550534725, 0.0, 0.0, 0.018581772223114967, 0.8441175222396851, 0.03622942790389061, 0.0, 0.08699031174182892, 0.039889201521873474, 0.0, 0.04677669331431389, 0.0, 0.0, 0.0, 0.003319110721349716, 0.0, 0.0, 0.00926328543573618, 0.01025673933327198, 0.06925083696842194, 0.7693482637405396, 0.023528430610895157, 0.20283637940883636, 0.0, 0.09688685089349747, 0.005426766350865364, 0.0030898586846888065, 0.0, 0.0, 0.0, 0.07952503114938736, 0.0, 0.08338458091020584, 0.8298692107200623, 0.019705532118678093, 0.0, 0.05397912114858627, 0.10050152987241745, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.025208432227373123, 0.03849238157272339, 0.0, 0.021855823695659637, 0.6734161376953125, 0.023124810308218002, 0.01510000228881836, 0.0006932728574611247, 0.2160048484802246, 0.06633556634187698, 0.0, 0.0, 0.032102931290864944, 0.0, 0.026136893779039383, 0.07859030365943909, 0.045753706246614456, 0.8504723906517029, 0.0168064646422863, 0.0, 0.05469775199890137, 0.19433769583702087, 0.0, 0.0, 0.009508059360086918, 0.0, 0.06765398383140564, 0.0, 0.0, 0.0, 0.0, 0.2950740158557892, 0.04089627042412758, 0.0, 0.06856352090835571, 0.04099712520837784, 0.7858380675315857, 0.009998817928135395, 0.021284140646457672, 0.020371172577142715, 0.0, 0.0, 0.1391090303659439, 0.05338720977306366, 0.06586397439241409, 0.060621149837970734, 0.001684725284576416, 0.23218099772930145, 0.0, 0.008111312054097652, 0.0, 0.00022287807951215655, 0.0, 0.0, 0.012082957662642002, 0.13994024693965912, 0.0, 0.001967928372323513, 0.06421709060668945, 0.008165859617292881, 0.0, 0.030286289751529694, 0.0, 0.19664841890335083, 0.012837735004723072, 0.21524858474731445, 0.013408636674284935, 0.2569780945777893, 0.0, 0.0, 0.07051020860671997, 0.0012384654255583882, 0.0, 0.0, 0.016954457387328148, 0.25503644347190857, 0.017841415479779243, 0.018293604254722595, 0.0, 0.0, 0.05977457016706467, 0.0, 0.0, 0.04697300121188164, 0.0, 0.0, 0.7108939290046692, 0.023400068283081055, 0.0, 0.1981804072856903, 0.0, 0.021614855155348778, 0.2787018418312073, 0.013044913299381733, 0.40043383836746216, 0.01755509525537491, 0.050077419728040695, 0.09501868486404419, 0.0], [0.003462751628831029, 0.0, 0.05440842732787132, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.006707304157316685, 0.03078734315931797, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0047960467636585236, 0.5222659707069397, 0.20228174328804016, 0.0, 0.020588433369994164, 0.45329517126083374, 0.4592130184173584, 0.08130230009555817, 0.00010202673001913354, 0.0019732769578695297, 0.0, 0.0, 0.0, 0.0, 0.0, 0.055904172360897064, 0.038243815302848816, 0.012974211014807224, 0.06619543582201004, 0.08281242102384567, 0.07364790886640549, 0.5307310223579407, 0.39147698879241943, 0.13229137659072876, 0.0, 0.0, 0.07212578505277634, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.05195176601409912, 0.014988735318183899, 0.0, 0.0, 0.47659701108932495, 0.47500669956207275, 0.152797132730484, 0.05822451412677765, 0.0242257472127676, 0.0, 0.02632325328886509, 0.09461438655853271, 0.033912111073732376, 0.020105289295315742, 0.0, 0.006093768402934074, 0.019168078899383545, 0.042855579406023026, 0.1030360758304596, 0.08168812841176987, 0.011185841634869576, 0.0, 0.0, 0.0, 0.02739063836634159, 0.033339470624923706, 0.0, 0.4760090410709381, 0.478395938873291, 0.0686580017209053, 0.03166605532169342, 0.08978006988763809, 0.028195111081004143, 0.012443998828530312, 0.011482972651720047, 0.08884375542402267, 0.09072105586528778, 0.06075528636574745, 0.46535372734069824, 0.09908346831798553, 0.04336170107126236, 0.030139047652482986, 0.0779339149594307, 0.03628992289304733, 0.019444014877080917, 0.018512165173888206, 0.33965176343917847, 0.15027448534965515, 0.011512724682688713, 0.0, 0.014637976884841919, 0.0, 0.013572749681770802, 0.03601234778761864, 0.08341794461011887, 0.4997505843639374, 0.1387326568365097, 0.15227314829826355, 0.07937689125537872, 0.0, 0.004628458991646767, 0.0, 0.03146471828222275, 0.01157876942306757, 0.005525305867195129, 0.4830550253391266, 0.14932939410209656, 0.0, 0.0, 0.03625820204615593, 0.0, 0.0, 0.0, 0.0, 0.002848030300810933, 0.0, 0.0, 0.001061472576111555, 0.04124411195516586, 0.04560111090540886, 0.0, 0.0, 0.0, 0.001054455409757793, 0.0, 0.0, 0.0, 0.008610332384705544, 0.0, 0.0, 0.0, 0.0, 0.0, 0.008406031876802444, 0.0, 0.011398238129913807, 0.0, 0.040891941636800766, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0029147565364837646, 0.04466214403510094, 0.10004148632287979, 0.5072978734970093, 0.3802497386932373, 0.1141529232263565, 0.0, 0.0, 0.0, 0.0, 0.009636053815484047, 0.0, 0.0011782834772020578, 0.06474968045949936, 0.14344771206378937, 0.07575975358486176, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.03088374435901642, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.10501641780138016, 0.0558789037168026, 0.032720040529966354, 0.05138268694281578, 0.023408569395542145, 0.1137533187866211, 0.0, 0.0, 0.03248245641589165, 0.0, 0.0, 0.0, 0.02110986039042473, 0.07782921195030212, 0.0, 0.0, 0.0, 0.021233247593045235, 0.010188975371420383, 0.01359838992357254, 0.020319269970059395, 0.0868312418460846, 0.01904197782278061, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0894603282213211, 0.2913667559623718, 0.3386174142360687, 0.07640054076910019, 0.079268679022789, 0.0, 0.0, 0.0, 0.037886954843997955, 0.0, 0.02771083638072014, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.2870592176914215, 0.3347180187702179, 0.0, 0.010467879474163055, 0.0857836976647377, 0.3503511846065521, 0.011893088929355145, 0.0, 0.0313226543366909, 0.0, 0.0, 0.0, 0.1661646068096161, 0.012323862873017788, 0.24610839784145355, 0.024332085624337196, 0.0, 0.0, 0.016977081075310707, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.04304327070713043, 0.08358314633369446, 0.0, 0.05846407637000084, 0.02534019947052002, 0.19586186110973358, 0.34409311413764954, 0.0034105826634913683, 0.018438613042235374, 0.07794955372810364, 0.36293885111808777, 0.07537972927093506, 0.0748029500246048, 0.008610673248767853, 0.0, 0.0011088444152846932, 0.15287600457668304, 0.27342385053634644, 0.029556669294834137, 0.0, 0.009130739606916904, 0.03156517818570137, 0.0, 0.007845078594982624, 0.0, 0.0, 0.0, 0.0, 0.0, 0.10217874497175217, 0.1285230666399002, 0.0010018388275057077, 0.030319545418024063, 0.0, 0.022213121876120567, 0.0, 0.2775467038154602, 0.35318809747695923, 0.03493016958236694, 0.08277607709169388, 0.03624628856778145, 0.0, 0.022362496703863144, 0.0, 0.0, 0.0, 0.009172801859676838, 0.07470807433128357, 0.35095030069351196, 0.04018858075141907, 0.1633470356464386, 0.1008152961730957, 0.19682127237319946, 0.10809095948934555, 0.004209874197840691, 0.07527127861976624, 0.0, 0.05355754867196083, 0.15126822888851166, 0.1009967178106308, 0.3162342309951782, 0.35035303235054016, 0.12149733304977417, 0.17901238799095154, 0.049825917929410934, 0.005332326050847769, 0.19757263362407684, 0.050891708582639694, 0.041563380509614944, 0.1416957974433899, 0.015051404014229774, 0.14576247334480286, 0.04277671128511429, 0.13522915542125702, 0.025611216202378273, 0.10046888887882233, 0.010255387984216213, 0.0, 0.030464690178632736, 0.0009530088282190263, 0.0, 0.0, 0.0, 0.07686641812324524, 0.1269839107990265, 0.0, 0.0, 0.0, 0.01744973286986351, 0.0, 0.26088008284568787, 0.30193862318992615, 0.41130363941192627, 0.06926874071359634, 0.0703723207116127, 0.069107286632061, 0.0029077385552227497, 0.06833234429359436, 0.0, 0.010575438849627972, 0.1207575798034668, 0.16024824976921082, 0.07056161761283875, 0.052614014595746994, 0.016421781852841377, 0.08688218891620636, 0.0], [0.04026730731129646, 0.0, 0.00015735904162283987, 0.0006498660077340901, 0.0, 0.00048658487503416836, 0.0005396066699177027, 0.003036509733647108, 0.0034246479626744986, 0.002276802435517311, 0.024616599082946777, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0028625947888940573, 0.003999045584350824, 0.007918082177639008, 0.0, 0.0, 0.0007288757478818297, 0.0020955379586666822, 0.0, 0.0013619855744764209, 0.0037217792123556137, 0.002636246383190155, 0.0, 0.0, 0.0, 0.0, 0.002206608420237899, 0.004753763787448406, 0.0007701350259594619, 0.0, 0.01181765366345644, 0.0, 0.002309043426066637, 0.0033714333549141884, 0.0, 0.0010640514083206654, 0.0017721100011840463, 0.011127117089927197, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0017398264026269317, 0.0, 0.002227826276794076, 0.0, 0.0, 0.0035408830735832453, 0.0, 0.0, 3.592060602386482e-05, 0.0003184769884683192, 0.001057855086401105, 0.0, 0.0, 0.0018469109199941158, 0.0012409959454089403, 0.0, 0.0038315930869430304, 0.0007558971992693841, 0.0021190838888287544, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0042216842994093895, 0.0015224128728732467, 0.0, 0.0, 0.0, 0.0025974931195378304, 0.0004922464722767472, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.00016258980031125247, 0.0, 0.001691933604888618, 0.0029652840457856655, 0.004569996148347855, 0.0, 0.0, 0.0, 0.003386289346963167, 0.0, 0.0, 0.0015012164367362857, 0.0, 0.0, 0.0016454443102702498, 0.0040139202028512955, 0.0067990245297551155, 0.0, 0.0, 0.0, 0.0, 0.0, 0.005357619374990463, 0.0, 0.0, 0.0, 0.0, 0.0016355204861611128, 0.0, 0.0, 0.00138184055685997, 0.0, 0.0, 0.002059401711449027, 0.004957709927111864, 0.002279609441757202, 0.0, 0.0, 0.00209897686727345, 0.009249333292245865, 0.007904156111180782, 0.0, 0.0, 0.0, 0.0, 0.006446224171668291, 0.00807473435997963, 0.004241887014359236, 0.00559097109362483, 0.0, 0.0, 0.0024690383579581976, 0.003673736471682787, 0.0039996858686208725, 0.006604751106351614, 0.0006108210072852671, 0.0, 0.0023676359560340643, 0.0, 0.005831256043165922, 0.0, 0.004080916754901409, 0.003039124421775341, 0.0, 0.0020002713426947594, 0.0016317504923790693, 0.0005174187244847417, 0.0, 0.0036780203226953745, 0.001139838364906609, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.00642419746145606, 0.0, 0.0011684726923704147, 0.0, 0.0, 0.0, 0.000252584955887869, 0.0, 0.0, 0.0, 0.0, 0.0, 0.006558170076459646, 0.004437263589352369, 0.004705917555838823, 0.01698058843612671, 0.012763802893459797, 0.039932601153850555, 1.537947654724121]]]},\n            {\n            'hltrCFG': {'tokenization_config': {\"token_prefix\": \"\", \"partial_token_prefix\": \"##\"}\n                }\n            })\n         }, function (err) {\n            console.log(err);\n        })"},"metadata":{}}]},{"cell_type":"markdown","source":"> - Above visualization shows how a pretrained DistilBERT model's prediction activated by input sentence tokens using dimention reduction.\n> - A group of neurons in DistilBERT tend to fire in response to commas and other punctuation. Other groups of neurons tend to fire in response to pronouns. we use this visualization to factorize neuron activity in individual FFNN layers.","metadata":{}},{"cell_type":"markdown","source":"## Our Finetuned Model V Activations:","metadata":{}},{"cell_type":"code","source":"from transformers import AutoTokenizer, AutoModelForSequenceClassification\nmodel_V = AutoModelForSequenceClassification.from_pretrained(\"/kaggle/input/model-new-up/MODELs/MODEL_V\",output_attentions=True)\ntokenizer_V = AutoTokenizer.from_pretrained(\"/kaggle/input/model-new-up/MODELs/MODEL_V\")\nmodel_V","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-31T22:25:48.090568Z","iopub.execute_input":"2023-05-31T22:25:48.091163Z","iopub.status.idle":"2023-05-31T22:25:52.849201Z","shell.execute_reply.started":"2023-05-31T22:25:48.091110Z","shell.execute_reply":"2023-05-31T22:25:52.848286Z"},"trusted":true},"execution_count":11,"outputs":[{"execution_count":11,"output_type":"execute_result","data":{"text/plain":"DistilBertForSequenceClassification(\n  (distilbert): DistilBertModel(\n    (embeddings): Embeddings(\n      (word_embeddings): Embedding(30523, 768, padding_idx=0)\n      (position_embeddings): Embedding(512, 768)\n      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n      (dropout): Dropout(p=0.1, inplace=False)\n    )\n    (transformer): Transformer(\n      (layer): ModuleList(\n        (0-5): 6 x TransformerBlock(\n          (attention): MultiHeadSelfAttention(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n          )\n          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          (ffn): FFN(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n            (activation): GELUActivation()\n          )\n          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n        )\n      )\n    )\n  )\n  (pre_classifier): Linear(in_features=768, out_features=768, bias=True)\n  (classifier): Linear(in_features=768, out_features=1, bias=True)\n  (dropout): Dropout(p=0.2, inplace=False)\n)"},"metadata":{}}]},{"cell_type":"code","source":"model_config_V = {\n    'embedding': \"embeddings.word_embeddings\",\n    'type': 'mlm',\n    'activations':['ffn\\.lin2'],\n    'token_prefix': '',\n    'partial_token_prefix': '##'\n}\nV_model_id = \"/kaggle/input/model-new-up/MODELs/MODEL_V\"","metadata":{"execution":{"iopub.status.busy":"2023-05-31T18:45:15.148450Z","iopub.execute_input":"2023-05-31T18:45:15.148932Z","iopub.status.idle":"2023-05-31T18:45:15.155773Z","shell.execute_reply.started":"2023-05-31T18:45:15.148898Z","shell.execute_reply":"2023-05-31T18:45:15.154335Z"},"trusted":true},"execution_count":59,"outputs":[]},{"cell_type":"code","source":"ecco_model = ecco.from_pretrained(V_model_id, \n                                  activations=True,\n                                  model_config=model_config_V)","metadata":{"execution":{"iopub.status.busy":"2023-05-31T18:45:15.159226Z","iopub.execute_input":"2023-05-31T18:45:15.160583Z","iopub.status.idle":"2023-05-31T18:45:16.347550Z","shell.execute_reply.started":"2023-05-31T18:45:15.160547Z","shell.execute_reply":"2023-05-31T18:45:16.346469Z"},"_kg_hide-output":true,"trusted":true},"execution_count":60,"outputs":[{"name":"stderr","text":"Some weights of the model checkpoint at /kaggle/input/model-new-up/MODELs/MODEL_V were not used when initializing DistilBertModel: ['classifier.bias', 'pre_classifier.bias', 'pre_classifier.weight', 'classifier.weight']\n- This IS expected if you are initializing DistilBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n- This IS NOT expected if you are initializing DistilBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n","output_type":"stream"}]},{"cell_type":"markdown","source":"### Factorizing Activations of a All Layer Layers","metadata":{}},{"cell_type":"code","source":"# Factorize the activations of all layers\ninputs = ecco_model.tokenizer([text], return_tensors=\"pt\")\noutput = ecco_model(inputs)\nnmf_1 = output.run_nmf(n_components=8)                           #non-negative matrix factorization on 8 components\nnmf_1.explore()","metadata":{"execution":{"iopub.status.busy":"2023-05-31T18:45:16.349092Z","iopub.execute_input":"2023-05-31T18:45:16.349550Z","iopub.status.idle":"2023-05-31T18:45:18.413730Z","shell.execute_reply.started":"2023-05-31T18:45:16.349519Z","shell.execute_reply":"2023-05-31T18:45:18.412099Z"},"trusted":true},"execution_count":61,"outputs":[{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"<html lang=\"en\">\n<script src=\"https://requirejs.org/docs/release/2.3.6/minified/require.js\"></script>\n<script>\n    var ecco_url = 'https://storage.googleapis.com/ml-intro/ecco/'\n    //var ecco_url = 'http://localhost:8000/'\n\n    if (window.ecco === undefined) window.ecco = {}\n\n    // Setup the paths of the script we'll be using\n    requirejs.config({\n        urlArgs: \"bust=\" + (new Date()).getTime(),\n        nodeRequire: require,\n        paths: {\n            d3: \"https://d3js.org/d3.v6.min\", // This is only for use in setup.html and basic.html\n            \"d3-array\": \"https://d3js.org/d3-array.v2.min\",\n            jquery: \"https://code.jquery.com/jquery-3.5.1.min\",\n            ecco: ecco_url + 'js/0.0.6/ecco-bundle.min',\n            xregexp: 'https://cdnjs.cloudflare.com/ajax/libs/xregexp/3.2.0/xregexp-all.min'\n        }\n    });\n\n    // Add the css file\n    //requirejs(['d3'],\n    //    function (d3) {\n    //        d3.select('#css').attr('href', ecco_url + 'html/styles.css')\n    //    })\n\n    console.log('Ecco initialize!!')\n\n    // returns a 'basic' object. basic.init() selects the html div we'll be\n    // rendering the html into, adds styles.css to the document.\n    define('basic', ['d3'],\n        function (d3) {\n            return {\n                init: function (viz_id = null) {\n                    if (viz_id == null) {\n                        viz_id = \"viz_\" + Math.round(Math.random() * 10000000)\n                    }\n                    // Select the div rendered below, change its id\n                    const div = d3.select('#basic').attr('id', viz_id),\n                        div_parent = d3.select('#' + viz_id).node().parentNode\n\n                    // Link to CSS file\n                    d3.select(div_parent).insert('link')\n                        .attr('rel', 'stylesheet')\n                        .attr('type', 'text/css')\n                        .attr('href', ecco_url + 'html/0.0.2/styles.css')\n\n                    return viz_id\n                }\n            }\n        }, function (err) {\n            console.log(err);\n        }\n    )\n</script>\n\n<head>\n    <link id='css' rel=\"stylesheet\" type=\"text/css\">\n</head>\n<div id=\"basic\"></div>\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.Javascript object>","application/javascript":"\n         requirejs(['basic', 'ecco'], function(basic, ecco){\n            const viz_id = basic.init()\n            \n            ecco.interactiveTokensAndFactorSparklines(viz_id, {'tokens': [{'token': '[CLS]', 'token_id': 101, 'type': 'input', 'position': 0}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 1}, {'token': 'vast', 'token_id': 6565, 'type': 'input', 'position': 2}, {'token': 'expanse', 'token_id': 22944, 'type': 'input', 'position': 3}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 4}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 5}, {'token': 'ocean', 'token_id': 4153, 'type': 'input', 'position': 6}, {'token': 'stretches', 'token_id': 14082, 'type': 'input', 'position': 7}, {'token': 'out', 'token_id': 2041, 'type': 'input', 'position': 8}, {'token': 'before', 'token_id': 2077, 'type': 'input', 'position': 9}, {'token': 'me', 'token_id': 2033, 'type': 'input', 'position': 10}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 11}, {'token': 'its', 'token_id': 2049, 'type': 'input', 'position': 12}, {'token': 'calm', 'token_id': 5475, 'type': 'input', 'position': 13}, {'token': 'waters', 'token_id': 5380, 'type': 'input', 'position': 14}, {'token': 'reflecting', 'token_id': 10842, 'type': 'input', 'position': 15}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 16}, {'token': 'sky', 'token_id': 3712, 'type': 'input', 'position': 17}, {'token': 'above', 'token_id': 2682, 'type': 'input', 'position': 18}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 19}, {'token': 'its', 'token_id': 2049, 'type': 'input', 'position': 20}, {'token': 'tran', 'token_id': 25283, 'type': 'input', 'position': 21}, {'token': '##quil', 'token_id': 26147, 'type': 'input', 'position': 22}, {'token': 'surface', 'token_id': 3302, 'type': 'input', 'position': 23}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 24}, {'token': 'und', 'token_id': 6151, 'type': 'input', 'position': 25}, {'token': '##ist', 'token_id': 2923, 'type': 'input', 'position': 26}, {'token': '##urbed', 'token_id': 29595, 'type': 'input', 'position': 27}, {'token': 'by', 'token_id': 2011, 'type': 'input', 'position': 28}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 29}, {'token': 'chaos', 'token_id': 8488, 'type': 'input', 'position': 30}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 31}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 32}, {'token': 'world', 'token_id': 2088, 'type': 'input', 'position': 33}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 34}, {'token': 'inspire', 'token_id': 18708, 'type': 'input', 'position': 35}, {'token': '##s', 'token_id': 2015, 'type': 'input', 'position': 36}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 37}, {'token': 'sense', 'token_id': 3168, 'type': 'input', 'position': 38}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 39}, {'token': 'serene', 'token_id': 25388, 'type': 'input', 'position': 40}, {'token': 'tran', 'token_id': 25283, 'type': 'input', 'position': 41}, {'token': '##quil', 'token_id': 26147, 'type': 'input', 'position': 42}, {'token': '##ity', 'token_id': 3012, 'type': 'input', 'position': 43}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 44}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 45}, {'token': 'soft', 'token_id': 3730, 'type': 'input', 'position': 46}, {'token': 'whisper', 'token_id': 7204, 'type': 'input', 'position': 47}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 48}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 49}, {'token': 'waves', 'token_id': 5975, 'type': 'input', 'position': 50}, {'token': 'kissing', 'token_id': 7618, 'type': 'input', 'position': 51}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 52}, {'token': 'shoreline', 'token_id': 17721, 'type': 'input', 'position': 53}, {'token': 'is', 'token_id': 2003, 'type': 'input', 'position': 54}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 55}, {'token': 'peaceful', 'token_id': 9379, 'type': 'input', 'position': 56}, {'token': 'symphony', 'token_id': 6189, 'type': 'input', 'position': 57}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 58}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 59}, {'token': 'mel', 'token_id': 11463, 'type': 'input', 'position': 60}, {'token': '##od', 'token_id': 7716, 'type': 'input', 'position': 61}, {'token': '##ious', 'token_id': 6313, 'type': 'input', 'position': 62}, {'token': 'song', 'token_id': 2299, 'type': 'input', 'position': 63}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 64}, {'token': 'nature', 'token_id': 3267, 'type': 'input', 'position': 65}, {'token': 'that', 'token_id': 2008, 'type': 'input', 'position': 66}, {'token': 'soothe', 'token_id': 28384, 'type': 'input', 'position': 67}, {'token': '##s', 'token_id': 2015, 'type': 'input', 'position': 68}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 69}, {'token': 'soul', 'token_id': 3969, 'type': 'input', 'position': 70}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 71}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 72}, {'token': 'gentle', 'token_id': 7132, 'type': 'input', 'position': 73}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 74}, {'token': 'rhythmic', 'token_id': 14797, 'type': 'input', 'position': 75}, {'token': 'motion', 'token_id': 4367, 'type': 'input', 'position': 76}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 77}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 78}, {'token': 'ocean', 'token_id': 4153, 'type': 'input', 'position': 79}, {'token': '’', 'token_id': 1521, 'type': 'input', 'position': 80}, {'token': 's', 'token_id': 1055, 'type': 'input', 'position': 81}, {'token': 'currents', 'token_id': 14731, 'type': 'input', 'position': 82}, {'token': 'em', 'token_id': 7861, 'type': 'input', 'position': 83}, {'token': '##bo', 'token_id': 5092, 'type': 'input', 'position': 84}, {'token': '##dies', 'token_id': 18389, 'type': 'input', 'position': 85}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 86}, {'token': 'silent', 'token_id': 4333, 'type': 'input', 'position': 87}, {'token': 'strength', 'token_id': 3997, 'type': 'input', 'position': 88}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 89}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 90}, {'token': 'sense', 'token_id': 3168, 'type': 'input', 'position': 91}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 92}, {'token': 'enduring', 'token_id': 16762, 'type': 'input', 'position': 93}, {'token': 'ser', 'token_id': 14262, 'type': 'input', 'position': 94}, {'token': '##enity', 'token_id': 20693, 'type': 'input', 'position': 95}, {'token': 'that', 'token_id': 2008, 'type': 'input', 'position': 96}, {'token': 'is', 'token_id': 2003, 'type': 'input', 'position': 97}, {'token': 'profoundly', 'token_id': 28089, 'type': 'input', 'position': 98}, {'token': 'peaceful', 'token_id': 9379, 'type': 'input', 'position': 99}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 100}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 101}, {'token': 'bound', 'token_id': 5391, 'type': 'input', 'position': 102}, {'token': '##less', 'token_id': 3238, 'type': 'input', 'position': 103}, {'token': 'beauty', 'token_id': 5053, 'type': 'input', 'position': 104}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 105}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 106}, {'token': 'sea', 'token_id': 2712, 'type': 'input', 'position': 107}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 108}, {'token': 'with', 'token_id': 2007, 'type': 'input', 'position': 109}, {'token': 'its', 'token_id': 2049, 'type': 'input', 'position': 110}, {'token': 'capt', 'token_id': 14408, 'type': 'input', 'position': 111}, {'token': '##ivating', 'token_id': 17441, 'type': 'input', 'position': 112}, {'token': 'calm', 'token_id': 5475, 'type': 'input', 'position': 113}, {'token': '##ness', 'token_id': 2791, 'type': 'input', 'position': 114}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 115}, {'token': 'offers', 'token_id': 4107, 'type': 'input', 'position': 116}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 117}, {'token': 'sanctuary', 'token_id': 8493, 'type': 'input', 'position': 118}, {'token': 'from', 'token_id': 2013, 'type': 'input', 'position': 119}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 120}, {'token': 'bust', 'token_id': 13950, 'type': 'input', 'position': 121}, {'token': '##ling', 'token_id': 2989, 'type': 'input', 'position': 122}, {'token': 'noise', 'token_id': 5005, 'type': 'input', 'position': 123}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 124}, {'token': 'everyday', 'token_id': 10126, 'type': 'input', 'position': 125}, {'token': 'life', 'token_id': 2166, 'type': 'input', 'position': 126}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 127}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 128}, {'token': 'haven', 'token_id': 4033, 'type': 'input', 'position': 129}, {'token': 'where', 'token_id': 2073, 'type': 'input', 'position': 130}, {'token': 'one', 'token_id': 2028, 'type': 'input', 'position': 131}, {'token': 'can', 'token_id': 2064, 'type': 'input', 'position': 132}, {'token': 'find', 'token_id': 2424, 'type': 'input', 'position': 133}, {'token': 'peace', 'token_id': 3521, 'type': 'input', 'position': 134}, {'token': 'and', 'token_id': 1998, 'type': 'input', 'position': 135}, {'token': 'solitude', 'token_id': 22560, 'type': 'input', 'position': 136}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 137}, {'token': 'it', 'token_id': 2009, 'type': 'input', 'position': 138}, {'token': '’', 'token_id': 1521, 'type': 'input', 'position': 139}, {'token': 's', 'token_id': 1055, 'type': 'input', 'position': 140}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 141}, {'token': 'place', 'token_id': 2173, 'type': 'input', 'position': 142}, {'token': 'where', 'token_id': 2073, 'type': 'input', 'position': 143}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 144}, {'token': 'mind', 'token_id': 2568, 'type': 'input', 'position': 145}, {'token': 'can', 'token_id': 2064, 'type': 'input', 'position': 146}, {'token': 'rest', 'token_id': 2717, 'type': 'input', 'position': 147}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 148}, {'token': 'allowing', 'token_id': 4352, 'type': 'input', 'position': 149}, {'token': 'thoughts', 'token_id': 4301, 'type': 'input', 'position': 150}, {'token': 'to', 'token_id': 2000, 'type': 'input', 'position': 151}, {'token': 'flow', 'token_id': 4834, 'type': 'input', 'position': 152}, {'token': 'as', 'token_id': 2004, 'type': 'input', 'position': 153}, {'token': 'freely', 'token_id': 10350, 'type': 'input', 'position': 154}, {'token': 'as', 'token_id': 2004, 'type': 'input', 'position': 155}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 156}, {'token': 'waters', 'token_id': 5380, 'type': 'input', 'position': 157}, {'token': 'themselves', 'token_id': 3209, 'type': 'input', 'position': 158}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 159}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 160}, {'token': 'ocean', 'token_id': 4153, 'type': 'input', 'position': 161}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 162}, {'token': 'in', 'token_id': 1999, 'type': 'input', 'position': 163}, {'token': 'its', 'token_id': 2049, 'type': 'input', 'position': 164}, {'token': 'infinite', 'token_id': 10709, 'type': 'input', 'position': 165}, {'token': 'tran', 'token_id': 25283, 'type': 'input', 'position': 166}, {'token': '##quil', 'token_id': 26147, 'type': 'input', 'position': 167}, {'token': '##ity', 'token_id': 3012, 'type': 'input', 'position': 168}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 169}, {'token': 'serves', 'token_id': 4240, 'type': 'input', 'position': 170}, {'token': 'as', 'token_id': 2004, 'type': 'input', 'position': 171}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 172}, {'token': 'reminder', 'token_id': 14764, 'type': 'input', 'position': 173}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 174}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 175}, {'token': 'simple', 'token_id': 3722, 'type': 'input', 'position': 176}, {'token': 'and', 'token_id': 1998, 'type': 'input', 'position': 177}, {'token': 'quiet', 'token_id': 4251, 'type': 'input', 'position': 178}, {'token': 'beauty', 'token_id': 5053, 'type': 'input', 'position': 179}, {'token': 'that', 'token_id': 2008, 'type': 'input', 'position': 180}, {'token': 'exists', 'token_id': 6526, 'type': 'input', 'position': 181}, {'token': 'in', 'token_id': 1999, 'type': 'input', 'position': 182}, {'token': 'our', 'token_id': 2256, 'type': 'input', 'position': 183}, {'token': 'world', 'token_id': 2088, 'type': 'input', 'position': 184}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 185}, {'token': '[SEP]', 'token_id': 102, 'type': 'input', 'position': 186}], 'factors': [[[0.029743395745754242, 0.006922493223100901, 0.006690161768347025, 0.0, 0.0, 0.0, 0.0, 0.026524698361754417, 0.049792882055044174, 0.044849738478660583, 0.03715037181973457, 0.42127469182014465, 0.00029792237910442054, 0.0, 0.0, 0.004608307499438524, 0.002689219079911709, 0.0, 0.027704034000635147, 0.09147462993860245, 0.0, 0.0, 0.0, 0.0, 0.44098663330078125, 0.05341913178563118, 0.040347833186388016, 0.0012064391048625112, 0.04653758183121681, 0.0021512906532734632, 0.0, 0.0, 0.0, 0.0, 0.4026270806789398, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.021922828629612923, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0757426917552948, 0.0, 0.0, 0.0005003131227567792, 0.4034358263015747, 0.0, 0.014672300778329372, 0.021494202315807343, 0.0, 0.0026356603484600782, 0.0, 0.0, 0.11329661309719086, 0.0, 0.0, 0.0, 0.0, 0.024072984233498573, 0.0, 0.0, 0.3608875572681427, 0.0, 0.0, 0.0, 0.0, 0.0, 0.06620768457651138, 0.03049962967634201, 0.0, 0.04272966459393501, 0.047150060534477234, 0.012731372378766537, 0.0, 0.012719077989459038, 0.008991039358079433, 0.4044245183467865, 0.0, 0.0, 0.0, 0.0, 0.013074647635221481, 0.0, 0.0980231761932373, 0.04108251631259918, 0.0027086164336651564, 0.0, 0.0822330191731453, 0.013311367481946945, 0.02952645532786846, 0.01976069062948227, 0.0, 0.0, 0.0, 0.0, 0.42768949270248413, 0.12396589666604996, 0.011547266505658627, 0.0, 0.0, 0.0, 0.027720604091882706, 0.41692665219306946, 0.021966274827718735, 0.0, 0.0, 0.05429404601454735, 0.01184279564768076, 0.026821838691830635, 0.025520429015159607, 0.0, 0.0, 0.0, 0.015805009752511978, 0.38255545496940613, 0.0, 0.0, 0.03739578649401665, 0.02755683660507202, 0.012282699346542358, 0.0, 0.0, 0.12391684949398041, 0.0, 0.06419405341148376, 0.04662410914897919, 0.07544051855802536, 0.04886934161186218, 0.0, 0.0, 0.029464101418852806, 0.010464579798281193, 0.007376667112112045, 0.014728697016835213, 0.02698398567736149, 0.4294579029083252, 0.030932657420635223, 0.00887883361428976, 0.07460267841815948, 0.007501909043639898, 0.10199933499097824, 0.010975775308907032, 0.06876073777675629, 0.0, 0.0, 0.0516974963247776, 0.0, 0.004793842788785696, 0.0, 0.421322762966156, 0.14007563889026642, 0.025781387463212013, 0.023709697648882866, 0.008697432465851307, 0.002904572756960988, 0.027785448357462883, 0.4303431212902069, 0.07042240351438522, 0.07989837229251862, 0.0, 0.0, 0.0, 0.0, 0.005987011361867189, 0.157097727060318, 0.0, 0.0, 0.12242941558361053, 0.01765892095863819, 0.06259439885616302, 0.0, 0.005778443068265915, 0.11899916082620621, 0.0], [0.09137486666440964, 0.0, 0.2129446119070053, 0.4193829596042633, 0.060845475643873215, 0.07586513459682465, 0.6689370274543762, 0.3900998830795288, 0.3354633152484894, 0.27484527230262756, 0.3552618622779846, 0.033528007566928864, 0.1590145379304886, 0.15112358331680298, 0.5906859636306763, 0.2666192054748535, 0.07899351418018341, 0.5185689926147461, 0.39926719665527344, 0.0671924501657486, 0.13570168614387512, 0.0, 0.017345979809761047, 0.4013182520866394, 0.0, 0.052884913980960846, 0.05351923406124115, 0.13235221803188324, 0.1254255622625351, 0.0, 0.31310802698135376, 0.023539740592241287, 0.01987651363015175, 0.6395487189292908, 0.0, 0.12285398691892624, 0.05282089114189148, 0.0, 0.15209251642227173, 0.0, 0.025862647220492363, 0.0, 0.0, 0.09629814326763153, 0.0, 0.0, 0.0, 0.1750432252883911, 0.0, 0.0, 0.5369739532470703, 0.17863133549690247, 0.040545523166656494, 0.4570435583591461, 0.03219065070152283, 0.0, 0.014616376720368862, 0.22659915685653687, 0.0, 0.0, 0.0, 0.0, 0.0, 0.11677893251180649, 0.0, 0.3363519608974457, 0.07088162004947662, 0.022349471226334572, 0.029406713321805, 0.0, 0.3620372712612152, 0.0, 0.0, 0.0335504524409771, 0.0, 0.0848780944943428, 0.22325538098812103, 0.0, 0.0, 0.5484890341758728, 0.028750957921147346, 0.03019292838871479, 0.39871254563331604, 0.0, 0.0, 0.04205577075481415, 0.0, 0.0, 0.14216144382953644, 0.0, 0.0, 0.10690249502658844, 0.0, 0.004151246510446072, 0.0, 0.0847323089838028, 0.07885098457336426, 0.023478621616959572, 0.03923424333333969, 0.06966209411621094, 0.0, 0.0, 0.009095478802919388, 0.0, 0.22063128650188446, 0.0, 0.0, 0.6159073114395142, 0.0, 0.0, 0.03869141638278961, 0.0, 0.0, 0.03538863733410835, 0.158341184258461, 0.0, 0.0957612469792366, 0.0, 0.1828574538230896, 0.06398064643144608, 0.0, 0.025855157524347305, 0.056689973920583725, 0.30129438638687134, 0.02109810896217823, 0.20718008279800415, 0.41032612323760986, 0.0, 0.0027926969341933727, 0.3158465325832367, 0.21331779658794403, 0.2823983132839203, 0.246294766664505, 0.1606307178735733, 0.16534683108329773, 0.029489928856492043, 0.21006573736667633, 0.06507280468940735, 0.16270489990711212, 0.09322284162044525, 0.1275879591703415, 0.03386574983596802, 0.4005955755710602, 0.23496541380882263, 0.10329215228557587, 0.5422928929328918, 0.2865724563598633, 0.2704952359199524, 0.05961001291871071, 0.1796063780784607, 0.504472553730011, 0.1924324333667755, 0.35094326734542847, 0.1895224153995514, 0.27654433250427246, 0.185315802693367, 0.10675787925720215, 0.5885326266288757, 0.380434513092041, 0.0, 0.03687666356563568, 0.64545738697052, 0.023317677900195122, 0.035772696137428284, 0.12498115748167038, 0.080622099339962, 0.0, 0.0, 0.16480842232704163, 0.0, 0.18279975652694702, 0.022457066923379898, 0.0, 0.14363466203212738, 0.0, 0.0, 0.015972666442394257, 0.0, 0.0, 0.27361270785331726, 0.15796858072280884, 0.2781504988670349, 0.1066787913441658, 0.22685860097408295, 0.5656246542930603, 0.07293710857629776, 0.0], [0.014406334608793259, 0.005296164192259312, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0009583255159668624, 0.0, 0.0037482879124581814, 0.0035583716817200184, 0.009485387243330479, 0.0037588304840028286, 0.0, 0.0, 0.009403052739799023, 0.002289712196215987, 0.0, 0.0028711925260722637, 0.45663732290267944, 0.00611518882215023, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0015388424508273602, 0.008186195977032185, 0.0, 0.0, 0.0, 0.0, 0.0, 0.29151666164398193, 0.0, 0.004150915890932083, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0033034496009349823, 0.006183264777064323, 0.7490178346633911, 0.011498160660266876, 0.0, 0.0, 0.0, 0.0, 0.0, 0.004537477623671293, 0.0, 0.0, 0.018197793513536453, 0.0, 0.0, 0.002343408763408661, 0.0, 0.0, 0.002003165427595377, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.00012212010915391147, 0.0062696789391338825, 0.0, 0.0, 0.6473981142044067, 0.002669750712811947, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0032248906791210175, 0.0, 0.0, 0.0009480667067691684, 0.0, 0.0021523265168070793, 0.0, 0.0, 0.0014113406650722027, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0008292117272503674, 0.007399286143481731, 0.003418386448174715, 0.003995117265731096, 0.0, 0.38220927119255066, 0.0017093964852392673, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.005586622282862663, 0.0, 0.0, 0.0, 0.0, 0.0, 0.22968868911266327, 0.0, 0.0, 0.0, 0.004634238313883543, 0.0, 0.0, 0.0, 0.0, 0.0, 7.634440407855436e-05, 0.0, 0.0, 0.002652187133207917, 0.0, 0.009811940602958202, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.5924381613731384, 0.011999992653727531, 0.006190821994096041, 0.003523700637742877, 0.0, 0.0, 0.013097228482365608, 0.003664397168904543, 0.0, 0.0, 0.0032692155800759792, 0.0027844698633998632, 0.007055352441966534, 0.0015082255704328418, 0.00480818934738636, 0.0011575883254408836, 0.0007894358714111149, 0.0018970119999721646, 0.025551812723279, 0.004303836263716221, 0.0, 0.007534333039075136, 1.1226650476455688, 0.021055376157164574, 0.0, 0.0, 0.016838228330016136, 0.0, 0.0, 0.0, 0.0024946534540504217, 0.00040061032632365823, 0.14926467835903168, 0.0037389465142041445, 0.0, 0.0, 0.0033047155011445284, 0.0, 0.0, 0.0009462910238653421, 0.0, 0.003356330329552293, 0.0015478910645470023, 0.0, 0.00555419921875, 0.008770261891186237, 0.0013515434693545103, 0.002097608521580696, 0.1945991814136505, 0.0], [0.052508339285850525, 0.0, 0.0033433393109589815, 0.0, 0.0, 0.0, 0.0, 0.0, 0.007200098130851984, 0.0027485121972858906, 0.024662796407938004, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0008899088716134429, 0.007874192669987679, 0.0, 0.0, 0.002649011556059122, 0.005540217738598585, 0.0011691083200275898, 0.0, 0.017509888857603073, 0.02020714245736599, 0.004373583011329174, 0.0, 0.0, 0.0, 0.0, 0.006719730794429779, 0.0, 0.061473336070775986, 0.0, 0.008094890974462032, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0006698071374557912, 0.0, 0.0, 0.001501113292761147, 0.0, 0.0, 0.0, 0.0, 0.0066514466889202595, 0.006064703222364187, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.005466039292514324, 0.008565318770706654, 0.005352070555090904, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.005179977975785732, 0.0, 0.016049202531576157, 0.01395911443978548, 0.0, 0.012730259448289871, 0.014095930382609367, 0.004632608033716679, 0.0, 0.003981107380241156, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0001375434803776443, 0.011121516115963459, 0.0, 0.0, 0.0, 0.0025762608274817467, 0.0, 0.0, 0.0, 0.011521415784955025, 0.01615596003830433, 0.0, 0.0, 0.005336771719157696, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0029391038697212934, 0.056714121252298355, 0.0, 0.0, 0.0, 0.006077973172068596, 0.002179201925173402, 0.007862890139222145, 0.007858281023800373, 0.0, 0.00015182001516222954, 0.010343037545681, 0.0015153837157413363, 0.0, 0.0, 0.0, 0.0, 0.018747134134173393, 0.009785478934645653, 0.0, 0.0, 0.0, 0.0, 0.0, 0.00596458138898015, 0.015602665953338146, 0.0026211875956505537, 0.0, 0.0, 0.0, 0.004457808565348387, 0.004108715336769819, 0.010633372701704502, 0.004845582880079746, 0.0, 0.004680740647017956, 0.0001537148782517761, 0.011190320365130901, 0.0, 0.004057183396071196, 0.005917873699218035, 0.0, 0.0, 0.0, 0.0028495898004621267, 0.06499844789505005, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0009410841157659888, 0.0021150032989680767, 0.0058576264418661594, 0.036368709057569504, 0.004994524642825127, 0.010212135501205921, 0.001661761780269444, 0.0, 0.0, 0.0, 0.0, 0.0007342626922763884, 0.0, 0.0, 0.007817578501999378, 0.004941945429891348, 0.007996749132871628, 0.02547454461455345, 0.004427060484886169, 0.0, 2.89825701713562], [0.04226412624120712, 0.8150121569633484, 0.1150820404291153, 0.0024702197406440973, 0.008289098739624023, 0.7868454456329346, 0.0, 0.0, 0.01453856285661459, 0.027648217976093292, 0.08015824854373932, 0.04582276567816734, 0.329101026058197, 0.020660817623138428, 0.0, 0.0, 0.7678307294845581, 0.008646398782730103, 0.01059836894273758, 0.04951586201786995, 0.3694365322589874, 0.022529156878590584, 0.030846767127513885, 0.0, 0.03860165923833847, 0.02564254216849804, 0.013616006821393967, 0.0, 0.05591438338160515, 0.7899143695831299, 0.006538449786603451, 0.0, 0.7499192953109741, 0.0011436641216278076, 0.0, 0.0, 0.0, 0.040641382336616516, 0.0, 0.0, 0.0, 0.010537277907133102, 0.0, 0.0, 0.023717831820249557, 0.755611777305603, 0.07574644684791565, 0.0, 0.0, 0.7863598465919495, 0.0, 0.0, 0.7286601662635803, 0.0, 0.009870046749711037, 0.02180841751396656, 0.0, 0.0, 0.0, 0.040781375020742416, 0.0, 0.014933495782315731, 0.0, 0.0, 0.0, 0.008990042842924595, 0.0, 0.0, 0.0, 0.7279991507530212, 0.0, 0.040588200092315674, 0.8099671006202698, 0.008299389854073524, 0.053005021065473557, 0.0, 0.0, 0.012267411686480045, 0.7819439768791199, 0.0, 0.05689217522740364, 0.22916078567504883, 0.0, 0.0, 0.0, 0.0, 0.030417421832680702, 0.023600034415721893, 0.0, 0.0, 0.01808794215321541, 0.0, 0.0, 0.010657290928065777, 0.011586409993469715, 0.0, 0.025555992498993874, 0.03325268253684044, 0.04047682508826256, 0.0, 0.046238332986831665, 0.7172890901565552, 0.0244680754840374, 0.05656271055340767, 0.01205764152109623, 0.004769256804138422, 0.7913892865180969, 0.0, 0.018364226445555687, 0.06528791040182114, 0.32361486554145813, 0.0, 0.0, 0.0, 0.014980150386691093, 0.0, 0.0, 0.0, 0.0, 0.0, 0.6913754940032959, 0.012521496042609215, 0.05745597183704376, 0.0, 0.0043752132914960384, 0.08417946100234985, 0.021449364721775055, 0.0013670196058228612, 0.057383257895708084, 0.0, 0.0, 0.059332601726055145, 0.0, 0.0, 0.0, 0.04845843091607094, 0.0, 0.002001402899622917, 0.1555246114730835, 0.003521119710057974, 0.020744824782013893, 0.01547103188931942, 0.0, 0.0, 0.640884280204773, 0.021360788494348526, 0.0, 0.0, 0.0, 0.0, 0.003533928655087948, 0.0, 0.0, 0.0, 0.0, 0.005053561646491289, 0.6874319911003113, 0.0, 0.0256551094353199, 0.0, 0.7788050770759583, 0.0, 0.03369930759072304, 0.07740414142608643, 0.3088324964046478, 0.03942648321390152, 0.021363765001296997, 0.013074841350317001, 0.03411552682518959, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.7072409987449646, 0.02834826149046421, 0.06763308495283127, 0.009663421660661697, 0.0, 0.020642854273319244, 0.0, 0.03373834863305092, 0.22377854585647583, 0.010260554030537605, 0.022565403953194618, 0.0], [0.033560313284397125, 0.0, 0.22088627517223358, 0.14651983976364136, 0.0, 0.0, 0.0, 0.08994259685277939, 0.0, 0.0, 0.0, 0.0, 0.0, 0.382264107465744, 0.028871584683656693, 0.20725871622562408, 0.0, 0.02963063307106495, 0.0, 0.0, 0.0, 0.5030587315559387, 0.42588308453559875, 0.09449460357427597, 0.0211603082716465, 0.26165327429771423, 0.248890683054924, 0.3088100850582123, 0.015520458109676838, 0.0, 0.19690942764282227, 0.0, 0.0, 0.0, 0.0, 0.3405066728591919, 0.08263321965932846, 0.008215438574552536, 0.27588823437690735, 0.07896450161933899, 0.5485104322433472, 0.4950920045375824, 0.4690485894680023, 0.31036150455474854, 0.011604021303355694, 0.008982938714325428, 0.4463896155357361, 0.3384222388267517, 0.0, 0.028146013617515564, 0.06529729813337326, 0.23242412507534027, 0.003584999591112137, 0.10734178125858307, 0.010092946700751781, 0.0, 0.5436052680015564, 0.2801937460899353, 0.0, 0.0, 0.40691977739334106, 0.30708086490631104, 0.400568425655365, 0.39830684661865234, 0.0660819560289383, 0.20593591034412384, 0.06696581840515137, 0.4957316219806671, 0.1155751422047615, 0.041562262922525406, 0.19334512948989868, 0.04828022047877312, 0.032808806747198105, 0.4816208779811859, 0.10995279252529144, 0.4326202869415283, 0.2939961850643158, 0.0, 0.0055962069891393185, 0.0, 0.046536751091480255, 0.07390890270471573, 0.1555914282798767, 0.29921606183052063, 0.2766559422016144, 0.18355277180671692, 0.006217191461473703, 0.4401731491088867, 0.344588041305542, 0.02116478979587555, 0.0, 0.2888280749320984, 0.06774181872606277, 0.42624539136886597, 0.38050830364227295, 0.414763480424881, 0.060082267969846725, 0.10175834596157074, 0.372467964887619, 0.515216052532196, 0.03144450858235359, 0.016628559678792953, 0.350839227437973, 0.2958014905452728, 0.3516679108142853, 0.0, 0.013084209524095058, 0.0, 0.01825345680117607, 0.03448937088251114, 0.08178848773241043, 0.5078865885734558, 0.45777568221092224, 0.4926922917366028, 0.2588904798030853, 0.0, 0.14256452023983002, 0.0, 0.33189958333969116, 0.03772147744894028, 0.0, 0.36647310853004456, 0.23877494037151337, 0.19412489235401154, 0.0, 0.20842382311820984, 0.07244995981454849, 0.0, 0.0, 0.17727811634540558, 0.0, 0.0, 0.023585600778460503, 0.1051148772239685, 0.4136022627353668, 0.058428045362234116, 0.3383398950099945, 0.0, 0.0, 0.0, 0.0, 0.0, 0.026266546919941902, 0.0, 0.0, 0.0005256838048808277, 0.007460192311555147, 0.16401690244674683, 0.0, 0.15298989415168762, 0.07171300798654556, 0.0, 0.1717834621667862, 0.0, 0.24973733723163605, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.3457806408405304, 0.45740988850593567, 0.41235166788101196, 0.22557522356510162, 0.0, 0.05435526743531227, 0.0, 0.0, 0.2959027886390686, 0.0, 0.015432165004312992, 0.4280444085597992, 0.10594986379146576, 0.4947974979877472, 0.3262506425380707, 0.0, 0.0747290626168251, 0.0, 0.0, 0.0, 0.0, 0.0], [0.06308940798044205, 0.05466219410300255, 0.0, 0.0, 0.0, 0.0, 0.0, 0.028621377423405647, 0.050471775233745575, 0.010003921575844288, 0.04931485280394554, 0.0, 0.24535857141017914, 0.0, 0.0, 0.01058275904506445, 0.0, 0.0, 0.0, 0.031170833855867386, 0.2200741171836853, 0.0, 0.0, 0.0, 0.01037346851080656, 0.046586647629737854, 0.052241697907447815, 0.0009550144895911217, 0.0766473039984703, 0.002865736372768879, 0.0, 0.0, 0.0, 0.0, 0.0, 0.11274552345275879, 0.4359757900238037, 0.8044565916061401, 0.2036849707365036, 0.2766483724117279, 0.008449813351035118, 0.0418362133204937, 0.0236512441188097, 0.11661142855882645, 0.0607483945786953, 0.12027671933174133, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.014416033402085304, 0.0, 0.47865208983421326, 0.8044960498809814, 0.0, 0.0, 0.16402080655097961, 0.7813495993614197, 0.06703779101371765, 0.061187468469142914, 0.05644598603248596, 0.0, 0.07549817115068436, 0.010888807475566864, 0.3012007772922516, 0.0, 0.31188932061195374, 0.09833305329084396, 0.0, 0.06265692412853241, 0.09029199182987213, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.04119748994708061, 0.14566536247730255, 0.0, 0.12464946508407593, 0.09522156417369843, 0.23274433612823486, 0.7967928647994995, 0.013173684477806091, 0.003174928016960621, 0.14346612989902496, 0.8146015405654907, 0.18075737357139587, 0.2695806324481964, 0.044774509966373444, 0.028789900243282318, 0.0, 0.2821175158023834, 0.4391977787017822, 0.09371887892484665, 0.0, 0.11012719571590424, 0.1278511881828308, 0.0, 0.014199082739651203, 0.0, 0.0, 0.0, 0.0, 0.0, 0.19825394451618195, 0.30572596192359924, 0.0658612996339798, 0.05883316695690155, 0.0, 0.04451796039938927, 0.0, 0.3957034945487976, 0.770898699760437, 0.028470154851675034, 0.09900061041116714, 0.03411224111914635, 0.0, 0.0017934078350663185, 0.0, 0.0, 0.0, 0.0, 0.10956467688083649, 0.6891818046569824, 0.04089343920350075, 0.24595440924167633, 0.18948429822921753, 0.29238516092300415, 0.24605117738246918, 0.01155272126197815, 0.17248037457466125, 0.0, 0.0669964924454689, 0.24415867030620575, 0.1325192004442215, 0.42303162813186646, 0.698082447052002, 0.18484531342983246, 0.2313651442527771, 0.03062617778778076, 0.0, 0.27651846408843994, 0.027108697220683098, 0.016754409298300743, 0.17751358449459076, 0.0, 0.2158774435520172, 0.012700160965323448, 0.1637583076953888, 0.01424212846904993, 0.06047128140926361, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.07483594119548798, 0.24639274179935455, 0.0, 0.0, 0.0, 0.0, 0.0, 0.21435803174972534, 0.4592134356498718, 0.7499702572822571, 0.04990120604634285, 0.11473564058542252, 0.11912748217582703, 0.0012624001828953624, 0.13668479025363922, 0.0, 0.0, 0.13006508350372314, 0.10949233174324036, 0.0337480753660202, 0.08209029585123062, 0.007612020242959261, 0.07168937474489212, 0.0], [0.01774933747947216, 0.0, 0.0033737870398908854, 0.0, 0.7973756194114685, 0.0029199218843132257, 0.0, 0.0, 0.1114860326051712, 0.19353139400482178, 0.005938425660133362, 0.005658237729221582, 0.0, 0.0, 0.0, 0.06430073082447052, 0.03224910795688629, 0.0, 0.06443218141794205, 0.04961702972650528, 0.0, 0.0, 0.010411123745143414, 0.0, 0.0, 0.0, 0.024482211098074913, 0.0, 0.28692224621772766, 0.03042083978652954, 0.003043354954570532, 0.8109540939331055, 0.0715823620557785, 0.008846900425851345, 0.0, 0.0, 0.1339889019727707, 0.0, 0.004481078125536442, 0.6338255405426025, 0.0, 0.0, 0.006786926183849573, 0.060050442814826965, 0.0326995886862278, 0.0, 0.0, 0.031140854582190514, 0.8765799403190613, 0.03945527225732803, 0.0, 0.08000519126653671, 0.0415484756231308, 0.0, 0.07360877841711044, 0.0, 0.0, 0.0, 0.0, 0.0, 0.005416145082563162, 0.02982628345489502, 0.024192197248339653, 0.04301876202225685, 0.7748070359230042, 0.01072894036769867, 0.13356663286685944, 0.0, 0.24780845642089844, 0.03727039322257042, 0.0, 0.041948333382606506, 0.0, 0.0, 0.0335233137011528, 0.0, 0.055759452283382416, 0.8471130132675171, 0.06571824848651886, 0.0, 0.07737050950527191, 0.10972409695386887, 0.0, 0.014150389470160007, 0.006628240924328566, 0.019426696002483368, 0.0, 0.0, 0.008595081977546215, 0.004773433785885572, 0.0, 0.01859111338853836, 0.6173470616340637, 0.023991061374545097, 0.028694070875644684, 0.009286628104746342, 0.15361200273036957, 0.10044202953577042, 0.002532162005081773, 0.0, 0.07708911597728729, 0.0, 0.021190794184803963, 0.10118358582258224, 0.03294016048312187, 0.872623860836029, 0.059819187968969345, 0.0, 0.0, 0.1549849510192871, 0.0, 0.0, 0.028838753700256348, 0.0, 0.06342097371816635, 0.0, 0.0, 0.0, 0.0, 0.3186172842979431, 0.03830726072192192, 0.01064301561564207, 0.09751132875680923, 0.04340556263923645, 0.7650490403175354, 0.011270835995674133, 0.004233680199831724, 0.008151433430612087, 0.0, 0.0, 0.11733119934797287, 0.04695788770914078, 0.05329301580786705, 0.04280105605721474, 0.0, 0.17767849564552307, 0.0, 0.0, 0.0, 0.036241430789232254, 0.02757057175040245, 0.0, 0.0, 0.14722612500190735, 0.0, 0.0, 0.0681256353855133, 0.009454984217882156, 0.0, 0.03150478005409241, 0.0, 0.15152841806411743, 0.0032681634183973074, 0.15635403990745544, 0.009602855890989304, 0.2696964740753174, 0.0, 0.0, 0.05916449800133705, 0.0, 0.0, 0.0, 0.0, 0.22547511756420135, 0.0021158664021641016, 0.006661366205662489, 0.0, 0.005199764389544725, 0.05214240401983261, 0.0, 0.0, 0.06879425793886185, 0.0, 0.00026902847457677126, 0.7310788035392761, 0.014601482078433037, 0.0, 0.14947232604026794, 0.0, 0.0029104307759553194, 0.19643540680408478, 0.047797273844480515, 0.41198641061782837, 0.047708749771118164, 0.01649654284119606, 0.0600125752389431, 0.0]]]},\n            {\n            'hltrCFG': {'tokenization_config': {\"token_prefix\": \"\", \"partial_token_prefix\": \"##\"}\n                }\n            })\n         }, function (err) {\n            console.log(err);\n        })"},"metadata":{}}]},{"cell_type":"markdown","source":"### Factorizing Activations of a Single Layer","metadata":{}},{"cell_type":"code","source":"# Factorize the activations of only the first layer\nnmf_2 = output.run_nmf(n_components=8, from_layer=0, to_layer=1)\nnmf_2.explore()","metadata":{"execution":{"iopub.status.busy":"2023-05-31T18:45:18.416586Z","iopub.execute_input":"2023-05-31T18:45:18.417525Z","iopub.status.idle":"2023-05-31T18:45:18.606552Z","shell.execute_reply.started":"2023-05-31T18:45:18.417464Z","shell.execute_reply":"2023-05-31T18:45:18.605090Z"},"trusted":true},"execution_count":62,"outputs":[{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"<html lang=\"en\">\n<script src=\"https://requirejs.org/docs/release/2.3.6/minified/require.js\"></script>\n<script>\n    var ecco_url = 'https://storage.googleapis.com/ml-intro/ecco/'\n    //var ecco_url = 'http://localhost:8000/'\n\n    if (window.ecco === undefined) window.ecco = {}\n\n    // Setup the paths of the script we'll be using\n    requirejs.config({\n        urlArgs: \"bust=\" + (new Date()).getTime(),\n        nodeRequire: require,\n        paths: {\n            d3: \"https://d3js.org/d3.v6.min\", // This is only for use in setup.html and basic.html\n            \"d3-array\": \"https://d3js.org/d3-array.v2.min\",\n            jquery: \"https://code.jquery.com/jquery-3.5.1.min\",\n            ecco: ecco_url + 'js/0.0.6/ecco-bundle.min',\n            xregexp: 'https://cdnjs.cloudflare.com/ajax/libs/xregexp/3.2.0/xregexp-all.min'\n        }\n    });\n\n    // Add the css file\n    //requirejs(['d3'],\n    //    function (d3) {\n    //        d3.select('#css').attr('href', ecco_url + 'html/styles.css')\n    //    })\n\n    console.log('Ecco initialize!!')\n\n    // returns a 'basic' object. basic.init() selects the html div we'll be\n    // rendering the html into, adds styles.css to the document.\n    define('basic', ['d3'],\n        function (d3) {\n            return {\n                init: function (viz_id = null) {\n                    if (viz_id == null) {\n                        viz_id = \"viz_\" + Math.round(Math.random() * 10000000)\n                    }\n                    // Select the div rendered below, change its id\n                    const div = d3.select('#basic').attr('id', viz_id),\n                        div_parent = d3.select('#' + viz_id).node().parentNode\n\n                    // Link to CSS file\n                    d3.select(div_parent).insert('link')\n                        .attr('rel', 'stylesheet')\n                        .attr('type', 'text/css')\n                        .attr('href', ecco_url + 'html/0.0.2/styles.css')\n\n                    return viz_id\n                }\n            }\n        }, function (err) {\n            console.log(err);\n        }\n    )\n</script>\n\n<head>\n    <link id='css' rel=\"stylesheet\" type=\"text/css\">\n</head>\n<div id=\"basic\"></div>\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.Javascript object>","application/javascript":"\n         requirejs(['basic', 'ecco'], function(basic, ecco){\n            const viz_id = basic.init()\n            \n            ecco.interactiveTokensAndFactorSparklines(viz_id, {'tokens': [{'token': '[CLS]', 'token_id': 101, 'type': 'input', 'position': 0}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 1}, {'token': 'vast', 'token_id': 6565, 'type': 'input', 'position': 2}, {'token': 'expanse', 'token_id': 22944, 'type': 'input', 'position': 3}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 4}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 5}, {'token': 'ocean', 'token_id': 4153, 'type': 'input', 'position': 6}, {'token': 'stretches', 'token_id': 14082, 'type': 'input', 'position': 7}, {'token': 'out', 'token_id': 2041, 'type': 'input', 'position': 8}, {'token': 'before', 'token_id': 2077, 'type': 'input', 'position': 9}, {'token': 'me', 'token_id': 2033, 'type': 'input', 'position': 10}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 11}, {'token': 'its', 'token_id': 2049, 'type': 'input', 'position': 12}, {'token': 'calm', 'token_id': 5475, 'type': 'input', 'position': 13}, {'token': 'waters', 'token_id': 5380, 'type': 'input', 'position': 14}, {'token': 'reflecting', 'token_id': 10842, 'type': 'input', 'position': 15}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 16}, {'token': 'sky', 'token_id': 3712, 'type': 'input', 'position': 17}, {'token': 'above', 'token_id': 2682, 'type': 'input', 'position': 18}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 19}, {'token': 'its', 'token_id': 2049, 'type': 'input', 'position': 20}, {'token': 'tran', 'token_id': 25283, 'type': 'input', 'position': 21}, {'token': '##quil', 'token_id': 26147, 'type': 'input', 'position': 22}, {'token': 'surface', 'token_id': 3302, 'type': 'input', 'position': 23}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 24}, {'token': 'und', 'token_id': 6151, 'type': 'input', 'position': 25}, {'token': '##ist', 'token_id': 2923, 'type': 'input', 'position': 26}, {'token': '##urbed', 'token_id': 29595, 'type': 'input', 'position': 27}, {'token': 'by', 'token_id': 2011, 'type': 'input', 'position': 28}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 29}, {'token': 'chaos', 'token_id': 8488, 'type': 'input', 'position': 30}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 31}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 32}, {'token': 'world', 'token_id': 2088, 'type': 'input', 'position': 33}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 34}, {'token': 'inspire', 'token_id': 18708, 'type': 'input', 'position': 35}, {'token': '##s', 'token_id': 2015, 'type': 'input', 'position': 36}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 37}, {'token': 'sense', 'token_id': 3168, 'type': 'input', 'position': 38}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 39}, {'token': 'serene', 'token_id': 25388, 'type': 'input', 'position': 40}, {'token': 'tran', 'token_id': 25283, 'type': 'input', 'position': 41}, {'token': '##quil', 'token_id': 26147, 'type': 'input', 'position': 42}, {'token': '##ity', 'token_id': 3012, 'type': 'input', 'position': 43}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 44}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 45}, {'token': 'soft', 'token_id': 3730, 'type': 'input', 'position': 46}, {'token': 'whisper', 'token_id': 7204, 'type': 'input', 'position': 47}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 48}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 49}, {'token': 'waves', 'token_id': 5975, 'type': 'input', 'position': 50}, {'token': 'kissing', 'token_id': 7618, 'type': 'input', 'position': 51}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 52}, {'token': 'shoreline', 'token_id': 17721, 'type': 'input', 'position': 53}, {'token': 'is', 'token_id': 2003, 'type': 'input', 'position': 54}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 55}, {'token': 'peaceful', 'token_id': 9379, 'type': 'input', 'position': 56}, {'token': 'symphony', 'token_id': 6189, 'type': 'input', 'position': 57}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 58}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 59}, {'token': 'mel', 'token_id': 11463, 'type': 'input', 'position': 60}, {'token': '##od', 'token_id': 7716, 'type': 'input', 'position': 61}, {'token': '##ious', 'token_id': 6313, 'type': 'input', 'position': 62}, {'token': 'song', 'token_id': 2299, 'type': 'input', 'position': 63}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 64}, {'token': 'nature', 'token_id': 3267, 'type': 'input', 'position': 65}, {'token': 'that', 'token_id': 2008, 'type': 'input', 'position': 66}, {'token': 'soothe', 'token_id': 28384, 'type': 'input', 'position': 67}, {'token': '##s', 'token_id': 2015, 'type': 'input', 'position': 68}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 69}, {'token': 'soul', 'token_id': 3969, 'type': 'input', 'position': 70}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 71}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 72}, {'token': 'gentle', 'token_id': 7132, 'type': 'input', 'position': 73}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 74}, {'token': 'rhythmic', 'token_id': 14797, 'type': 'input', 'position': 75}, {'token': 'motion', 'token_id': 4367, 'type': 'input', 'position': 76}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 77}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 78}, {'token': 'ocean', 'token_id': 4153, 'type': 'input', 'position': 79}, {'token': '’', 'token_id': 1521, 'type': 'input', 'position': 80}, {'token': 's', 'token_id': 1055, 'type': 'input', 'position': 81}, {'token': 'currents', 'token_id': 14731, 'type': 'input', 'position': 82}, {'token': 'em', 'token_id': 7861, 'type': 'input', 'position': 83}, {'token': '##bo', 'token_id': 5092, 'type': 'input', 'position': 84}, {'token': '##dies', 'token_id': 18389, 'type': 'input', 'position': 85}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 86}, {'token': 'silent', 'token_id': 4333, 'type': 'input', 'position': 87}, {'token': 'strength', 'token_id': 3997, 'type': 'input', 'position': 88}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 89}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 90}, {'token': 'sense', 'token_id': 3168, 'type': 'input', 'position': 91}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 92}, {'token': 'enduring', 'token_id': 16762, 'type': 'input', 'position': 93}, {'token': 'ser', 'token_id': 14262, 'type': 'input', 'position': 94}, {'token': '##enity', 'token_id': 20693, 'type': 'input', 'position': 95}, {'token': 'that', 'token_id': 2008, 'type': 'input', 'position': 96}, {'token': 'is', 'token_id': 2003, 'type': 'input', 'position': 97}, {'token': 'profoundly', 'token_id': 28089, 'type': 'input', 'position': 98}, {'token': 'peaceful', 'token_id': 9379, 'type': 'input', 'position': 99}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 100}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 101}, {'token': 'bound', 'token_id': 5391, 'type': 'input', 'position': 102}, {'token': '##less', 'token_id': 3238, 'type': 'input', 'position': 103}, {'token': 'beauty', 'token_id': 5053, 'type': 'input', 'position': 104}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 105}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 106}, {'token': 'sea', 'token_id': 2712, 'type': 'input', 'position': 107}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 108}, {'token': 'with', 'token_id': 2007, 'type': 'input', 'position': 109}, {'token': 'its', 'token_id': 2049, 'type': 'input', 'position': 110}, {'token': 'capt', 'token_id': 14408, 'type': 'input', 'position': 111}, {'token': '##ivating', 'token_id': 17441, 'type': 'input', 'position': 112}, {'token': 'calm', 'token_id': 5475, 'type': 'input', 'position': 113}, {'token': '##ness', 'token_id': 2791, 'type': 'input', 'position': 114}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 115}, {'token': 'offers', 'token_id': 4107, 'type': 'input', 'position': 116}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 117}, {'token': 'sanctuary', 'token_id': 8493, 'type': 'input', 'position': 118}, {'token': 'from', 'token_id': 2013, 'type': 'input', 'position': 119}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 120}, {'token': 'bust', 'token_id': 13950, 'type': 'input', 'position': 121}, {'token': '##ling', 'token_id': 2989, 'type': 'input', 'position': 122}, {'token': 'noise', 'token_id': 5005, 'type': 'input', 'position': 123}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 124}, {'token': 'everyday', 'token_id': 10126, 'type': 'input', 'position': 125}, {'token': 'life', 'token_id': 2166, 'type': 'input', 'position': 126}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 127}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 128}, {'token': 'haven', 'token_id': 4033, 'type': 'input', 'position': 129}, {'token': 'where', 'token_id': 2073, 'type': 'input', 'position': 130}, {'token': 'one', 'token_id': 2028, 'type': 'input', 'position': 131}, {'token': 'can', 'token_id': 2064, 'type': 'input', 'position': 132}, {'token': 'find', 'token_id': 2424, 'type': 'input', 'position': 133}, {'token': 'peace', 'token_id': 3521, 'type': 'input', 'position': 134}, {'token': 'and', 'token_id': 1998, 'type': 'input', 'position': 135}, {'token': 'solitude', 'token_id': 22560, 'type': 'input', 'position': 136}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 137}, {'token': 'it', 'token_id': 2009, 'type': 'input', 'position': 138}, {'token': '’', 'token_id': 1521, 'type': 'input', 'position': 139}, {'token': 's', 'token_id': 1055, 'type': 'input', 'position': 140}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 141}, {'token': 'place', 'token_id': 2173, 'type': 'input', 'position': 142}, {'token': 'where', 'token_id': 2073, 'type': 'input', 'position': 143}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 144}, {'token': 'mind', 'token_id': 2568, 'type': 'input', 'position': 145}, {'token': 'can', 'token_id': 2064, 'type': 'input', 'position': 146}, {'token': 'rest', 'token_id': 2717, 'type': 'input', 'position': 147}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 148}, {'token': 'allowing', 'token_id': 4352, 'type': 'input', 'position': 149}, {'token': 'thoughts', 'token_id': 4301, 'type': 'input', 'position': 150}, {'token': 'to', 'token_id': 2000, 'type': 'input', 'position': 151}, {'token': 'flow', 'token_id': 4834, 'type': 'input', 'position': 152}, {'token': 'as', 'token_id': 2004, 'type': 'input', 'position': 153}, {'token': 'freely', 'token_id': 10350, 'type': 'input', 'position': 154}, {'token': 'as', 'token_id': 2004, 'type': 'input', 'position': 155}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 156}, {'token': 'waters', 'token_id': 5380, 'type': 'input', 'position': 157}, {'token': 'themselves', 'token_id': 3209, 'type': 'input', 'position': 158}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 159}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 160}, {'token': 'ocean', 'token_id': 4153, 'type': 'input', 'position': 161}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 162}, {'token': 'in', 'token_id': 1999, 'type': 'input', 'position': 163}, {'token': 'its', 'token_id': 2049, 'type': 'input', 'position': 164}, {'token': 'infinite', 'token_id': 10709, 'type': 'input', 'position': 165}, {'token': 'tran', 'token_id': 25283, 'type': 'input', 'position': 166}, {'token': '##quil', 'token_id': 26147, 'type': 'input', 'position': 167}, {'token': '##ity', 'token_id': 3012, 'type': 'input', 'position': 168}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 169}, {'token': 'serves', 'token_id': 4240, 'type': 'input', 'position': 170}, {'token': 'as', 'token_id': 2004, 'type': 'input', 'position': 171}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 172}, {'token': 'reminder', 'token_id': 14764, 'type': 'input', 'position': 173}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 174}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 175}, {'token': 'simple', 'token_id': 3722, 'type': 'input', 'position': 176}, {'token': 'and', 'token_id': 1998, 'type': 'input', 'position': 177}, {'token': 'quiet', 'token_id': 4251, 'type': 'input', 'position': 178}, {'token': 'beauty', 'token_id': 5053, 'type': 'input', 'position': 179}, {'token': 'that', 'token_id': 2008, 'type': 'input', 'position': 180}, {'token': 'exists', 'token_id': 6526, 'type': 'input', 'position': 181}, {'token': 'in', 'token_id': 1999, 'type': 'input', 'position': 182}, {'token': 'our', 'token_id': 2256, 'type': 'input', 'position': 183}, {'token': 'world', 'token_id': 2088, 'type': 'input', 'position': 184}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 185}, {'token': '[SEP]', 'token_id': 102, 'type': 'input', 'position': 186}], 'factors': [[[0.02944551594555378, 0.026202082633972168, 0.0, 0.018091905862092972, 0.03870848938822746, 0.001335563836619258, 0.0, 0.08321927487850189, 0.18194280564785004, 0.22232525050640106, 0.14354228973388672, 0.08524391055107117, 0.0, 0.00040934313437901437, 0.053452230989933014, 0.05606776103377342, 0.08049427717924118, 0.02959708496928215, 0.11379546672105789, 0.4519398510456085, 0.016638556495308876, 0.0, 0.0037464334163814783, 0.032233450561761856, 0.04799061268568039, 0.06339415162801743, 0.06738106161355972, 0.007632484193891287, 0.2040669023990631, 0.04559936001896858, 0.004895497113466263, 0.051286451518535614, 0.026503518223762512, 0.04496951401233673, 0.030572235584259033, 0.0, 0.15855634212493896, 0.0455617755651474, 0.011657651513814926, 0.0, 0.0, 0.0, 0.000442092539742589, 0.05994756892323494, 0.4441329836845398, 0.02384720928966999, 0.0, 0.0, 0.015529634431004524, 0.0, 0.0, 0.0454193539917469, 0.0, 0.0, 0.12256776541471481, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.05264709144830704, 0.0, 0.132863849401474, 0.0, 0.008123625069856644, 0.33924785256385803, 0.0, 0.0, 0.0, 0.0, 0.000851588963996619, 0.0, 0.0, 0.0, 0.0024456200189888477, 0.022899853065609932, 0.0, 0.023297930136322975, 0.008844258263707161, 0.047259777784347534, 0.0, 0.0, 0.0027254207525402308, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0028529074043035507, 0.005120276473462582, 0.10005360841751099, 0.16261722147464752, 0.0, 0.0, 0.36796024441719055, 0.0, 0.010234531946480274, 0.004605405032634735, 0.018920309841632843, 0.0073363156989216805, 0.0, 0.0, 0.0, 0.08296690136194229, 0.0, 0.0, 0.0, 0.0, 0.012726571410894394, 0.0, 0.12027014791965485, 0.030058158561587334, 0.014068441465497017, 0.12067817151546478, 0.0, 0.033910494297742844, 0.01873292587697506, 0.016068559139966965, 0.020917285233736038, 0.037557050585746765, 0.06097005680203438, 0.06821577996015549, 0.07531002908945084, 0.06312978267669678, 0.300229549407959, 0.22834353148937225, 0.2568037509918213, 0.1636679321527481, 0.059711944311857224, 0.2723669707775116, 0.03103991039097309, 0.48113545775413513, 0.2074120193719864, 0.16150793433189392, 0.20457999408245087, 0.1318671703338623, 0.15536673367023468, 0.33478301763534546, 0.16290614008903503, 0.1270921379327774, 0.26470765471458435, 0.1741580069065094, 0.13762162625789642, 0.2151457816362381, 0.12430465966463089, 0.27515003085136414, 0.08980758488178253, 0.3885301649570465, 0.08648902177810669, 0.345539391040802, 0.09840479493141174, 0.07060561329126358, 0.13752083480358124, 0.5051928758621216, 0.09167920053005219, 0.00605526939034462, 0.1087058037519455, 0.2660926580429077, 0.029601728543639183, 0.006836307235062122, 0.0, 0.013051139190793037, 0.068209208548069, 0.09980589896440506, 0.16172489523887634, 0.2755085229873657, 0.06519098579883575, 0.04623439908027649, 0.09353052079677582, 0.08622467517852783, 0.05224775895476341, 0.21359704434871674, 0.05426428094506264, 0.07226264476776123, 0.19182851910591125, 0.18833814561367035, 0.30352893471717834, 0.1180807501077652, 0.12265840172767639, 0.42432844638824463, 0.20532149076461792], [0.003303857985883951, 0.0, 0.054227981716394424, 0.0, 1.10982084274292, 0.0, 0.0, 0.0, 0.08042021840810776, 0.11015292257070541, 0.022599171847105026, 0.0, 0.0, 0.0, 0.0409860759973526, 0.0, 0.004812410566955805, 0.012881866656243801, 0.03958429396152496, 0.0, 0.0, 0.0, 0.028995370492339134, 0.011856851167976856, 0.0, 0.0, 0.0, 0.0, 0.1573222428560257, 0.018148943781852722, 0.0, 0.9774999022483826, 0.0, 0.08099967241287231, 0.0, 0.027952106669545174, 0.12953805923461914, 0.004220161121338606, 0.0, 1.1286166906356812, 0.0, 0.011404452845454216, 0.03600352630019188, 0.06623126566410065, 0.0, 0.00630624545738101, 0.0, 0.0, 1.1219167709350586, 0.007131099235266447, 0.0, 0.02262110449373722, 0.06899362057447433, 0.0, 0.21432696282863617, 0.0, 0.0, 0.02722053788602352, 0.028712375089526176, 0.0078050605952739716, 0.010768233798444271, 0.03207049146294594, 0.033167514950037, 0.12353050708770752, 1.1735504865646362, 0.08614751696586609, 0.22616301476955414, 0.0, 0.1619441658258438, 0.08475267887115479, 0.034493640065193176, 0.04885091632604599, 0.007231690920889378, 0.0, 0.0902537852525711, 0.0, 0.0520823672413826, 1.0536860227584839, 0.05092201009392738, 0.0, 0.09505179524421692, 0.15050457417964935, 0.0, 0.04610582813620567, 0.01806686818599701, 0.009386475197970867, 0.014475899748504162, 0.0, 0.08525852113962173, 0.06377821415662766, 0.0, 0.007273918483406305, 1.1432180404663086, 0.06207582727074623, 0.03173522278666496, 0.010382498614490032, 0.2002488523721695, 0.15827149152755737, 0.03884025663137436, 0.0, 0.011576884426176548, 0.016200074926018715, 0.08405838161706924, 0.18615354597568512, 0.005644049029797316, 1.1115115880966187, 0.04311821609735489, 0.0, 0.037472400814294815, 0.1764044463634491, 0.0, 0.0, 0.029490765184164047, 0.0, 0.10959701985120773, 0.0, 0.006549849174916744, 0.0, 0.0, 0.2913114130496979, 0.0670965164899826, 0.014888034202158451, 0.10084617137908936, 0.0, 1.0634174346923828, 0.008295265026390553, 0.0, 0.0, 0.0, 0.0, 0.0, 0.026232920587062836, 0.0, 0.0, 0.0, 0.0868072584271431, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.023532984778285027, 0.0, 0.0, 0.0, 0.0, 0.16439449787139893, 0.0, 0.06247846409678459, 0.0, 0.06235906481742859, 0.0, 0.014883964322507381, 0.02545822411775589, 0.0, 0.0, 0.0, 0.0, 0.13479727506637573, 0.010272235609591007, 0.062748484313488, 0.0005867293803021312, 0.03494305908679962, 0.050214920192956924, 0.0, 0.0, 0.07457280158996582, 0.0, 0.0, 1.0061849355697632, 0.0, 0.0, 0.14933298528194427, 0.0, 0.0, 0.11197442561388016, 0.0, 0.17924249172210693, 0.006567663978785276, 0.018015768378973007, 0.0, 0.0], [0.0015795857179909945, 1.174405813217163, 0.044195178896188736, 0.0, 0.0, 1.2314666509628296, 0.0, 0.0, 0.0, 0.02259250171482563, 0.03536178916692734, 0.0, 0.0, 0.0, 0.0, 0.0, 1.11521577835083, 0.0, 0.010300970636308193, 0.0, 0.0010602716356515884, 0.0, 0.0, 0.0029944265261292458, 0.0, 0.04564392939209938, 0.0, 0.0, 0.0578915998339653, 1.0793144702911377, 0.07345420122146606, 0.03109806589782238, 1.1388229131698608, 0.03073142282664776, 0.009030885994434357, 0.0, 0.03941573575139046, 0.0260982196778059, 0.0181184783577919, 0.0, 0.0, 0.0, 0.0, 0.033294349908828735, 0.015242478810250759, 1.0234819650650024, 0.01168869063258171, 0.0, 0.0, 1.1558881998062134, 0.0, 0.016158612444996834, 1.0354083776474, 0.0, 0.07841253280639648, 0.0, 0.0, 0.020020948722958565, 0.0265835952013731, 0.06063880771398544, 0.004207022488117218, 0.0, 0.013500526547431946, 0.0, 0.0, 0.0002673536946531385, 0.0819002017378807, 0.0, 0.04091263562440872, 1.0903840065002441, 0.017444724217057228, 0.03509248048067093, 1.1207880973815918, 0.0, 0.008313106372952461, 0.0, 0.0, 0.011226054280996323, 1.076563835144043, 0.0, 0.023634297773241997, 0.06642655283212662, 0.0, 0.017228983342647552, 0.010578782297670841, 0.0, 0.0, 0.0, 0.014829272404313087, 0.0, 0.0, 0.0014565530000254512, 0.0, 0.00570468557998538, 0.00855958741158247, 0.0, 0.0865049734711647, 0.03933757171034813, 0.07548148185014725, 0.0, 0.0, 0.9740642309188843, 0.02326164022088051, 0.007367664482444525, 0.002753453329205513, 0.0, 1.1286144256591797, 0.0, 0.0, 0.005347662605345249, 0.0, 0.0, 0.0, 0.0, 0.018102196976542473, 0.0, 0.0, 0.0, 0.0, 0.0, 0.9532694816589355, 0.0, 0.007200479507446289, 0.0, 0.0, 0.06656354665756226, 0.01744905114173889, 0.0, 0.0048163398168981075, 0.0, 0.0, 0.059001754969358444, 0.0, 0.0, 0.0, 0.009491300210356712, 0.0, 0.0, 0.07425833493471146, 0.0106325913220644, 0.02707529440522194, 0.0, 0.0, 0.0, 0.8791259527206421, 0.056511878967285156, 0.01845865696668625, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.911929726600647, 0.0, 0.0, 0.0, 1.0646517276763916, 0.005077811423689127, 0.0, 0.09596723318099976, 0.01547722052782774, 0.025153381749987602, 0.0, 0.0, 0.0, 0.0, 0.0, 0.008062710054218769, 0.0, 0.0, 0.0, 1.113268494606018, 0.030914030969142914, 0.031333476305007935, 0.0, 0.0, 0.039724987000226974, 0.0, 0.028173433616757393, 0.2469170093536377, 0.002200845628976822, 0.0, 0.004704452119767666], [0.010474209673702717, 0.0, 0.009657983668148518, 0.008651750162243843, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.00947354082018137, 0.26989325881004333, 0.0, 0.0, 0.0, 0.0, 0.0013430222170427442, 0.0, 0.0025899342726916075, 0.019468508660793304, 0.0, 0.0, 0.0, 0.0, 0.2890462577342987, 0.003452928736805916, 0.0, 0.0, 0.013046922162175179, 0.003970078192651272, 0.0, 0.0, 0.0, 6.713056063745171e-05, 0.29848766326904297, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.001201667939312756, 0.0, 0.005191085394471884, 0.01926206797361374, 0.01038806326687336, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.008567427285015583, 0.0, 0.013136113062500954, 0.0, 0.0, 0.010870669037103653, 0.278987854719162, 0.006074121687561274, 0.014820613898336887, 0.010741213336586952, 0.0, 0.013830428011715412, 0.0, 0.0, 0.04692867398262024, 0.0, 0.0, 0.0, 0.002645155880600214, 0.036063600331544876, 0.0036750398576259613, 0.0, 0.28274214267730713, 0.0, 0.0033112061209976673, 0.0036184557247906923, 0.0014399171341210604, 0.0, 0.029711419716477394, 0.02031416818499565, 0.0, 0.01868738792836666, 0.02237357385456562, 0.0045881615951657295, 0.015869276598095894, 0.009801381267607212, 0.003719005733728409, 0.28430330753326416, 0.0, 0.00623139226809144, 0.001559767872095108, 0.0, 0.012234210036695004, 0.00245294370688498, 0.050024550408124924, 0.00909188948571682, 0.0, 0.00667387992143631, 0.04401812329888344, 0.03232662007212639, 0.01513739489018917, 0.009162687696516514, 0.01612432673573494, 0.018166005611419678, 0.009905287995934486, 0.013622216880321503, 0.30137139558792114, 0.05157560855150223, 0.0, 0.0, 0.0, 0.002953319577500224, 0.02837350033223629, 0.3259996175765991, 0.01094057783484459, 0.02176099456846714, 0.017002325505018234, 0.03963560238480568, 0.02404126152396202, 0.015092635527253151, 0.0292055681347847, 0.013868454843759537, 0.008743283338844776, 0.0, 0.018115781247615814, 0.2768932580947876, 0.0, 0.0026324603240936995, 0.0, 0.0, 0.0, 0.0, 0.0, 0.05592784658074379, 0.0, 0.0, 0.0008341277134604752, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.26275634765625, 0.0, 0.0, 0.012437338009476662, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.005729530472308397, 0.2595907151699066, 0.025576135143637657, 0.008062804117798805, 0.008163966238498688, 0.0016768935602158308, 2.236148066003807e-05, 0.011770937591791153, 0.2738363742828369, 0.004328266251832247, 0.008434438146650791, 0.0, 0.0, 0.0, 0.0, 0.0, 0.08003097027540207, 0.0, 0.0, 0.05213144049048424, 0.0, 0.01891973428428173, 0.0, 0.0018262852681800723, 0.012601079419255257, 0.0], [0.018666591495275497, 0.0, 0.24185585975646973, 0.2669545114040375, 0.0, 0.0, 0.0, 0.28152671456336975, 0.09135286509990692, 0.01974961720407009, 0.040513381361961365, 0.0, 0.0, 0.40568631887435913, 0.12844331562519073, 0.4217856526374817, 0.0, 0.09255530685186386, 0.06193609908223152, 0.0, 0.0, 0.3435761630535126, 0.24334435164928436, 0.16288791596889496, 0.0, 0.16656452417373657, 0.2417735457420349, 0.33722367882728577, 0.011545831337571144, 0.0, 0.331548273563385, 0.0, 0.0, 0.0, 0.0, 0.4951586425304413, 0.05548178777098656, 0.0, 0.33684006333351135, 0.01021553110331297, 0.5109493136405945, 0.36820322275161743, 0.3010788857936859, 0.2692059576511383, 0.0, 0.0, 0.4820443093776703, 0.5507116913795471, 0.0, 0.0072735450230538845, 0.15881992876529694, 0.3149305284023285, 0.0, 0.2915700376033783, 0.02058441750705242, 0.0, 0.5054493546485901, 0.42630088329315186, 0.0, 0.0, 0.3401501178741455, 0.1754022240638733, 0.3294571042060852, 0.4933270812034607, 0.0021601419430226088, 0.2863653600215912, 0.08967053145170212, 0.598906934261322, 0.08952002972364426, 0.015894554555416107, 0.37921813130378723, 0.0, 0.015597483143210411, 0.4873749613761902, 0.06124061346054077, 0.6039709448814392, 0.4750179350376129, 0.0, 0.0, 0.0, 0.050790511071681976, 0.03959087282419205, 0.3123549818992615, 0.17850646376609802, 0.18228396773338318, 0.16299697756767273, 0.0, 0.5288805961608887, 0.5100994110107422, 0.030656680464744568, 0.0399291068315506, 0.3589187562465668, 0.01989842765033245, 0.49861305952072144, 0.27613309025764465, 0.43056821823120117, 0.05801652371883392, 0.1359645575284958, 0.583267331123352, 0.5949797034263611, 0.0, 0.006969719659537077, 0.28489950299263, 0.2591518759727478, 0.5268732309341431, 0.0, 0.018778694793581963, 0.00034998010960407555, 0.02578863501548767, 0.0054579139687120914, 0.06781836599111557, 0.49342790246009827, 0.4855096638202667, 0.5263970494270325, 0.3233739733695984, 0.0, 0.29271888732910156, 0.0, 0.3749852478504181, 0.022739887237548828, 0.0, 0.29648593068122864, 0.2114660143852234, 0.41890087723731995, 0.027704648673534393, 0.4761134386062622, 0.23059771955013275, 0.0, 0.0, 0.23316368460655212, 0.04365953058004379, 0.03664162755012512, 0.07914740592241287, 0.1832132637500763, 0.4390445947647095, 0.0, 0.460316002368927, 0.0, 0.0, 0.03028634376823902, 0.020351586863398552, 0.0, 0.159490704536438, 0.03195279836654663, 0.0, 0.18238745629787445, 0.07764363288879395, 0.25273147225379944, 0.0, 0.33253222703933716, 0.4003782868385315, 0.009264621883630753, 0.3580186069011688, 0.0, 0.42773860692977905, 0.010485988110303879, 0.0389852449297905, 0.14236772060394287, 0.1584193855524063, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.45246005058288574, 0.4101172387599945, 0.2973051071166992, 0.2597948908805847, 0.0, 0.2513371407985687, 0.0, 0.0, 0.481023371219635, 0.0, 0.0, 0.4532058835029602, 0.023099320009350777, 0.5005654096603394, 0.5315288305282593, 0.04299379885196686, 0.3449452221393585, 0.011666984297335148, 0.1063285619020462, 0.010146679356694221, 0.0, 0.012528589926660061], [0.0, 0.06579706072807312, 0.05629194527864456, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0368984080851078, 0.021436123177409172, 0.04619613289833069, 0.01843602955341339, 1.2840003967285156, 0.01745760813355446, 0.0, 0.0, 0.03742166981101036, 0.05690506100654602, 0.03312418609857559, 0.008406737819314003, 1.1419569253921509, 0.05103952810168266, 0.020000150427222252, 0.010630184784531593, 0.06747506558895111, 0.05310126766562462, 0.030368834733963013, 0.0, 0.03574518486857414, 0.020377835258841515, 0.0, 0.0, 0.0, 0.02293046936392784, 0.0, 0.029076019302010536, 0.0, 0.0025820196606218815, 0.019443141296505928, 0.03425830602645874, 0.025633210316300392, 0.04470822215080261, 0.005485401023179293, 0.0, 0.0, 0.043824583292007446, 0.025548608973622322, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0768611803650856, 0.019683530554175377, 0.0, 0.0, 0.0, 0.0, 0.0, 0.21708443760871887, 0.0, 0.0, 0.0, 0.0714755654335022, 0.013323232531547546, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.01621379889547825, 0.004528959281742573, 0.021054567769169807, 0.0, 0.0, 0.0, 0.0, 0.21964631974697113, 0.0, 0.012251609936356544, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.006315846461802721, 0.00031126130488701165, 0.0, 0.03539551421999931, 0.04503334313631058, 0.0, 0.014581127092242241, 0.09813545644283295, 0.012449381873011589, 0.0, 0.0, 0.02378152124583721, 0.013133497908711433, 0.0, 0.005562596954405308, 0.0, 0.0, 0.0, 0.0, 0.19619499146938324, 1.145524263381958, 0.03503980487585068, 0.01601318269968033, 0.0, 0.014017660170793533, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.006804184056818485, 0.0, 0.005330364219844341, 0.04170992970466614, 0.0, 0.0, 0.0, 0.0, 0.0, 0.03321131318807602, 0.0, 0.01596863567829132, 0.0, 0.041721537709236145, 0.0, 0.0, 0.22317393124103546, 0.012213636189699173, 0.19298261404037476, 0.003814988536760211, 0.017824867740273476, 0.0, 0.0483182817697525, 0.030934346839785576, 0.0, 0.0, 0.0, 0.0, 0.0, 0.018305588513612747, 0.0, 0.010487272404134274, 0.0, 0.015929341316223145, 0.046277064830064774, 0.0, 0.2636224031448364, 0.0, 0.0, 0.0, 0.0, 0.06923475116491318, 1.1109652519226074, 0.04792020097374916, 0.044326301664114, 0.006697559263557196, 0.006778343115001917, 0.0, 0.04512589052319527, 0.07474777847528458, 0.05289481207728386, 0.010178249329328537, 0.0, 0.007518233731389046, 0.0, 0.04570725932717323, 0.0, 0.0, 0.015963461250066757, 0.0, 0.012388207949697971, 0.10648823529481888, 0.014867722988128662, 0.0, 0.0], [0.003987184260040522, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.03093843348324299, 0.011520410887897015, 0.034350719302892685, 0.0, 0.0, 0.0, 0.0067471168003976345, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.041903894394636154, 0.051163896918296814, 0.0, 0.12668922543525696, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.13429972529411316, 1.1637340784072876, 0.06209102272987366, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.03783262148499489, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.06610824912786484, 0.0, 0.31488776206970215, 1.263821005821228, 0.0, 0.008645858615636826, 0.14525461196899414, 1.2063480615615845, 0.07342023402452469, 0.0441829077899456, 0.04294580966234207, 0.0, 0.0, 0.0, 0.3240465521812439, 0.0, 0.2597699761390686, 0.1913454830646515, 0.07310119271278381, 0.10144643485546112, 0.1488122195005417, 0.026729771867394447, 0.016696419566869736, 0.0, 0.007626454811543226, 0.030240466818213463, 0.031728923320770264, 0.04971130192279816, 0.1938900202512741, 0.3342929184436798, 0.007758402731269598, 0.18849056959152222, 0.17365582287311554, 0.19269634783267975, 1.3748937845230103, 0.14684824645519257, 0.12421154230833054, 0.19760723412036896, 1.3565995693206787, 0.16686902940273285, 0.060440126806497574, 0.037050191313028336, 0.08490997552871704, 0.021358976140618324, 0.3255869746208191, 0.344207763671875, 0.046166036278009415, 0.007787387352436781, 0.11247163265943527, 0.21531230211257935, 0.04497162252664566, 0.10205614566802979, 0.03721243888139725, 0.0, 0.0, 0.07974156737327576, 0.017429346218705177, 0.279754638671875, 0.15931348502635956, 0.10489723831415176, 0.10468650609254837, 0.033926501870155334, 0.018712276592850685, 0.1572290062904358, 0.30452993512153625, 1.2800384759902954, 0.06732308864593506, 0.237065389752388, 0.1376214623451233, 0.08602365106344223, 0.087501659989357, 0.04663534089922905, 0.05273301154375076, 0.02951625920832157, 0.09359671175479889, 0.048614468425512314, 1.1557583808898926, 0.005362890660762787, 0.0, 0.1752060353755951, 0.03673049435019493, 0.03285469114780426, 0.0, 0.0, 0.0, 0.0, 0.0, 0.040855202823877335, 0.17706289887428284, 0.8655204176902771, 0.00022800422448199242, 0.01116153784096241, 0.0, 0.0, 0.09384765475988388, 0.0, 0.0, 0.0, 0.0, 0.10158737003803253, 0.0, 0.03622184321284294, 0.0, 0.04932963848114014, 0.0, 0.006510654930025339, 0.0, 0.0, 0.0, 0.026733115315437317, 0.0, 0.025175806134939194, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.12957213819026947, 1.0696592330932617, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.2133980244398117, 0.0, 0.09558078646659851, 0.0, 0.0031113384757190943, 0.0, 0.0], [0.055203113704919815, 0.07683824002742767, 0.2786591351032257, 0.36664697527885437, 0.08027883619070053, 0.09563375264406204, 0.8541499972343445, 0.25490301847457886, 0.21801890432834625, 0.16896870732307434, 0.23475782573223114, 0.05724503472447395, 0.0829034224152565, 0.25763624906539917, 0.5423086285591125, 0.1991012543439865, 0.07441312074661255, 0.5819413661956787, 0.29669857025146484, 0.0, 0.0008303975337184966, 0.0650591105222702, 0.07762390375137329, 0.3487263023853302, 0.04047142714262009, 0.01793822832405567, 0.012234777212142944, 0.044456638395786285, 0.0966465175151825, 0.03478889912366867, 0.265403151512146, 0.029351554811000824, 0.028186749666929245, 0.7431153059005737, 0.05488080531358719, 0.07859129458665848, 0.045071303844451904, 0.025372253730893135, 0.07394567131996155, 0.018226221203804016, 0.10949939489364624, 0.013725937344133854, 0.008827653713524342, 0.022302046418190002, 0.0, 0.0, 0.047686416655778885, 0.03276602551341057, 0.007642371580004692, 0.016546977683901787, 0.6289435625076294, 0.15164156258106232, 0.01522835809737444, 0.4082765579223633, 0.013775416649878025, 0.02336815372109413, 0.085680291056633, 0.12598808109760284, 0.0107857845723629, 0.02935534156858921, 0.0, 0.0, 0.0, 0.0, 0.0, 0.18959209322929382, 0.0, 0.0006695505580864847, 0.0, 0.0, 0.1461666375398636, 0.0, 0.0, 0.11801168322563171, 0.0, 0.027057448402047157, 0.06516557931900024, 0.0, 0.0, 0.7130569815635681, 0.0, 0.020817268639802933, 0.3469288647174835, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.020283062011003494, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.01770493946969509, 0.0, 0.0, 0.062209852039813995, 0.007658706977963448, 0.0375639982521534, 0.0, 0.0, 0.7182305455207825, 0.0, 0.0, 0.0, 0.0, 0.0, 0.044664908200502396, 0.009426337666809559, 0.0, 0.003197300946339965, 0.0, 0.055897168815135956, 0.0, 0.0, 0.0, 0.0, 0.05112582445144653, 0.0, 0.0, 0.16910973191261292, 0.0, 0.0, 0.15044480562210083, 0.0, 0.051653239876031876, 0.04687877744436264, 0.024038460105657578, 0.055236171931028366, 0.0, 0.03701231628656387, 0.0, 0.0007145872223190963, 0.0011643330799415708, 0.051558129489421844, 0.017334794625639915, 0.2005421221256256, 0.0, 0.0, 0.2133949249982834, 0.04812512919306755, 0.08522523939609528, 0.0, 0.0, 0.06257753819227219, 0.0, 0.08628155291080475, 0.0, 0.06044011190533638, 0.019865108653903008, 0.0, 0.5101449489593506, 0.13587288558483124, 0.0, 0.0, 0.7183535099029541, 0.0, 0.0, 0.0, 0.030767066404223442, 0.0, 0.0, 0.013162647373974323, 0.0, 0.026312049478292465, 0.0, 0.0, 0.0, 0.0, 0.0, 0.013285612687468529, 0.0, 0.0, 0.04242559149861336, 0.0, 0.0, 0.0, 0.05699793994426727, 0.5691341161727905, 0.0, 0.009624016471207142]]]},\n            {\n            'hltrCFG': {'tokenization_config': {\"token_prefix\": \"\", \"partial_token_prefix\": \"##\"}\n                }\n            })\n         }, function (err) {\n            console.log(err);\n        })"},"metadata":{}}]},{"cell_type":"markdown","source":"## Our Finetuned Model A Activations Factorization:","metadata":{}},{"cell_type":"code","source":"from transformers import AutoTokenizer, AutoModelForSequenceClassification\nmodel_A = AutoModelForSequenceClassification.from_pretrained(\"/kaggle/input/model-new-up/MODELs/MODEL_A\",output_attentions=True)\ntokenizer_A = AutoTokenizer.from_pretrained(\"/kaggle/input/model-new-up/MODELs/MODEL_A\")\nmodel_A","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-31T22:26:02.604354Z","iopub.execute_input":"2023-05-31T22:26:02.605390Z","iopub.status.idle":"2023-05-31T22:26:06.481211Z","shell.execute_reply.started":"2023-05-31T22:26:02.605351Z","shell.execute_reply":"2023-05-31T22:26:06.480287Z"},"trusted":true},"execution_count":12,"outputs":[{"execution_count":12,"output_type":"execute_result","data":{"text/plain":"DistilBertForSequenceClassification(\n  (distilbert): DistilBertModel(\n    (embeddings): Embeddings(\n      (word_embeddings): Embedding(35676, 768, padding_idx=0)\n      (position_embeddings): Embedding(512, 768)\n      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n      (dropout): Dropout(p=0.1, inplace=False)\n    )\n    (transformer): Transformer(\n      (layer): ModuleList(\n        (0-5): 6 x TransformerBlock(\n          (attention): MultiHeadSelfAttention(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n          )\n          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          (ffn): FFN(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n            (activation): GELUActivation()\n          )\n          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n        )\n      )\n    )\n  )\n  (pre_classifier): Linear(in_features=768, out_features=768, bias=True)\n  (classifier): Linear(in_features=768, out_features=1, bias=True)\n  (dropout): Dropout(p=0.2, inplace=False)\n)"},"metadata":{}}]},{"cell_type":"code","source":"model_config_A = {\n    'embedding': \"embeddings.word_embeddings\",\n    'type': 'mlm',\n    'activations':['ffn\\.lin2'],\n    'token_prefix': '',\n    'partial_token_prefix': '##'\n}\nA_model_id = \"/kaggle/input/model-new-up/MODELs/MODEL_A\"","metadata":{"execution":{"iopub.status.busy":"2023-05-31T18:45:20.324196Z","iopub.execute_input":"2023-05-31T18:45:20.324655Z","iopub.status.idle":"2023-05-31T18:45:20.331241Z","shell.execute_reply.started":"2023-05-31T18:45:20.324590Z","shell.execute_reply":"2023-05-31T18:45:20.329968Z"},"trusted":true},"execution_count":64,"outputs":[]},{"cell_type":"code","source":"ecco_model = ecco.from_pretrained(A_model_id, \n                                  activations=True,\n                                  model_config=model_config_A)","metadata":{"execution":{"iopub.status.busy":"2023-05-31T18:45:20.335004Z","iopub.execute_input":"2023-05-31T18:45:20.335399Z","iopub.status.idle":"2023-05-31T18:45:21.745003Z","shell.execute_reply.started":"2023-05-31T18:45:20.335367Z","shell.execute_reply":"2023-05-31T18:45:21.743703Z"},"_kg_hide-output":true,"trusted":true},"execution_count":65,"outputs":[{"name":"stderr","text":"Some weights of the model checkpoint at /kaggle/input/model-new-up/MODELs/MODEL_A were not used when initializing DistilBertModel: ['classifier.bias', 'pre_classifier.bias', 'pre_classifier.weight', 'classifier.weight']\n- This IS expected if you are initializing DistilBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n- This IS NOT expected if you are initializing DistilBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n","output_type":"stream"}]},{"cell_type":"markdown","source":"### Factorizing Activations of a All Layer Layers","metadata":{}},{"cell_type":"code","source":"# Factorize the activations of all layers\ninputs = ecco_model.tokenizer([text], return_tensors=\"pt\")\noutput = ecco_model(inputs)\nnmf_1 = output.run_nmf(n_components=8)                           #non-negative matrix factorization on 8 components\nnmf_1.explore()","metadata":{"execution":{"iopub.status.busy":"2023-05-31T18:45:21.746686Z","iopub.execute_input":"2023-05-31T18:45:21.747056Z","iopub.status.idle":"2023-05-31T18:45:27.204988Z","shell.execute_reply.started":"2023-05-31T18:45:21.747025Z","shell.execute_reply":"2023-05-31T18:45:27.203892Z"},"trusted":true},"execution_count":66,"outputs":[{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"<html lang=\"en\">\n<script src=\"https://requirejs.org/docs/release/2.3.6/minified/require.js\"></script>\n<script>\n    var ecco_url = 'https://storage.googleapis.com/ml-intro/ecco/'\n    //var ecco_url = 'http://localhost:8000/'\n\n    if (window.ecco === undefined) window.ecco = {}\n\n    // Setup the paths of the script we'll be using\n    requirejs.config({\n        urlArgs: \"bust=\" + (new Date()).getTime(),\n        nodeRequire: require,\n        paths: {\n            d3: \"https://d3js.org/d3.v6.min\", // This is only for use in setup.html and basic.html\n            \"d3-array\": \"https://d3js.org/d3-array.v2.min\",\n            jquery: \"https://code.jquery.com/jquery-3.5.1.min\",\n            ecco: ecco_url + 'js/0.0.6/ecco-bundle.min',\n            xregexp: 'https://cdnjs.cloudflare.com/ajax/libs/xregexp/3.2.0/xregexp-all.min'\n        }\n    });\n\n    // Add the css file\n    //requirejs(['d3'],\n    //    function (d3) {\n    //        d3.select('#css').attr('href', ecco_url + 'html/styles.css')\n    //    })\n\n    console.log('Ecco initialize!!')\n\n    // returns a 'basic' object. basic.init() selects the html div we'll be\n    // rendering the html into, adds styles.css to the document.\n    define('basic', ['d3'],\n        function (d3) {\n            return {\n                init: function (viz_id = null) {\n                    if (viz_id == null) {\n                        viz_id = \"viz_\" + Math.round(Math.random() * 10000000)\n                    }\n                    // Select the div rendered below, change its id\n                    const div = d3.select('#basic').attr('id', viz_id),\n                        div_parent = d3.select('#' + viz_id).node().parentNode\n\n                    // Link to CSS file\n                    d3.select(div_parent).insert('link')\n                        .attr('rel', 'stylesheet')\n                        .attr('type', 'text/css')\n                        .attr('href', ecco_url + 'html/0.0.2/styles.css')\n\n                    return viz_id\n                }\n            }\n        }, function (err) {\n            console.log(err);\n        }\n    )\n</script>\n\n<head>\n    <link id='css' rel=\"stylesheet\" type=\"text/css\">\n</head>\n<div id=\"basic\"></div>\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.Javascript object>","application/javascript":"\n         requirejs(['basic', 'ecco'], function(basic, ecco){\n            const viz_id = basic.init()\n            \n            ecco.interactiveTokensAndFactorSparklines(viz_id, {'tokens': [{'token': '[CLS]', 'token_id': 101, 'type': 'input', 'position': 0}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 1}, {'token': 'vast', 'token_id': 6565, 'type': 'input', 'position': 2}, {'token': 'expanse', 'token_id': 22944, 'type': 'input', 'position': 3}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 4}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 5}, {'token': 'ocean', 'token_id': 4153, 'type': 'input', 'position': 6}, {'token': 'st', 'token_id': 2358, 'type': 'input', 'position': 7}, {'token': '##ret', 'token_id': 13465, 'type': 'input', 'position': 8}, {'token': '##c', 'token_id': 2278, 'type': 'input', 'position': 9}, {'token': 'hes', 'token_id': 32979, 'type': 'input', 'position': 10}, {'token': 'out', 'token_id': 2041, 'type': 'input', 'position': 11}, {'token': 'before', 'token_id': 2077, 'type': 'input', 'position': 12}, {'token': 'me', 'token_id': 2033, 'type': 'input', 'position': 13}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 14}, {'token': 'its', 'token_id': 2049, 'type': 'input', 'position': 15}, {'token': 'ca', 'token_id': 6187, 'type': 'input', 'position': 16}, {'token': 'lm', 'token_id': 33415, 'type': 'input', 'position': 17}, {'token': 'waters', 'token_id': 5380, 'type': 'input', 'position': 18}, {'token': 'reflecting', 'token_id': 10842, 'type': 'input', 'position': 19}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 20}, {'token': 'sky', 'token_id': 3712, 'type': 'input', 'position': 21}, {'token': 'above', 'token_id': 2682, 'type': 'input', 'position': 22}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 23}, {'token': 'its', 'token_id': 2049, 'type': 'input', 'position': 24}, {'token': 'tranquil', 'token_id': 33738, 'type': 'input', 'position': 25}, {'token': 'surface', 'token_id': 3302, 'type': 'input', 'position': 26}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 27}, {'token': 'undisturbed', 'token_id': 33021, 'type': 'input', 'position': 28}, {'token': 'by', 'token_id': 2011, 'type': 'input', 'position': 29}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 30}, {'token': 'chaos', 'token_id': 8488, 'type': 'input', 'position': 31}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 32}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 33}, {'token': 'world', 'token_id': 2088, 'type': 'input', 'position': 34}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 35}, {'token': 'inspires', 'token_id': 34870, 'type': 'input', 'position': 36}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 37}, {'token': 'sense', 'token_id': 3168, 'type': 'input', 'position': 38}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 39}, {'token': 'serene', 'token_id': 25388, 'type': 'input', 'position': 40}, {'token': 'tranquil', 'token_id': 33738, 'type': 'input', 'position': 41}, {'token': 'it', 'token_id': 2009, 'type': 'input', 'position': 42}, {'token': '##y', 'token_id': 2100, 'type': 'input', 'position': 43}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 44}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 45}, {'token': 'soft', 'token_id': 3730, 'type': 'input', 'position': 46}, {'token': 'whisper', 'token_id': 7204, 'type': 'input', 'position': 47}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 48}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 49}, {'token': 'waves', 'token_id': 5975, 'type': 'input', 'position': 50}, {'token': 'kissing', 'token_id': 7618, 'type': 'input', 'position': 51}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 52}, {'token': 's', 'token_id': 1055, 'type': 'input', 'position': 53}, {'token': 'hor', 'token_id': 31448, 'type': 'input', 'position': 54}, {'token': 'eli', 'token_id': 12005, 'type': 'input', 'position': 55}, {'token': '##ne', 'token_id': 2638, 'type': 'input', 'position': 56}, {'token': 'is', 'token_id': 2003, 'type': 'input', 'position': 57}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 58}, {'token': 'peaceful', 'token_id': 9379, 'type': 'input', 'position': 59}, {'token': 'symphony', 'token_id': 6189, 'type': 'input', 'position': 60}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 61}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 62}, {'token': 'mel', 'token_id': 11463, 'type': 'input', 'position': 63}, {'token': 'odious', 'token_id': 31124, 'type': 'input', 'position': 64}, {'token': 'song', 'token_id': 2299, 'type': 'input', 'position': 65}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 66}, {'token': 'nature', 'token_id': 3267, 'type': 'input', 'position': 67}, {'token': 'that', 'token_id': 2008, 'type': 'input', 'position': 68}, {'token': 'soo', 'token_id': 17111, 'type': 'input', 'position': 69}, {'token': '##t', 'token_id': 2102, 'type': 'input', 'position': 70}, {'token': 'hes', 'token_id': 32979, 'type': 'input', 'position': 71}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 72}, {'token': 'soul', 'token_id': 3969, 'type': 'input', 'position': 73}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 74}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 75}, {'token': 'gentle', 'token_id': 7132, 'type': 'input', 'position': 76}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 77}, {'token': 'rhythmic', 'token_id': 14797, 'type': 'input', 'position': 78}, {'token': 'motion', 'token_id': 4367, 'type': 'input', 'position': 79}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 80}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 81}, {'token': 'ocean', 'token_id': 4153, 'type': 'input', 'position': 82}, {'token': '’', 'token_id': 1521, 'type': 'input', 'position': 83}, {'token': 's', 'token_id': 1055, 'type': 'input', 'position': 84}, {'token': 'currents', 'token_id': 14731, 'type': 'input', 'position': 85}, {'token': 'em', 'token_id': 7861, 'type': 'input', 'position': 86}, {'token': '##bo', 'token_id': 5092, 'type': 'input', 'position': 87}, {'token': '##dies', 'token_id': 18389, 'type': 'input', 'position': 88}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 89}, {'token': 'silent', 'token_id': 4333, 'type': 'input', 'position': 90}, {'token': 'strength', 'token_id': 3997, 'type': 'input', 'position': 91}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 92}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 93}, {'token': 'sense', 'token_id': 3168, 'type': 'input', 'position': 94}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 95}, {'token': 'enduring', 'token_id': 16762, 'type': 'input', 'position': 96}, {'token': 'ser', 'token_id': 14262, 'type': 'input', 'position': 97}, {'token': '##enity', 'token_id': 20693, 'type': 'input', 'position': 98}, {'token': 'that', 'token_id': 2008, 'type': 'input', 'position': 99}, {'token': 'is', 'token_id': 2003, 'type': 'input', 'position': 100}, {'token': 'profoundly', 'token_id': 28089, 'type': 'input', 'position': 101}, {'token': 'peaceful', 'token_id': 9379, 'type': 'input', 'position': 102}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 103}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 104}, {'token': 'bound', 'token_id': 5391, 'type': 'input', 'position': 105}, {'token': '##less', 'token_id': 3238, 'type': 'input', 'position': 106}, {'token': 'beauty', 'token_id': 5053, 'type': 'input', 'position': 107}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 108}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 109}, {'token': 'sea', 'token_id': 2712, 'type': 'input', 'position': 110}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 111}, {'token': 'with', 'token_id': 2007, 'type': 'input', 'position': 112}, {'token': 'its', 'token_id': 2049, 'type': 'input', 'position': 113}, {'token': 'capt', 'token_id': 14408, 'type': 'input', 'position': 114}, {'token': '##ivating', 'token_id': 17441, 'type': 'input', 'position': 115}, {'token': 'ca', 'token_id': 6187, 'type': 'input', 'position': 116}, {'token': 'lm', 'token_id': 33415, 'type': 'input', 'position': 117}, {'token': 'ness', 'token_id': 23384, 'type': 'input', 'position': 118}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 119}, {'token': 'offers', 'token_id': 4107, 'type': 'input', 'position': 120}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 121}, {'token': 'sanctuary', 'token_id': 8493, 'type': 'input', 'position': 122}, {'token': 'from', 'token_id': 2013, 'type': 'input', 'position': 123}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 124}, {'token': 'bust', 'token_id': 13950, 'type': 'input', 'position': 125}, {'token': '##ling', 'token_id': 2989, 'type': 'input', 'position': 126}, {'token': 'noise', 'token_id': 5005, 'type': 'input', 'position': 127}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 128}, {'token': 'everyday', 'token_id': 10126, 'type': 'input', 'position': 129}, {'token': 'life', 'token_id': 2166, 'type': 'input', 'position': 130}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 131}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 132}, {'token': 'haven', 'token_id': 4033, 'type': 'input', 'position': 133}, {'token': 'where', 'token_id': 2073, 'type': 'input', 'position': 134}, {'token': 'one', 'token_id': 2028, 'type': 'input', 'position': 135}, {'token': 'can', 'token_id': 2064, 'type': 'input', 'position': 136}, {'token': 'find', 'token_id': 2424, 'type': 'input', 'position': 137}, {'token': 'peace', 'token_id': 3521, 'type': 'input', 'position': 138}, {'token': 'and', 'token_id': 1998, 'type': 'input', 'position': 139}, {'token': 'solitude', 'token_id': 22560, 'type': 'input', 'position': 140}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 141}, {'token': 'it', 'token_id': 2009, 'type': 'input', 'position': 142}, {'token': '’', 'token_id': 1521, 'type': 'input', 'position': 143}, {'token': 's', 'token_id': 1055, 'type': 'input', 'position': 144}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 145}, {'token': 'place', 'token_id': 2173, 'type': 'input', 'position': 146}, {'token': 'where', 'token_id': 2073, 'type': 'input', 'position': 147}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 148}, {'token': 'mind', 'token_id': 2568, 'type': 'input', 'position': 149}, {'token': 'can', 'token_id': 2064, 'type': 'input', 'position': 150}, {'token': 'rest', 'token_id': 2717, 'type': 'input', 'position': 151}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 152}, {'token': 'allowing', 'token_id': 4352, 'type': 'input', 'position': 153}, {'token': 'thoughts', 'token_id': 4301, 'type': 'input', 'position': 154}, {'token': 'to', 'token_id': 2000, 'type': 'input', 'position': 155}, {'token': 'flow', 'token_id': 4834, 'type': 'input', 'position': 156}, {'token': 'as', 'token_id': 2004, 'type': 'input', 'position': 157}, {'token': 'freely', 'token_id': 10350, 'type': 'input', 'position': 158}, {'token': 'as', 'token_id': 2004, 'type': 'input', 'position': 159}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 160}, {'token': 'waters', 'token_id': 5380, 'type': 'input', 'position': 161}, {'token': 'them', 'token_id': 2068, 'type': 'input', 'position': 162}, {'token': 'selves', 'token_id': 35100, 'type': 'input', 'position': 163}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 164}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 165}, {'token': 'ocean', 'token_id': 4153, 'type': 'input', 'position': 166}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 167}, {'token': 'in', 'token_id': 1999, 'type': 'input', 'position': 168}, {'token': 'its', 'token_id': 2049, 'type': 'input', 'position': 169}, {'token': 'infinite', 'token_id': 10709, 'type': 'input', 'position': 170}, {'token': 'tranquil', 'token_id': 33738, 'type': 'input', 'position': 171}, {'token': 'it', 'token_id': 2009, 'type': 'input', 'position': 172}, {'token': '##y', 'token_id': 2100, 'type': 'input', 'position': 173}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 174}, {'token': 'serves', 'token_id': 4240, 'type': 'input', 'position': 175}, {'token': 'as', 'token_id': 2004, 'type': 'input', 'position': 176}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 177}, {'token': 'reminder', 'token_id': 14764, 'type': 'input', 'position': 178}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 179}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 180}, {'token': 'simple', 'token_id': 3722, 'type': 'input', 'position': 181}, {'token': 'and', 'token_id': 1998, 'type': 'input', 'position': 182}, {'token': 'quiet', 'token_id': 4251, 'type': 'input', 'position': 183}, {'token': 'beauty', 'token_id': 5053, 'type': 'input', 'position': 184}, {'token': 'that', 'token_id': 2008, 'type': 'input', 'position': 185}, {'token': 'exists', 'token_id': 6526, 'type': 'input', 'position': 186}, {'token': 'in', 'token_id': 1999, 'type': 'input', 'position': 187}, {'token': 'our', 'token_id': 2256, 'type': 'input', 'position': 188}, {'token': 'world', 'token_id': 2088, 'type': 'input', 'position': 189}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 190}, {'token': '[SEP]', 'token_id': 102, 'type': 'input', 'position': 191}], 'factors': [[[0.005210626870393753, 0.02102605253458023, 0.017401376739144325, 0.024254394695162773, 0.5522880554199219, 0.007574473507702351, 0.0, 0.007559733930975199, 0.0, 0.006842043250799179, 0.0, 0.01728399656713009, 0.084248848259449, 0.0, 0.0211234949529171, 0.0, 0.001982509857043624, 0.0, 0.000905489781871438, 0.0469430536031723, 0.034702688455581665, 0.009174571372568607, 0.044542696326971054, 0.03256143257021904, 0.0, 0.0, 0.0, 0.0, 0.0, 0.11987455934286118, 0.0, 0.0, 0.5026326179504395, 0.0, 0.005427069496363401, 0.0, 0.001398076768964529, 0.0, 0.0009040324366651475, 0.3166629374027252, 0.0, 0.0, 0.011407840065658092, 0.019282542169094086, 0.0, 0.0025486527010798454, 0.0, 0.016049474477767944, 0.5528629422187805, 0.032163508236408234, 0.0, 0.040509603917598724, 0.003564037149772048, 0.00931351725012064, 0.0, 0.0, 0.010089965537190437, 0.04002721235156059, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.31273511052131653, 0.0, 0.03783031553030014, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.022693868726491928, 0.0, 0.05903968960046768, 0.5385347604751587, 0.031546663492918015, 0.002518771216273308, 0.0384710393846035, 0.07231990993022919, 0.0, 0.01261406484991312, 0.007782336790114641, 0.012152633629739285, 0.003152397694066167, 0.0, 0.007848994806408882, 0.005030571948736906, 0.0, 0.01892402581870556, 0.3214494585990906, 0.0028648937586694956, 0.01285571325570345, 0.0, 0.08972475677728653, 0.059951480478048325, 0.0, 0.0, 0.022838084027171135, 0.031108619645237923, 0.021305767819285393, 0.03366684913635254, 0.05779644846916199, 0.5637385249137878, 0.04802769050002098, 0.01880965754389763, 0.047612760215997696, 0.11649128049612045, 0.0407366044819355, 0.007887790910899639, 0.008824052289128304, 0.0231298990547657, 0.02510889619588852, 0.022685764357447624, 0.0031482998747378588, 0.01241386216133833, 0.0, 0.0, 0.1812925934791565, 0.038972217589616776, 0.0, 0.03997021168470383, 0.03436184674501419, 0.4717845618724823, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0053344652988016605, 0.0, 0.0, 0.0, 0.0, 0.03331363573670387, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.006356824655085802, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.027881557121872902, 0.0, 0.011517059989273548, 0.0, 0.059739451855421066, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.012115512043237686, 0.13627539575099945, 0.019817329943180084, 0.0, 0.008415686897933483, 0.01978622004389763, 0.028916439041495323, 0.0, 0.0, 0.017203189432621002, 0.0, 0.0, 0.30925917625427246, 0.0, 0.0, 0.026409950107336044, 0.0, 0.002724819118157029, 0.06644579023122787, 0.0, 0.1556142270565033, 0.0, 0.0, 0.0, 0.0], [0.02452952228486538, 0.0, 0.0, 0.001210210146382451, 0.0, 0.0, 0.0, 0.014840484596788883, 0.020807359367609024, 0.004819556605070829, 0.0, 0.0, 0.004014361649751663, 0.01352611556649208, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.004396762233227491, 0.0, 0.0005540359416045249, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0004320780572015792, 0.0, 0.0, 0.0, 0.0, 0.001630777376703918, 0.003793112700805068, 0.0, 0.0, 0.0, 0.002690043533220887, 0.0364258848130703, 0.0, 0.00041600633994676173, 0.0, 0.0, 0.0, 0.0, 0.0021041168365627527, 0.01691831275820732, 0.0, 0.007867694832384586, 0.009008154273033142, 0.004021582193672657, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.004798545502126217, 0.0, 0.0, 0.0, 0.0, 0.004247212782502174, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.003266087966039777, 0.0, 0.00747556472197175, 0.00196140818297863, 0.0, 0.008329089730978012, 0.014501448720693588, 0.0, 0.0, 0.0021267589181661606, 0.002852102741599083, 0.0, 0.004472215194255114, 0.007007571402937174, 0.008655768819153309, 0.003199188504368067, 0.010623137466609478, 0.0, 0.008377566933631897, 0.0, 0.0020532363560050726, 0.0, 0.03971777856349945, 0.0, 0.007785346824675798, 0.014785103499889374, 0.005298876669257879, 0.014855342917144299, 0.017285482957959175, 0.005131491459906101, 0.0, 0.0, 0.005450449883937836, 0.0036806263960897923, 0.0, 0.008799202740192413, 0.002353640040382743, 0.009655648842453957, 0.013764830306172371, 0.004188795108348131, 0.0, 0.002429315820336342, 0.0020124835427850485, 0.0037706370931118727, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.007709421683102846, 0.0, 0.0011810456635430455, 0.0, 0.0, 0.0, 0.0, 0.016428852453827858, 0.007536102551966906, 0.007946846075356007, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0021796284709125757, 0.0, 0.0019498391775414348, 0.0, 0.0, 0.008590116165578365, 0.0, 0.0, 0.0031228801235556602, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.002319274703040719, 0.0, 0.0, 0.0, 0.000805719755589962, 0.010877218097448349, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.014917646534740925, 0.00960361398756504, 0.0, 2.7368204593658447], [0.0, 0.0, 0.22751596570014954, 0.25602421164512634, 0.058089155703783035, 0.07916012406349182, 0.4620644748210907, 0.14219903945922852, 0.14230027794837952, 0.1370844691991806, 0.1759270280599594, 0.20607303082942963, 0.15142317116260529, 0.18838033080101013, 0.0, 0.06556406617164612, 0.049306489527225494, 0.059397317469120026, 0.3202046751976013, 0.10854896157979965, 0.05641700699925423, 0.3364506959915161, 0.2588798701763153, 0.036907847970724106, 0.11672168225049973, 0.11166740953922272, 0.2909846603870392, 0.04370686039328575, 0.1448027342557907, 0.07134859263896942, 0.011170119978487492, 0.2758261561393738, 0.0867486447095871, 0.11108744144439697, 0.4378055930137634, 0.015085631050169468, 0.11184611916542053, 0.01114564761519432, 0.10846792906522751, 0.0, 0.06376045197248459, 0.08809923380613327, 0.023223169147968292, 0.004138107877224684, 0.0, 0.0, 0.013243701308965683, 0.09315290302038193, 0.0, 0.026386789977550507, 0.38257768750190735, 0.14989855885505676, 0.0514487698674202, 0.22502340376377106, 0.22747117280960083, 0.15296877920627594, 0.09722749888896942, 0.007537189871072769, 0.0, 0.11580515652894974, 0.14240127801895142, 0.0, 0.0, 0.039527371525764465, 0.0569407194852829, 0.03796158730983734, 0.0, 0.23859241604804993, 0.0031732176430523396, 0.0, 0.0, 0.05974050238728523, 0.0, 0.19249245524406433, 0.0, 0.0, 0.07607326656579971, 0.0, 0.05207907408475876, 0.11791889369487762, 0.0, 0.031049778684973717, 0.4202496111392975, 0.031880322843790054, 0.09308230876922607, 0.2995801568031311, 0.0, 0.0, 0.0, 0.0, 0.07130499184131622, 0.15793655812740326, 0.0, 0.0, 0.05795405060052872, 0.0, 0.08035308867692947, 0.0, 0.052013907581567764, 0.0, 0.0, 0.07161852717399597, 0.08425036072731018, 0.0, 0.0, 0.06712812930345535, 0.06504502892494202, 0.18482884764671326, 0.01321297138929367, 0.05530226603150368, 0.424638956785202, 0.0, 0.0, 0.007636052090674639, 0.0, 0.0, 0.0, 0.030331172049045563, 0.13451355695724487, 0.0, 0.010231961496174335, 0.0, 0.14945608377456665, 0.0, 0.0, 0.0, 0.0, 0.13881616294384003, 0.011801162734627724, 0.1970161646604538, 0.23635455965995789, 0.0, 0.0, 0.2008853405714035, 0.03659605234861374, 0.06923823058605194, 0.05746005102992058, 0.06340739130973816, 0.1706741601228714, 0.0, 0.21004503965377808, 0.0, 0.05667519196867943, 0.005863938946276903, 0.009476078674197197, 0.0, 0.12974154949188232, 0.005700361914932728, 0.0, 0.21906302869319916, 0.032305143773555756, 0.10563980042934418, 0.0, 0.051354218274354935, 0.2500823438167572, 0.008501073345541954, 0.1441592425107956, 0.0, 0.13177593052387238, 0.0, 0.0, 0.2618054449558258, 0.06929059326648712, 0.11800356954336166, 0.0, 0.04403630644083023, 0.43746158480644226, 0.0, 0.013154126703739166, 0.05786028504371643, 0.15418028831481934, 0.08921749889850616, 0.04044533520936966, 0.004645132459700108, 0.0, 0.038776472210884094, 0.0, 0.0, 0.07702960073947906, 0.0, 0.0, 0.12978217005729675, 0.0, 0.11297325044870377, 0.21873803436756134, 0.009663470089435577, 0.10424219816923141, 0.003375859698280692, 0.08598487079143524, 0.30497002601623535, 0.0, 0.0], [0.054269060492515564, 1.1081233024597168, 0.11100985109806061, 0.009080689400434494, 0.02304161712527275, 1.3908003568649292, 0.0, 0.06776933372020721, 0.009240962564945221, 0.0, 0.0, 0.0, 0.0, 0.03275156393647194, 0.0, 0.38873329758644104, 0.0001946956617757678, 0.048511385917663574, 0.0, 0.0, 1.2571344375610352, 0.020859431475400925, 0.011074921116232872, 0.0, 0.36036449670791626, 0.0, 0.0, 0.0, 0.0, 0.0, 1.144600749015808, 0.0, 0.0016259938711300492, 1.2638405561447144, 0.061530329287052155, 0.0, 0.0, 0.337152898311615, 0.0, 0.0, 0.0, 0.0, 0.11501698195934296, 0.012307604774832726, 0.0, 1.0428746938705444, 0.09124842286109924, 0.0, 0.024177521467208862, 1.2497299909591675, 0.0, 0.0, 1.0592103004455566, 0.11542883515357971, 0.0, 0.0, 0.015650223940610886, 0.0, 0.2843928933143616, 0.0, 0.0, 0.0, 0.19923308491706848, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.8232413530349731, 0.0, 0.0, 0.8942972421646118, 0.0, 0.0, 0.0, 0.0, 0.0, 1.197803020477295, 0.0, 0.01545693352818489, 0.24803106486797333, 0.0, 0.0002245583018520847, 0.0, 0.0, 0.26062196493148804, 0.024800783023238182, 0.0, 0.0, 0.2411138117313385, 0.0, 0.0, 0.015646876767277718, 0.025992376729846, 0.0, 0.023677323013544083, 0.02453969046473503, 0.04750390723347664, 0.0, 0.013332798145711422, 1.0472224950790405, 0.028652582317590714, 0.09071379154920578, 0.056769270449876785, 0.04302903637290001, 1.248460054397583, 0.039090801030397415, 0.0, 0.022019531577825546, 0.41526103019714355, 0.02804015390574932, 0.024111930280923843, 0.021959705278277397, 0.05405469983816147, 0.015444807708263397, 0.0, 0.0, 0.28522342443466187, 0.0, 0.0, 1.119449257850647, 0.03484037145972252, 0.09303038567304611, 0.0, 0.0, 0.04606731981039047, 0.0017007384449243546, 0.0, 0.29451870918273926, 0.0, 0.0, 0.04968690872192383, 0.0, 0.0, 0.0, 0.06156633421778679, 0.0, 0.0, 0.15582147240638733, 0.0, 0.048925332725048065, 0.22209081053733826, 0.0, 0.0, 0.8525344729423523, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0007430558325722814, 0.0, 0.01669636182487011, 0.0, 0.0, 0.9759725332260132, 0.0, 0.05661610886454582, 0.0, 0.0, 1.1892491579055786, 0.02815001644194126, 0.024520568549633026, 0.07790951430797577, 0.40122097730636597, 0.027150213718414307, 0.0, 0.12430358678102493, 0.035094112157821655, 0.0, 0.0, 0.0, 0.18263429403305054, 0.0, 0.0, 0.97624272108078, 0.0, 0.08676660805940628, 0.0, 0.012280025519430637, 0.04110724478960037, 0.0, 0.01219638530164957, 0.23317913711071014, 0.015105425380170345, 0.004719369113445282, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0005737848696298897, 0.03641505166888237, 0.0, 0.0, 0.01956709660589695, 0.0, 0.007373409811407328, 0.004866975825279951, 0.00042142189340665936, 0.0, 0.2711144983768463, 0.0, 0.0, 0.0, 0.0, 0.0, 0.00774214556440711, 0.0, 0.0008060230757109821, 0.0, 0.0, 0.0, 0.013770992867648602, 0.004871128126978874, 0.0, 0.0, 0.0, 0.0, 0.008496692404150963, 0.0, 0.003866947488859296, 2.021380662918091, 0.014750037342309952, 0.0024753976613283157, 0.0035614841617643833, 0.006770291831344366, 0.0, 0.0, 0.004764018580317497, 0.0, 0.005315244663506746, 0.0014669361989945173, 0.0, 0.0, 0.008778496645390987, 0.0, 0.0, 0.0, 0.01565628871321678, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.02163422852754593, 0.0, 0.0, 0.5965684652328491, 0.0038246777839958668, 0.0, 0.01264633797109127, 0.0, 0.0013320328434929252, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.01558908075094223, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0017596472753211856, 2.4127468350343406e-05, 0.0, 0.0006331975455395877, 0.0, 1.6350146532058716, 0.016775382682681084, 0.0, 0.0, 0.0025043326895684004, 0.0, 0.0, 0.0, 0.003958334214985371, 0.011004649102687836, 0.0, 0.0, 0.0, 0.0, 0.0008084062719717622, 0.0, 0.07956653833389282, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.01942608132958412, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.5425134301185608, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0049046725034713745, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.011271840892732143, 0.0, 0.0, 0.009092534892261028, 0.024413222447037697, 1.3101515769958496, 0.017118439078330994, 0.011635910719633102, 0.013893733732402325, 0.0007770819938741624, 0.0, 0.0, 0.0030015448573976755, 0.0, 0.0, 0.13081863522529602, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.39620545506477356, 0.0], [0.0, 0.0, 0.01889055408537388, 0.0, 0.0, 0.0, 0.0, 0.015950996428728104, 0.027505401521921158, 0.0, 0.0, 0.024736914783716202, 0.07227742671966553, 0.034973181784152985, 0.21241207420825958, 0.0, 0.0, 0.0, 0.0, 0.0233404953032732, 0.008040977641940117, 0.0, 0.03367394208908081, 0.1420501172542572, 0.0, 0.0, 0.0, 0.17953602969646454, 0.0, 0.06155557185411453, 0.0, 0.0, 0.010926960967481136, 0.008058584295213223, 0.0, 0.2276669442653656, 0.0, 0.004287847317755222, 0.0, 0.0, 0.0, 0.0, 0.03451600298285484, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.02823251113295555, 0.0, 0.0004945213440805674, 0.0, 0.026216702535748482, 0.0, 0.050979338586330414, 0.0, 0.0, 0.0, 0.14911076426506042, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.03542036935687065, 0.0, 0.0, 0.0, 0.0, 0.0, 0.05061601102352142, 0.0, 0.0, 0.11279304325580597, 0.0, 0.0, 0.0, 0.0, 0.0, 0.038050223141908646, 0.01662345603108406, 0.0, 0.044520605355501175, 0.06134367734193802, 0.052984610199928284, 0.008206305094063282, 0.030079497024416924, 0.024224426597356796, 0.19243524968624115, 0.035457152873277664, 0.0, 0.0013066724641248584, 0.0356389619410038, 0.023516301065683365, 0.0, 0.136189803481102, 0.12433048337697983, 0.05171017348766327, 0.004731515888124704, 0.0002442844561301172, 0.0, 0.08238335698843002, 0.06690838932991028, 0.0003918740840163082, 0.004544333089143038, 0.0, 0.0, 0.2654680609703064, 0.13946782052516937, 0.015566053800284863, 0.07075541466474533, 0.030839603394269943, 0.020930364727973938, 0.05167338252067566, 0.02162102982401848, 0.3138839304447174, 0.1983131319284439, 0.15774337947368622, 0.10360238701105118, 0.1473936140537262, 0.03330106660723686, 0.11305249482393265, 0.05891503766179085, 0.04360106587409973, 0.07686983793973923, 0.11074917763471603, 0.1236671432852745, 0.3877682089805603, 0.25800248980522156, 0.13484284281730652, 0.3301713764667511, 0.28126049041748047, 0.33131083846092224, 0.24676688015460968, 0.11571889370679855, 0.3327215611934662, 0.06486089527606964, 0.27386775612831116, 0.24854077398777008, 0.20575028657913208, 0.32465454936027527, 0.29320764541625977, 0.21873030066490173, 0.368325799703598, 0.21873131394386292, 0.1507214605808258, 0.3575786352157593, 0.22517792880535126, 0.4461324214935303, 0.29434460401535034, 0.14942029118537903, 0.34922271966934204, 0.17181217670440674, 0.3710334897041321, 0.1804412305355072, 0.3321036398410797, 0.15818467736244202, 0.09605846554040909, 0.23149468004703522, 0.1566212773323059, 0.1017538532614708, 0.07770682126283646, 0.0, 0.3596884310245514, 0.20132799446582794, 0.09713239222764969, 0.054560765624046326, 0.019693119451403618, 0.11056959629058838, 0.06339693814516068, 0.4132566452026367, 0.2861626148223877, 0.3368743360042572, 0.2525739073753357, 0.17020738124847412, 0.2081948220729828, 0.16681000590324402, 0.14576715230941772, 0.32095807790756226, 0.10427750647068024, 0.10292196273803711, 0.33064377307891846, 0.27522802352905273, 0.2972002923488617, 0.1900160312652588, 0.10560153424739838, 0.25944358110427856, 0.0], [0.28927868604660034, 0.2246074676513672, 0.03648531064391136, 0.004446469713002443, 0.038154762238264084, 0.032169684767723083, 0.0, 0.24720655381679535, 0.14395040273666382, 0.31617677211761475, 0.3315489590167999, 0.2953045070171356, 0.24156275391578674, 0.24329626560211182, 0.5714095830917358, 0.494491308927536, 0.0659327432513237, 0.14313924312591553, 0.0, 0.07871007919311523, 0.02722959592938423, 0.00707355746999383, 0.09041102230548859, 0.3008970320224762, 0.4988495111465454, 0.1611933410167694, 0.16405147314071655, 0.5283066034317017, 0.28085047006607056, 0.3823719918727875, 0.276116281747818, 0.0, 0.0, 0.0, 0.04001101478934288, 0.42552661895751953, 0.3926483392715454, 0.7918989658355713, 0.21909955143928528, 0.3778359293937683, 0.0, 0.050845395773649216, 0.25414377450942993, 0.2232494205236435, 0.0, 0.34872111678123474, 0.0, 0.0, 0.07640503346920013, 0.1081572100520134, 0.0, 0.08833982050418854, 0.2399318814277649, 0.27412089705467224, 0.2461770921945572, 0.1310742199420929, 0.26768922805786133, 0.7553736567497253, 0.8699881434440613, 0.0, 0.16779185831546783, 0.7182785868644714, 0.8948146104812622, 0.26581910252571106, 0.43780457973480225, 0.26186853647232056, 0.4928708076477051, 0.25557029247283936, 0.7966669797897339, 0.16088584065437317, 0.4320599138736725, 0.48013025522232056, 0.5163155198097229, 0.17168912291526794, 0.43734052777290344, 0.5202856659889221, 0.0669875293970108, 0.6416887640953064, 0.0, 0.06905020028352737, 0.18382561206817627, 0.15651245415210724, 0.056732531636953354, 0.3205259144306183, 0.3393813967704773, 0.0, 0.2706742286682129, 0.2530384659767151, 0.37450873851776123, 0.8110318779945374, 0.04831705987453461, 0.12299925833940506, 0.6406617760658264, 0.787092387676239, 0.2366880625486374, 0.43692657351493835, 0.057153668254613876, 0.04963603988289833, 0.0, 0.4317833185195923, 0.4447776973247528, 0.05483360216021538, 0.0, 0.0, 0.2361469864845276, 0.0055394540540874004, 0.03933916240930557, 0.0, 0.0, 0.019791264086961746, 0.0, 0.25448185205459595, 0.3160551190376282, 0.4527922570705414, 0.008369674906134605, 0.0, 0.014109368436038494, 0.10607044398784637, 0.08354806900024414, 0.24179120361804962, 0.17618940770626068, 0.4652503728866577, 0.0, 0.1644376963376999, 0.1475117802619934, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.06754237413406372, 0.14270108938217163, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.06088225916028023, 0.0, 0.0, 0.020159142091870308, 0.0525437667965889, 0.08224895596504211, 0.09231557697057724, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.01928340457379818, 0.0, 0.0, 0.011513966135680676, 0.0, 0.013460549525916576, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.09686189889907837, 0.12857291102409363, 0.2661987245082855, 0.0, 0.0, 0.1314309686422348, 0.09536195546388626, 0.01292232796549797, 0.03893963247537613, 0.16977578401565552, 0.260325163602829, 0.0, 0.0, 0.0, 0.0, 0.03475761041045189, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3250713646411896, 0.1583012193441391, 0.08743618428707123, 0.2472361922264099, 0.0, 0.0, 0.0, 0.11383677273988724, 0.05348483473062515, 0.1929265409708023, 0.13489608466625214, 0.03967522084712982, 0.0, 0.05488529056310654, 0.0, 0.14992719888687134, 0.7524881362915039, 0.6218623518943787, 0.3307313024997711, 0.4644179046154022, 0.08733770251274109, 0.11811923235654831, 0.0, 0.0, 0.0, 0.48657527565956116, 0.11940231919288635, 0.0, 0.2684393525123596, 0.0, 0.03375214710831642, 0.32261204719543457, 0.0, 0.0, 0.0, 0.0, 0.18938526511192322, 0.0, 0.5116006135940552, 0.22169825434684753, 0.8877619504928589, 0.7639923095703125, 0.5287696719169617, 0.640480637550354, 0.0, 0.1471995711326599, 0.8298819661140442, 0.7398234009742737, 0.15481947362422943, 0.10963836312294006, 0.1568872630596161, 0.20046041905879974, 0.0, 0.0, 0.03380586951971054, 0.15662378072738647, 0.30117619037628174, 0.010731825605034828, 0.0, 0.7717165350914001, 0.4447678327560425, 0.0, 0.0, 0.4060714542865753, 0.22658513486385345, 0.5592813491821289, 0.0751299262046814, 0.1485244333744049, 0.03961610421538353, 0.8947063684463501, 0.5365686416625977, 0.2191036194562912, 0.15396273136138916, 0.3530471920967102, 0.0, 0.16691192984580994, 0.7089280486106873, 0.1057920977473259, 0.8452433347702026, 0.4977531135082245, 0.06380406022071838, 0.019325071945786476, 0.0, 0.0, 0.04752185568213463, 0.3050130307674408, 0.4623837172985077, 0.3123326599597931, 0.360074520111084, 0.07675326615571976, 0.6431366801261902, 0.3574858605861664, 0.0, 0.0, 0.5215516090393066, 0.13071449100971222, 0.5208015441894531, 0.7090020775794983, 0.8445408344268799, 0.11475612968206406, 0.18792837858200073, 0.49515390396118164, 0.799094557762146, 0.002775644650682807, 0.1308702528476715, 0.37462523579597473, 0.307456910610199, 0.43605276942253113, 0.0, 0.015935612842440605, 0.0, 0.0, 0.0, 0.11808265000581741, 0.6806830763816833, 0.7917909026145935, 0.7888787984848022, 0.5574902892112732, 0.3858228325843811, 0.0, 0.17252370715141296, 0.0, 0.269840806722641, 0.06962008774280548, 0.0826459527015686, 0.6113867163658142, 0.6865007281303406, 0.5106952786445618, 0.013460821472108364, 0.08989033102989197, 0.006319159641861916, 0.0, 0.0, 0.03738937899470329, 0.0, 0.0, 0.0, 0.02899875119328499, 0.2811077833175659, 0.004041092935949564, 0.22303633391857147, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.05388020724058151, 0.0, 0.05582820996642113, 0.00022349947539623827, 0.001863907091319561, 0.12080444395542145, 0.007763402070850134, 0.10160701721906662, 0.0, 0.0, 0.0, 0.0, 0.05287843197584152, 0.0, 0.0, 0.0, 0.0, 0.0, 0.03006579354405403, 0.3554331958293915, 0.6447171568870544, 0.3417806625366211, 0.5334489345550537, 0.0, 0.07130977511405945, 0.0, 0.0, 0.1326986849308014, 0.0, 0.0, 0.1248367577791214, 0.004337038844823837, 0.36634719371795654, 0.18040181696414948, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]]]},\n            {\n            'hltrCFG': {'tokenization_config': {\"token_prefix\": \"\", \"partial_token_prefix\": \"##\"}\n                }\n            })\n         }, function (err) {\n            console.log(err);\n        })"},"metadata":{}}]},{"cell_type":"markdown","source":"### Factorizing Activations of a Single Layer","metadata":{}},{"cell_type":"code","source":"# Factorize the activations of only the first layer\nnmf_2 = output.run_nmf(n_components=8, from_layer=0, to_layer=1)\nnmf_2.explore()","metadata":{"execution":{"iopub.status.busy":"2023-05-31T18:45:27.206457Z","iopub.execute_input":"2023-05-31T18:45:27.207149Z","iopub.status.idle":"2023-05-31T18:45:27.488353Z","shell.execute_reply.started":"2023-05-31T18:45:27.207109Z","shell.execute_reply":"2023-05-31T18:45:27.487313Z"},"trusted":true},"execution_count":67,"outputs":[{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"<html lang=\"en\">\n<script src=\"https://requirejs.org/docs/release/2.3.6/minified/require.js\"></script>\n<script>\n    var ecco_url = 'https://storage.googleapis.com/ml-intro/ecco/'\n    //var ecco_url = 'http://localhost:8000/'\n\n    if (window.ecco === undefined) window.ecco = {}\n\n    // Setup the paths of the script we'll be using\n    requirejs.config({\n        urlArgs: \"bust=\" + (new Date()).getTime(),\n        nodeRequire: require,\n        paths: {\n            d3: \"https://d3js.org/d3.v6.min\", // This is only for use in setup.html and basic.html\n            \"d3-array\": \"https://d3js.org/d3-array.v2.min\",\n            jquery: \"https://code.jquery.com/jquery-3.5.1.min\",\n            ecco: ecco_url + 'js/0.0.6/ecco-bundle.min',\n            xregexp: 'https://cdnjs.cloudflare.com/ajax/libs/xregexp/3.2.0/xregexp-all.min'\n        }\n    });\n\n    // Add the css file\n    //requirejs(['d3'],\n    //    function (d3) {\n    //        d3.select('#css').attr('href', ecco_url + 'html/styles.css')\n    //    })\n\n    console.log('Ecco initialize!!')\n\n    // returns a 'basic' object. basic.init() selects the html div we'll be\n    // rendering the html into, adds styles.css to the document.\n    define('basic', ['d3'],\n        function (d3) {\n            return {\n                init: function (viz_id = null) {\n                    if (viz_id == null) {\n                        viz_id = \"viz_\" + Math.round(Math.random() * 10000000)\n                    }\n                    // Select the div rendered below, change its id\n                    const div = d3.select('#basic').attr('id', viz_id),\n                        div_parent = d3.select('#' + viz_id).node().parentNode\n\n                    // Link to CSS file\n                    d3.select(div_parent).insert('link')\n                        .attr('rel', 'stylesheet')\n                        .attr('type', 'text/css')\n                        .attr('href', ecco_url + 'html/0.0.2/styles.css')\n\n                    return viz_id\n                }\n            }\n        }, function (err) {\n            console.log(err);\n        }\n    )\n</script>\n\n<head>\n    <link id='css' rel=\"stylesheet\" type=\"text/css\">\n</head>\n<div id=\"basic\"></div>\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.Javascript object>","application/javascript":"\n         requirejs(['basic', 'ecco'], function(basic, ecco){\n            const viz_id = basic.init()\n            \n            ecco.interactiveTokensAndFactorSparklines(viz_id, {'tokens': [{'token': '[CLS]', 'token_id': 101, 'type': 'input', 'position': 0}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 1}, {'token': 'vast', 'token_id': 6565, 'type': 'input', 'position': 2}, {'token': 'expanse', 'token_id': 22944, 'type': 'input', 'position': 3}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 4}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 5}, {'token': 'ocean', 'token_id': 4153, 'type': 'input', 'position': 6}, {'token': 'st', 'token_id': 2358, 'type': 'input', 'position': 7}, {'token': '##ret', 'token_id': 13465, 'type': 'input', 'position': 8}, {'token': '##c', 'token_id': 2278, 'type': 'input', 'position': 9}, {'token': 'hes', 'token_id': 32979, 'type': 'input', 'position': 10}, {'token': 'out', 'token_id': 2041, 'type': 'input', 'position': 11}, {'token': 'before', 'token_id': 2077, 'type': 'input', 'position': 12}, {'token': 'me', 'token_id': 2033, 'type': 'input', 'position': 13}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 14}, {'token': 'its', 'token_id': 2049, 'type': 'input', 'position': 15}, {'token': 'ca', 'token_id': 6187, 'type': 'input', 'position': 16}, {'token': 'lm', 'token_id': 33415, 'type': 'input', 'position': 17}, {'token': 'waters', 'token_id': 5380, 'type': 'input', 'position': 18}, {'token': 'reflecting', 'token_id': 10842, 'type': 'input', 'position': 19}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 20}, {'token': 'sky', 'token_id': 3712, 'type': 'input', 'position': 21}, {'token': 'above', 'token_id': 2682, 'type': 'input', 'position': 22}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 23}, {'token': 'its', 'token_id': 2049, 'type': 'input', 'position': 24}, {'token': 'tranquil', 'token_id': 33738, 'type': 'input', 'position': 25}, {'token': 'surface', 'token_id': 3302, 'type': 'input', 'position': 26}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 27}, {'token': 'undisturbed', 'token_id': 33021, 'type': 'input', 'position': 28}, {'token': 'by', 'token_id': 2011, 'type': 'input', 'position': 29}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 30}, {'token': 'chaos', 'token_id': 8488, 'type': 'input', 'position': 31}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 32}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 33}, {'token': 'world', 'token_id': 2088, 'type': 'input', 'position': 34}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 35}, {'token': 'inspires', 'token_id': 34870, 'type': 'input', 'position': 36}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 37}, {'token': 'sense', 'token_id': 3168, 'type': 'input', 'position': 38}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 39}, {'token': 'serene', 'token_id': 25388, 'type': 'input', 'position': 40}, {'token': 'tranquil', 'token_id': 33738, 'type': 'input', 'position': 41}, {'token': 'it', 'token_id': 2009, 'type': 'input', 'position': 42}, {'token': '##y', 'token_id': 2100, 'type': 'input', 'position': 43}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 44}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 45}, {'token': 'soft', 'token_id': 3730, 'type': 'input', 'position': 46}, {'token': 'whisper', 'token_id': 7204, 'type': 'input', 'position': 47}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 48}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 49}, {'token': 'waves', 'token_id': 5975, 'type': 'input', 'position': 50}, {'token': 'kissing', 'token_id': 7618, 'type': 'input', 'position': 51}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 52}, {'token': 's', 'token_id': 1055, 'type': 'input', 'position': 53}, {'token': 'hor', 'token_id': 31448, 'type': 'input', 'position': 54}, {'token': 'eli', 'token_id': 12005, 'type': 'input', 'position': 55}, {'token': '##ne', 'token_id': 2638, 'type': 'input', 'position': 56}, {'token': 'is', 'token_id': 2003, 'type': 'input', 'position': 57}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 58}, {'token': 'peaceful', 'token_id': 9379, 'type': 'input', 'position': 59}, {'token': 'symphony', 'token_id': 6189, 'type': 'input', 'position': 60}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 61}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 62}, {'token': 'mel', 'token_id': 11463, 'type': 'input', 'position': 63}, {'token': 'odious', 'token_id': 31124, 'type': 'input', 'position': 64}, {'token': 'song', 'token_id': 2299, 'type': 'input', 'position': 65}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 66}, {'token': 'nature', 'token_id': 3267, 'type': 'input', 'position': 67}, {'token': 'that', 'token_id': 2008, 'type': 'input', 'position': 68}, {'token': 'soo', 'token_id': 17111, 'type': 'input', 'position': 69}, {'token': '##t', 'token_id': 2102, 'type': 'input', 'position': 70}, {'token': 'hes', 'token_id': 32979, 'type': 'input', 'position': 71}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 72}, {'token': 'soul', 'token_id': 3969, 'type': 'input', 'position': 73}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 74}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 75}, {'token': 'gentle', 'token_id': 7132, 'type': 'input', 'position': 76}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 77}, {'token': 'rhythmic', 'token_id': 14797, 'type': 'input', 'position': 78}, {'token': 'motion', 'token_id': 4367, 'type': 'input', 'position': 79}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 80}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 81}, {'token': 'ocean', 'token_id': 4153, 'type': 'input', 'position': 82}, {'token': '’', 'token_id': 1521, 'type': 'input', 'position': 83}, {'token': 's', 'token_id': 1055, 'type': 'input', 'position': 84}, {'token': 'currents', 'token_id': 14731, 'type': 'input', 'position': 85}, {'token': 'em', 'token_id': 7861, 'type': 'input', 'position': 86}, {'token': '##bo', 'token_id': 5092, 'type': 'input', 'position': 87}, {'token': '##dies', 'token_id': 18389, 'type': 'input', 'position': 88}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 89}, {'token': 'silent', 'token_id': 4333, 'type': 'input', 'position': 90}, {'token': 'strength', 'token_id': 3997, 'type': 'input', 'position': 91}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 92}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 93}, {'token': 'sense', 'token_id': 3168, 'type': 'input', 'position': 94}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 95}, {'token': 'enduring', 'token_id': 16762, 'type': 'input', 'position': 96}, {'token': 'ser', 'token_id': 14262, 'type': 'input', 'position': 97}, {'token': '##enity', 'token_id': 20693, 'type': 'input', 'position': 98}, {'token': 'that', 'token_id': 2008, 'type': 'input', 'position': 99}, {'token': 'is', 'token_id': 2003, 'type': 'input', 'position': 100}, {'token': 'profoundly', 'token_id': 28089, 'type': 'input', 'position': 101}, {'token': 'peaceful', 'token_id': 9379, 'type': 'input', 'position': 102}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 103}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 104}, {'token': 'bound', 'token_id': 5391, 'type': 'input', 'position': 105}, {'token': '##less', 'token_id': 3238, 'type': 'input', 'position': 106}, {'token': 'beauty', 'token_id': 5053, 'type': 'input', 'position': 107}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 108}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 109}, {'token': 'sea', 'token_id': 2712, 'type': 'input', 'position': 110}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 111}, {'token': 'with', 'token_id': 2007, 'type': 'input', 'position': 112}, {'token': 'its', 'token_id': 2049, 'type': 'input', 'position': 113}, {'token': 'capt', 'token_id': 14408, 'type': 'input', 'position': 114}, {'token': '##ivating', 'token_id': 17441, 'type': 'input', 'position': 115}, {'token': 'ca', 'token_id': 6187, 'type': 'input', 'position': 116}, {'token': 'lm', 'token_id': 33415, 'type': 'input', 'position': 117}, {'token': 'ness', 'token_id': 23384, 'type': 'input', 'position': 118}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 119}, {'token': 'offers', 'token_id': 4107, 'type': 'input', 'position': 120}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 121}, {'token': 'sanctuary', 'token_id': 8493, 'type': 'input', 'position': 122}, {'token': 'from', 'token_id': 2013, 'type': 'input', 'position': 123}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 124}, {'token': 'bust', 'token_id': 13950, 'type': 'input', 'position': 125}, {'token': '##ling', 'token_id': 2989, 'type': 'input', 'position': 126}, {'token': 'noise', 'token_id': 5005, 'type': 'input', 'position': 127}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 128}, {'token': 'everyday', 'token_id': 10126, 'type': 'input', 'position': 129}, {'token': 'life', 'token_id': 2166, 'type': 'input', 'position': 130}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 131}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 132}, {'token': 'haven', 'token_id': 4033, 'type': 'input', 'position': 133}, {'token': 'where', 'token_id': 2073, 'type': 'input', 'position': 134}, {'token': 'one', 'token_id': 2028, 'type': 'input', 'position': 135}, {'token': 'can', 'token_id': 2064, 'type': 'input', 'position': 136}, {'token': 'find', 'token_id': 2424, 'type': 'input', 'position': 137}, {'token': 'peace', 'token_id': 3521, 'type': 'input', 'position': 138}, {'token': 'and', 'token_id': 1998, 'type': 'input', 'position': 139}, {'token': 'solitude', 'token_id': 22560, 'type': 'input', 'position': 140}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 141}, {'token': 'it', 'token_id': 2009, 'type': 'input', 'position': 142}, {'token': '’', 'token_id': 1521, 'type': 'input', 'position': 143}, {'token': 's', 'token_id': 1055, 'type': 'input', 'position': 144}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 145}, {'token': 'place', 'token_id': 2173, 'type': 'input', 'position': 146}, {'token': 'where', 'token_id': 2073, 'type': 'input', 'position': 147}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 148}, {'token': 'mind', 'token_id': 2568, 'type': 'input', 'position': 149}, {'token': 'can', 'token_id': 2064, 'type': 'input', 'position': 150}, {'token': 'rest', 'token_id': 2717, 'type': 'input', 'position': 151}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 152}, {'token': 'allowing', 'token_id': 4352, 'type': 'input', 'position': 153}, {'token': 'thoughts', 'token_id': 4301, 'type': 'input', 'position': 154}, {'token': 'to', 'token_id': 2000, 'type': 'input', 'position': 155}, {'token': 'flow', 'token_id': 4834, 'type': 'input', 'position': 156}, {'token': 'as', 'token_id': 2004, 'type': 'input', 'position': 157}, {'token': 'freely', 'token_id': 10350, 'type': 'input', 'position': 158}, {'token': 'as', 'token_id': 2004, 'type': 'input', 'position': 159}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 160}, {'token': 'waters', 'token_id': 5380, 'type': 'input', 'position': 161}, {'token': 'them', 'token_id': 2068, 'type': 'input', 'position': 162}, {'token': 'selves', 'token_id': 35100, 'type': 'input', 'position': 163}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 164}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 165}, {'token': 'ocean', 'token_id': 4153, 'type': 'input', 'position': 166}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 167}, {'token': 'in', 'token_id': 1999, 'type': 'input', 'position': 168}, {'token': 'its', 'token_id': 2049, 'type': 'input', 'position': 169}, {'token': 'infinite', 'token_id': 10709, 'type': 'input', 'position': 170}, {'token': 'tranquil', 'token_id': 33738, 'type': 'input', 'position': 171}, {'token': 'it', 'token_id': 2009, 'type': 'input', 'position': 172}, {'token': '##y', 'token_id': 2100, 'type': 'input', 'position': 173}, {'token': ',', 'token_id': 1010, 'type': 'input', 'position': 174}, {'token': 'serves', 'token_id': 4240, 'type': 'input', 'position': 175}, {'token': 'as', 'token_id': 2004, 'type': 'input', 'position': 176}, {'token': 'a', 'token_id': 1037, 'type': 'input', 'position': 177}, {'token': 'reminder', 'token_id': 14764, 'type': 'input', 'position': 178}, {'token': 'of', 'token_id': 1997, 'type': 'input', 'position': 179}, {'token': 'the', 'token_id': 1996, 'type': 'input', 'position': 180}, {'token': 'simple', 'token_id': 3722, 'type': 'input', 'position': 181}, {'token': 'and', 'token_id': 1998, 'type': 'input', 'position': 182}, {'token': 'quiet', 'token_id': 4251, 'type': 'input', 'position': 183}, {'token': 'beauty', 'token_id': 5053, 'type': 'input', 'position': 184}, {'token': 'that', 'token_id': 2008, 'type': 'input', 'position': 185}, {'token': 'exists', 'token_id': 6526, 'type': 'input', 'position': 186}, {'token': 'in', 'token_id': 1999, 'type': 'input', 'position': 187}, {'token': 'our', 'token_id': 2256, 'type': 'input', 'position': 188}, {'token': 'world', 'token_id': 2088, 'type': 'input', 'position': 189}, {'token': '.', 'token_id': 1012, 'type': 'input', 'position': 190}, {'token': '[SEP]', 'token_id': 102, 'type': 'input', 'position': 191}], 'factors': [[[0.0, 0.004168111365288496, 0.0, 0.006583159789443016, 0.0, 0.0, 0.0, 0.03474302962422371, 0.004025869071483612, 0.02637246623635292, 0.0006468333303928375, 0.025519991293549538, 0.06670429557561874, 0.0329475961625576, 0.47280728816986084, 0.0, 0.011000516824424267, 0.056297678500413895, 0.0, 0.0, 0.020464444532990456, 0.0, 0.027924006804823875, 0.37200844287872314, 0.0, 0.0, 0.0, 0.3628728687763214, 0.0, 0.08713102340698242, 0.0, 0.0, 0.0, 0.0, 0.0, 0.46607035398483276, 0.024082276970148087, 0.0, 0.0, 0.0, 0.0, 0.0, 0.07890061289072037, 0.06910868734121323, 0.41918841004371643, 0.033625636249780655, 0.0, 0.0, 0.013190417550504208, 0.0, 0.0, 0.007938940078020096, 0.0, 0.05443470925092697, 0.015638014301657677, 0.0169057659804821, 0.060421738773584366, 0.1036234125494957, 0.0, 0.0, 0.0, 0.38256925344467163, 0.0, 0.0028774570673704147, 0.0, 0.0, 0.0, 0.0, 0.10524049401283264, 0.0, 0.031426332890987396, 0.0005540369311347604, 0.0, 0.0, 0.26546424627304077, 0.0, 0.0, 0.36293572187423706, 0.0, 0.0, 0.0, 0.0, 0.0, 0.02329850383102894, 0.05081157386302948, 0.0, 0.041157547384500504, 0.017122888937592506, 0.0077836778946220875, 0.0, 0.0, 0.0, 0.4049482047557831, 0.0, 0.0, 0.0, 0.0, 0.01745336689054966, 0.0, 0.13938842713832855, 0.0923302099108696, 0.0, 0.0, 0.38979026675224304, 0.0, 0.0, 0.0, 0.004158953670412302, 0.0, 0.0, 0.0, 0.3837849795818329, 0.11521695554256439, 0.0, 0.0, 0.0, 0.012050633318722248, 0.036516834050416946, 0.006457838229835033, 0.39217421412467957, 0.0366373248398304, 0.0, 0.0, 0.1046442911028862, 0.0, 0.036347951740026474, 0.0414317362010479, 0.017042648047208786, 0.0, 0.003544469131156802, 0.07042316347360611, 0.5416151285171509, 0.06881195306777954, 0.05248958244919777, 0.22028811275959015, 0.13626118004322052, 0.13651828467845917, 0.07017123699188232, 0.0292220301926136, 0.2820957601070404, 0.0, 0.47760796546936035, 0.15810562670230865, 0.08826994895935059, 0.12955941259860992, 0.05724151059985161, 0.06947340071201324, 0.2144165337085724, 0.09857750684022903, 0.07103446125984192, 0.13887184858322144, 0.10036680847406387, 0.5859442353248596, 0.10562165826559067, 0.05606804043054581, 0.21072915196418762, 0.02369767427444458, 0.22032365202903748, 0.04318612441420555, 0.20574404299259186, 0.06334768980741501, 0.02194320224225521, 0.12083038687705994, 0.12916381657123566, 0.4478795826435089, 0.10062167793512344, 0.008504068478941917, 0.5489590167999268, 0.20647834241390228, 0.021262498572468758, 0.0180339552462101, 0.04071299731731415, 0.1230345368385315, 0.10558170080184937, 0.5844407677650452, 0.11825740337371826, 0.20882564783096313, 0.02896902710199356, 0.040941271930933, 0.04546106234192848, 0.03942522406578064, 0.042346321046352386, 0.26859936118125916, 0.03924677520990372, 0.05680505931377411, 0.22862330079078674, 0.09501750767230988, 0.2314334213733673, 0.0586991161108017, 0.06342781335115433, 0.4421192705631256, 0.07837291806936264], [0.0, 0.02094173990190029, 0.051425203680992126, 0.0, 1.1615028381347656, 0.0, 0.0, 0.020073873922228813, 0.0, 0.03156239539384842, 0.0, 0.031457267701625824, 0.0955335944890976, 0.015519773587584496, 0.0, 0.0, 0.024767277762293816, 0.01889643631875515, 0.0, 0.0, 0.0, 0.0, 0.04685387387871742, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.12784627079963684, 0.0, 0.0, 1.0911712646484375, 0.027172371745109558, 0.06976479291915894, 0.0, 0.04775039851665497, 0.0, 0.0, 0.996565580368042, 0.0, 0.017534315586090088, 0.06898346543312073, 0.0450403094291687, 0.0004277656553313136, 0.013054396025836468, 0.0, 0.0, 1.1072999238967896, 0.0, 0.0, 0.021712835878133774, 0.015043104067444801, 0.07795015722513199, 0.05274787172675133, 0.00912486668676138, 0.028806131333112717, 0.1285451203584671, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.015571155585348606, 0.011222021654248238, 0.8470008969306946, 0.028605511412024498, 0.1177418977022171, 0.0, 0.01884828321635723, 0.0, 0.0, 0.006185113452374935, 0.0, 0.0, 0.0, 0.0, 0.0, 0.020810185000300407, 0.9616608619689941, 0.0, 0.0, 0.041646670550107956, 0.07101038843393326, 0.0, 0.00924503244459629, 0.0017877033678814769, 0.0021624222863465548, 0.009926258586347103, 0.0, 0.047706592828035355, 0.0, 0.0, 0.0, 0.9841648936271667, 0.014097956009209156, 0.003875333582982421, 0.0, 0.13411341607570648, 0.12679392099380493, 0.0, 0.0, 0.0, 0.0, 0.054920658469200134, 0.05833999440073967, 0.0, 1.003307819366455, 0.0, 0.0, 0.0, 0.09760072827339172, 0.0, 0.0, 0.0, 0.020991986617445946, 0.019602682441473007, 0.0, 0.0, 0.005354039836674929, 0.0, 0.0, 0.2357092648744583, 0.00018719735089689493, 0.0068743727169930935, 0.031448863446712494, 0.0, 1.116856336593628, 0.055888280272483826, 0.0, 0.0, 0.0, 0.0, 0.06955721974372864, 0.1114373505115509, 0.11681246012449265, 0.05773915722966194, 0.00015745444397907704, 0.16988049447536469, 0.0, 0.0, 0.024910183623433113, 0.053797051310539246, 0.06019298732280731, 0.0, 0.03934258967638016, 0.10768395662307739, 0.0753333643078804, 0.0837491974234581, 0.16896779835224152, 0.02014049142599106, 0.0, 0.06729508936405182, 0.0, 0.21085378527641296, 0.029016515240073204, 0.21809859573841095, 0.0279843807220459, 0.22362250089645386, 0.03447657451033592, 0.04055389389395714, 0.1029297485947609, 0.0298378374427557, 0.0033933587837964296, 0.0, 0.0, 0.0, 0.20575866103172302, 0.04684712737798691, 0.05780502408742905, 0.019852498546242714, 0.059379883110523224, 0.01772438734769821, 0.0, 0.013383779674768448, 0.19442832469940186, 0.0, 0.0016398105071857572, 1.0731772184371948, 0.010617476888000965, 0.00020747390226460993, 0.2178216129541397, 0.0, 0.014236190356314182, 0.14585936069488525, 0.0627611055970192, 0.280550479888916, 0.09320339560508728, 0.0807783305644989, 0.0007129902369342744, 0.024166611954569817], [0.0, 0.7967383861541748, 0.010515306144952774, 0.0, 0.0, 0.9511952996253967, 0.0, 0.015243640169501305, 0.0046011218801140785, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.24499104917049408, 0.0, 0.023033862933516502, 0.0, 0.0, 0.9046792387962341, 0.0, 0.0, 0.0, 0.20917288959026337, 0.0, 0.0, 0.0, 0.0, 0.023562610149383545, 0.7106413245201111, 0.0, 0.0, 0.806309163570404, 0.001292490167543292, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.13817989826202393, 0.0, 0.020449070259928703, 0.7926686406135559, 0.013314137235283852, 0.0, 0.030694054439663887, 0.8911030888557434, 0.0, 0.01104027684777975, 0.8415425419807434, 0.10954130440950394, 0.008225925266742706, 0.01932603493332863, 0.017738644033670425, 0.06586053967475891, 0.0, 0.0, 0.0, 0.004710335284471512, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.05076175555586815, 0.0, 0.016189176589250565, 0.0, 0.6314355134963989, 0.0, 0.020151644945144653, 0.7241675853729248, 0.0, 0.0, 0.0, 0.0, 0.0, 0.7660109996795654, 0.0, 0.0, 0.1073521077632904, 0.0, 0.007332815323024988, 0.00522798765450716, 0.0, 0.008626406081020832, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.08788098394870758, 0.06398512423038483, 0.024791928008198738, 0.0, 0.015649911016225815, 0.7226084470748901, 0.0, 0.0, 0.000621478189714253, 0.0, 0.7554526329040527, 0.0, 0.0, 0.025718482211232185, 0.19527983665466309, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.011882390826940536, 0.7584139704704285, 0.0028904532082378864, 0.00475983927026391, 0.0, 0.0, 0.04620742052793503, 0.022325394675135612, 0.0, 0.0033679858315736055, 0.0, 0.0, 0.052093882113695145, 0.0, 0.0025621806271374226, 0.0, 0.024721557274460793, 0.0, 0.0, 0.10520529001951218, 0.0017461100360378623, 0.09736983478069305, 0.0, 0.02266329526901245, 0.0, 0.6565411686897278, 0.056519754230976105, 0.025859083980321884, 0.014990381896495819, 0.0, 0.0027126106433570385, 0.01211465522646904, 0.01031016930937767, 0.0, 0.024876423180103302, 0.004577460698783398, 0.028947187587618828, 0.7847245931625366, 0.022031841799616814, 0.1324353665113449, 0.06937795132398605, 0.06398914009332657, 0.7995532155036926, 0.0067172483541071415, 0.0, 0.0873136892914772, 0.26812544465065, 0.001347695360891521, 0.0063096038065850735, 0.14728325605392456, 0.009111076593399048, 0.0, 0.012497229501605034, 0.04462809860706329, 0.0, 0.002532103331759572, 0.01062535960227251, 0.8691038489341736, 0.028719326481223106, 0.044062718749046326, 0.0, 0.02446719817817211, 0.05955733358860016, 0.008401031605899334, 0.056939903646707535, 0.21418842673301697, 0.020312905311584473, 0.023959187790751457, 0.023898659273982048], [0.012650275602936745, 0.07921355962753296, 0.06577398627996445, 0.06522398442029953, 0.09919613599777222, 0.09462838619947433, 0.14223796129226685, 0.19100777804851532, 0.12770992517471313, 0.20914332568645477, 0.26043763756752014, 0.21907643973827362, 0.2274775356054306, 0.24578258395195007, 0.16938166320323944, 0.23099246621131897, 0.14092373847961426, 0.23660585284233093, 0.1131763681769371, 0.10677512735128403, 0.12121821939945221, 0.12160901725292206, 0.130888894200325, 0.12086623907089233, 0.2066013067960739, 0.21081876754760742, 0.12991264462471008, 0.15512904524803162, 0.23470301926136017, 0.15070068836212158, 0.11124363541603088, 0.09806329756975174, 0.07904386520385742, 0.0733492523431778, 0.10544400662183762, 0.07881202548742294, 0.14284701645374298, 0.07385113090276718, 0.06307602673768997, 0.08346258848905563, 0.04503737762570381, 0.13474871218204498, 0.10524988174438477, 0.08606717735528946, 0.04010964557528496, 0.021652376279234886, 0.01881505362689495, 0.013570527546107769, 0.01061311736702919, 0.0, 0.017100587487220764, 0.017127839848399162, 0.01957809552550316, 0.048313774168491364, 0.05244932696223259, 0.014987318776547909, 0.014572057873010635, 0.03591742739081383, 0.0, 0.0, 0.0056844474747776985, 0.0, 0.0, 0.003499663434922695, 0.03700621798634529, 0.0, 0.0, 0.011888206005096436, 0.0, 0.003414214588701725, 0.0003294483758509159, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.009706632234156132, 0.003023323602974415, 0.0192309208214283, 0.0, 0.002379362005740404, 0.0019601236563175917, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.008484667167067528, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.003351215273141861, 0.0, 0.0, 0.050957124680280685, 0.0, 0.0, 0.009078223258256912, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.00038310399395413697, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.003567873500287533, 0.0, 0.01264114212244749, 0.03581295162439346, 0.02580469846725464, 0.025909340009093285, 0.019456084817647934, 0.02990950085222721, 0.03536320850253105, 0.04578728973865509, 0.024198705330491066, 0.033839866518974304, 0.051310375332832336, 0.0408235639333725, 0.05167040973901749, 0.0340365506708622, 0.03232090547680855, 0.018917277455329895, 0.005696386564522982, 0.023868920281529427, 0.011901133693754673, 0.0317450650036335, 0.0, 0.0032303831540048122, 0.0031824172474443913, 0.020323961973190308, 0.021674318239092827, 0.02232985571026802, 0.008358356542885303, 0.014174371026456356, 0.005235809367150068, 0.020135672762989998, 0.03671921417117119, 0.06286317110061646, 0.0, 0.0, 0.014791058376431465, 0.0, 0.004773127380758524, 0.06518777459859848, 0.002795589156448841, 0.07610753178596497, 0.05504703149199486, 0.04310930147767067, 0.0, 0.008945273235440254, 1.4486526197288185e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0009690144215710461, 0.0, 0.0, 0.001348835532553494, 0.006113256793469191, 0.026415960863232613, 0.00784927699714899, 0.013454850763082504], [0.031348951160907745, 0.0, 0.3194851577281952, 0.4388832151889801, 0.0, 0.0, 0.23063145577907562, 0.01778290420770645, 0.05220624431967735, 0.0017308082897216082, 0.0, 0.07850631326436996, 0.03809433430433273, 0.03514827415347099, 0.0, 0.0004267526383046061, 0.14226335287094116, 0.014667840674519539, 0.3288516402244568, 0.4322982430458069, 0.0, 0.3155229687690735, 0.1805696189403534, 0.0, 0.0, 0.0, 0.2931688725948334, 0.0, 0.008018034510314465, 0.001018058741465211, 0.0, 0.3876035511493683, 0.0, 0.006471309345215559, 0.2459273785352707, 0.0, 0.010243297554552555, 0.0, 0.3437485992908478, 0.0, 0.5830643177032471, 0.20570331811904907, 0.05781186372041702, 0.12787951529026031, 0.0, 0.0, 0.4875120222568512, 0.5521067380905151, 0.0, 0.008941290900111198, 0.42958202958106995, 0.34989282488822937, 0.0, 0.053934402763843536, 0.1034846305847168, 0.2409726083278656, 0.0927807167172432, 0.0, 0.0, 0.5546292066574097, 0.36552777886390686, 0.0, 0.0, 0.22350870072841644, 0.0, 0.24476324021816254, 0.0, 0.22069105505943298, 0.0, 0.1323530673980713, 0.014892819337546825, 0.0, 0.0, 0.3467576801776886, 0.0, 0.0, 0.45658236742019653, 0.0, 0.5636751055717468, 0.48422473669052124, 0.0, 0.0, 0.2971497178077698, 0.0074135675095021725, 0.0372510589659214, 0.5292679667472839, 0.12214399129152298, 0.13121822476387024, 0.18306873738765717, 0.0, 0.5675877928733826, 0.5019762516021729, 0.0, 0.0, 0.3897256553173065, 0.0, 0.5080695748329163, 0.23139405250549316, 0.5171197652816772, 0.03291639685630798, 0.11196219176054001, 0.5618293285369873, 0.7089813947677612, 0.0, 0.0, 0.2968286871910095, 0.25625887513160706, 0.480260968208313, 0.0, 0.0, 0.27347061038017273, 0.0, 0.0, 0.01930648274719715, 0.34335532784461975, 0.32671767473220825, 0.14015305042266846, 0.006798886694014072, 0.2333400845527649, 0.0, 0.18779511749744415, 0.0, 0.45020461082458496, 0.0, 0.0, 0.2899637818336487, 0.16190718114376068, 0.4581625759601593, 0.00907823909074068, 0.5537040829658508, 0.3827120065689087, 0.0, 0.0, 0.355435311794281, 0.08985245227813721, 0.13963623344898224, 0.1721457988023758, 0.23051108419895172, 0.599742591381073, 0.03299570828676224, 0.5823078155517578, 0.0, 0.02714536339044571, 0.046870891004800797, 0.11906077712774277, 0.0, 0.3148908317089081, 0.10873398929834366, 0.054102059453725815, 0.32329559326171875, 0.1639057993888855, 0.3514508306980133, 0.01939733512699604, 0.42248204350471497, 0.5529324412345886, 0.04634736105799675, 0.4623594284057617, 0.05328831821680069, 0.5001856088638306, 0.06173576042056084, 0.03080410696566105, 0.3934343755245209, 0.07899822294712067, 0.06072811782360077, 0.0, 0.019107423722743988, 0.38232141733169556, 0.0, 0.039200909435749054, 0.11549626290798187, 0.5153955817222595, 0.243168905377388, 0.07695469260215759, 0.15657289326190948, 0.0, 0.3065827488899231, 0.04567103832960129, 0.04552740976214409, 0.4983766973018646, 0.043824877589941025, 0.05895378440618515, 0.485635370016098, 0.07956552505493164, 0.6371440887451172, 0.5083460211753845, 0.09751798957586288, 0.4201031029224396, 0.057965800166130066, 0.21113677322864532, 0.32844579219818115, 0.0, 0.03446109965443611], [0.04791339486837387, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.016352219507098198, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0056970855221152306, 0.0, 0.05994395539164543, 0.04630332440137863, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.06118996813893318, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.005640325602144003, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0035381345078349113, 0.007648357190191746, 0.06703606992959976, 0.0, 0.0, 0.0, 0.0817340761423111, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.12374958395957947, 0.0, 0.004797552712261677, 0.15754486620426178, 0.05577193945646286, 0.02525399997830391, 0.08914291858673096, 0.08008948713541031, 0.017852699384093285, 0.13168393075466156, 0.02061072550714016, 0.026830241084098816, 0.14085455238819122, 0.1800042688846588, 0.10485778003931046, 0.26463615894317627, 0.26355791091918945, 0.04670626297593117, 0.1779245287179947, 0.17441008985042572, 0.17096059024333954, 0.22256188094615936, 0.15978163480758667, 0.14837993681430817, 0.27392181754112244, 0.2733137011528015, 0.17489668726921082, 0.2859697937965393, 0.16916897892951965, 0.23503842949867249, 0.14764678478240967, 0.3895520865917206, 0.4711702764034271, 0.21490147709846497, 0.10229837149381638, 0.31687355041503906, 0.4444630444049835, 0.2642090916633606, 0.30592721700668335, 0.21114420890808105, 0.35202592611312866, 0.3977487087249756, 0.29012781381607056, 0.40964803099632263, 0.520248293876648, 0.5026209950447083, 0.2788168787956238, 0.28886526823043823, 0.3670854866504669, 0.6112028956413269, 0.3257269859313965, 0.47575104236602783, 0.40857183933258057, 0.41207510232925415, 0.2300301194190979, 0.43415990471839905, 0.4402182400226593, 0.2019868940114975, 0.2673930823802948, 0.18799448013305664, 0.18204334378242493, 0.018942955881357193, 0.007407049182802439, 0.0, 0.0, 0.0, 0.0, 0.006686428561806679, 0.019690539687871933, 0.007434654049575329, 0.0, 0.019477181136608124, 0.0, 0.004805600270628929, 0.058400627225637436, 0.09614258259534836, 0.08122467994689941, 0.0, 0.008907250128686428, 0.0007700411370024085, 0.006170183885842562, 0.003660807153210044, 0.03241937607526779, 0.0, 0.0, 0.02956155128777027, 0.0, 0.03537636995315552, 0.0, 0.03680773824453354, 0.01656847447156906, 0.03836612030863762, 0.014198112301528454, 0.005906642414629459, 0.019048400223255157, 0.010345070622861385, 0.0, 0.0, 0.0, 0.0, 0.042820967733860016, 0.10129494220018387, 0.008949183858931065, 0.0, 0.04279744625091553, 0.041681982576847076, 0.0, 0.052077312022447586, 0.04095778614282608, 0.0177069790661335, 0.003953906707465649, 0.03183889016509056, 0.030984953045845032, 0.015248137526214123, 0.06259728223085403, 0.0, 0.002678767079487443, 0.1040601059794426, 0.04011617600917816, 0.09380456805229187, 0.0400465652346611, 0.03251097723841667, 0.06674247980117798, 0.22845157980918884], [0.0, 0.07069972902536392, 0.05051474645733833, 0.0, 0.0, 0.0, 0.0, 0.02712063305079937, 0.0, 0.012250227853655815, 0.0, 0.06874936819076538, 0.03088303841650486, 0.06800194829702377, 0.0, 0.22687411308288574, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.002351800911128521, 0.0, 0.2174697071313858, 0.0, 0.0, 0.0, 0.0, 0.10835354775190353, 0.099263034760952, 0.0, 0.0, 0.004964068531990051, 0.0, 0.0, 0.03934747725725174, 1.1703604459762573, 0.023208511993288994, 0.0, 0.0, 0.0, 0.13018809258937836, 0.007514247205108404, 0.0, 0.04154588282108307, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.07177643477916718, 0.09073635935783386, 0.0, 0.0, 0.030953990295529366, 0.16041089594364166, 1.3451429605484009, 0.0, 0.0, 0.02161300554871559, 1.1545751094818115, 0.00422415928915143, 0.03877100348472595, 0.0, 0.0, 0.0, 0.052665818482637405, 0.0, 0.012598022818565369, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.18051408231258392, 0.0, 0.04494093731045723, 0.03091484121978283, 0.06061146408319473, 1.1551308631896973, 0.010964034125208855, 0.021758560091257095, 0.004827207885682583, 1.1552937030792236, 0.033238839358091354, 0.0, 0.0, 0.0, 0.0, 0.0689328983426094, 0.09836308658123016, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.07998272031545639, 0.22841280698776245, 0.0, 0.0, 0.0039035161025822163, 0.01989024132490158, 0.0, 0.0, 0.05342122167348862, 1.149350643157959, 0.0, 0.046339500695466995, 0.0, 0.00615586806088686, 0.0028417245484888554, 0.0022053346037864685, 0.0, 0.036600373685359955, 0.05493997037410736, 0.0, 1.180161714553833, 0.03818877041339874, 0.14755779504776, 0.2819945514202118, 0.14291714131832123, 0.15200015902519226, 0.0, 0.09093781560659409, 0.0, 0.054225143045186996, 0.12591393291950226, 0.04801858961582184, 0.23642520606517792, 1.2213599681854248, 0.11956530809402466, 0.1927299052476883, 0.15931721031665802, 0.07402943074703217, 0.17343750596046448, 0.0940406396985054, 0.0, 0.13893571496009827, 0.0618995763361454, 0.2176820933818817, 0.005475946236401796, 0.1737474501132965, 0.0, 0.14213897287845612, 0.028524337336421013, 0.012039335444569588, 0.15729522705078125, 0.0, 0.0, 0.0, 0.0, 0.0, 0.08123326301574707, 0.26279303431510925, 0.0, 0.0, 0.13887321949005127, 0.0043711489997804165, 0.0, 0.050895147025585175, 0.19387100636959076, 1.2304948568344116, 0.05209881812334061, 0.06187208369374275, 0.076168492436409, 0.07410158962011337, 0.12196039408445358, 0.034864701330661774, 0.04493449255824089, 0.17566566169261932, 0.09588471800088882, 0.15915554761886597, 0.07115024328231812, 0.012718303129076958, 0.0, 0.0], [0.051313839852809906, 0.0, 0.0, 0.0, 0.0, 0.0, 0.011777135543525219, 0.0, 0.0007592649781145155, 0.02775978296995163, 0.18775604665279388, 0.012537508271634579, 0.024342911317944527, 0.0, 0.018242662772536278, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0021534739062190056, 0.0, 0.027508758008480072, 0.000682770914863795, 0.12097012251615524, 0.12312999367713928, 0.18921853601932526, 0.1375146359205246, 0.11057095229625702, 0.09583107382059097, 0.09480797499418259, 0.06888970732688904, 0.06347418576478958, 0.13372263312339783, 0.16194726526737213, 0.12805438041687012, 0.12128477543592453, 0.10506696999073029, 0.11880180984735489, 0.09743129462003708, 0.11051853001117706, 0.05415823683142662, 0.09109830111265182, 0.14023457467556, 0.150446355342865, 0.0804547369480133, 0.08720234781503677, 0.11990010738372803, 0.13883140683174133, 0.15282918512821198, 0.21201059222221375, 0.2986994981765747, 0.2682299315929413, 0.42211347818374634, 0.23506364226341248, 0.29049205780029297, 0.39107218384742737, 0.35813072323799133, 0.344603031873703, 0.4259893596172333, 0.5469961762428284, 0.5059076547622681, 0.4438015818595886, 0.696173369884491, 0.6845083236694336, 0.7137088179588318, 0.5929418802261353, 0.7366358637809753, 0.6558579802513123, 0.7214391827583313, 0.86838698387146, 0.7185313701629639, 0.5157915353775024, 0.7318438291549683, 0.6440234780311584, 0.36041662096977234, 0.5473276376724243, 0.2162020206451416, 0.2046584188938141, 0.29207977652549744, 0.24259600043296814, 0.17155498266220093, 0.18539434671401978, 0.20686188340187073, 0.12305493652820587, 0.10777214914560318, 0.09076923877000809, 0.07565868645906448, 0.18468493223190308, 0.12352992594242096, 0.1263098418712616, 0.17981112003326416, 0.11562694609165192, 0.06658599525690079, 0.1427702009677887, 0.02097739465534687, 0.0, 0.00630556046962738, 0.059432271867990494, 0.014616694301366806, 0.006727224215865135, 0.019563347101211548, 0.03895184025168419, 0.02005833573639393, 0.0, 0.0, 0.009173264726996422, 0.0013224862050265074, 0.00013614150520879775, 0.0, 0.01131234597414732, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.004778479691594839, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0121386107057333, 0.0, 0.0, 0.0, 0.0, 0.00787060335278511, 0.0]]]},\n            {\n            'hltrCFG': {'tokenization_config': {\"token_prefix\": \"\", \"partial_token_prefix\": \"##\"}\n                }\n            })\n         }, function (err) {\n            console.log(err);\n        })"},"metadata":{}}]},{"cell_type":"markdown","source":"> - Factors that focus on specific portions of the text (beginning, middle, and end. This is interesting as the only signal the model gets about how these tokens are connected are the positional encodings. This indicates the neurons that are keeping track of the order of words. Further examination is required to assess whether these FFNN neurons directly respond to the time signal, or if they are responding to a specific trigger in the self-attention layer's transformation of the hidden state.\n> - Factors corresponding to linguistic features, for example, we can see factors for pronouns, auxiliary verbs, other verbs and noun such as **\"ocean\"**,**\"waters\"**, linking verbs **(is, are,** were), and a factor that favors nouns and their adjectives.\n> - A factor corresponding to commas, a syntactic feature.\n> - We can observe that the model concentrates on such words that are related to emotions, though this is human intuition as the model never sees the actual emotions. words like **\"tranquil\", \"peaceful\", \"solitude\", \"symphony\"**","metadata":{}},{"cell_type":"markdown","source":"## Pre-trained RoBERTa Model Neuron Factorization:","metadata":{}},{"cell_type":"code","source":"lm = ecco.from_pretrained('roberta-base', activations=True)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-31T18:45:27.490123Z","iopub.execute_input":"2023-05-31T18:45:27.491273Z","iopub.status.idle":"2023-05-31T18:45:29.893510Z","shell.execute_reply.started":"2023-05-31T18:45:27.491233Z","shell.execute_reply":"2023-05-31T18:45:29.892219Z"},"_kg_hide-input":false,"trusted":true},"execution_count":68,"outputs":[{"name":"stderr","text":"Some weights of the model checkpoint at roberta-base were not used when initializing RobertaModel: ['lm_head.bias', 'lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.dense.bias', 'lm_head.dense.weight']\n- This IS expected if you are initializing RobertaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n- This IS NOT expected if you are initializing RobertaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\nSome weights of RobertaModel were not initialized from the model checkpoint at roberta-base and are newly initialized: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight']\nYou should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n","output_type":"stream"}]},{"cell_type":"code","source":"# Factorize the activations of all layers\ninputs = lm.tokenizer([text], return_tensors=\"pt\")\noutput = lm(inputs)\nnmf_1 = output.run_nmf(n_components=10)                         #non-negative matrix factorization on 10 components\nnmf_1.explore()","metadata":{"execution":{"iopub.status.busy":"2023-05-31T18:45:52.295919Z","iopub.execute_input":"2023-05-31T18:45:52.296347Z","iopub.status.idle":"2023-05-31T18:45:54.761006Z","shell.execute_reply.started":"2023-05-31T18:45:52.296314Z","shell.execute_reply":"2023-05-31T18:45:54.759564Z"},"trusted":true},"execution_count":75,"outputs":[{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"<html lang=\"en\">\n<script src=\"https://requirejs.org/docs/release/2.3.6/minified/require.js\"></script>\n<script>\n    var ecco_url = 'https://storage.googleapis.com/ml-intro/ecco/'\n    //var ecco_url = 'http://localhost:8000/'\n\n    if (window.ecco === undefined) window.ecco = {}\n\n    // Setup the paths of the script we'll be using\n    requirejs.config({\n        urlArgs: \"bust=\" + (new Date()).getTime(),\n        nodeRequire: require,\n        paths: {\n            d3: \"https://d3js.org/d3.v6.min\", // This is only for use in setup.html and basic.html\n            \"d3-array\": \"https://d3js.org/d3-array.v2.min\",\n            jquery: \"https://code.jquery.com/jquery-3.5.1.min\",\n            ecco: ecco_url + 'js/0.0.6/ecco-bundle.min',\n            xregexp: 'https://cdnjs.cloudflare.com/ajax/libs/xregexp/3.2.0/xregexp-all.min'\n        }\n    });\n\n    // Add the css file\n    //requirejs(['d3'],\n    //    function (d3) {\n    //        d3.select('#css').attr('href', ecco_url + 'html/styles.css')\n    //    })\n\n    console.log('Ecco initialize!!')\n\n    // returns a 'basic' object. basic.init() selects the html div we'll be\n    // rendering the html into, adds styles.css to the document.\n    define('basic', ['d3'],\n        function (d3) {\n            return {\n                init: function (viz_id = null) {\n                    if (viz_id == null) {\n                        viz_id = \"viz_\" + Math.round(Math.random() * 10000000)\n                    }\n                    // Select the div rendered below, change its id\n                    const div = d3.select('#basic').attr('id', viz_id),\n                        div_parent = d3.select('#' + viz_id).node().parentNode\n\n                    // Link to CSS file\n                    d3.select(div_parent).insert('link')\n                        .attr('rel', 'stylesheet')\n                        .attr('type', 'text/css')\n                        .attr('href', ecco_url + 'html/0.0.2/styles.css')\n\n                    return viz_id\n                }\n            }\n        }, function (err) {\n            console.log(err);\n        }\n    )\n</script>\n\n<head>\n    <link id='css' rel=\"stylesheet\" type=\"text/css\">\n</head>\n<div id=\"basic\"></div>\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.Javascript object>","application/javascript":"\n         requirejs(['basic', 'ecco'], function(basic, ecco){\n            const viz_id = basic.init()\n            \n            ecco.interactiveTokensAndFactorSparklines(viz_id, {'tokens': [{'token': '<s>', 'token_id': 0, 'type': 'input', 'position': 0}, {'token': 'ĠThe', 'token_id': 20, 'type': 'input', 'position': 1}, {'token': 'Ġvast', 'token_id': 4714, 'type': 'input', 'position': 2}, {'token': 'Ġexp', 'token_id': 4553, 'type': 'input', 'position': 3}, {'token': 'anse', 'token_id': 31327, 'type': 'input', 'position': 4}, {'token': 'Ġof', 'token_id': 9, 'type': 'input', 'position': 5}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 6}, {'token': 'Ġocean', 'token_id': 6444, 'type': 'input', 'position': 7}, {'token': 'Ġstretches', 'token_id': 15706, 'type': 'input', 'position': 8}, {'token': 'Ġout', 'token_id': 66, 'type': 'input', 'position': 9}, {'token': 'Ġbefore', 'token_id': 137, 'type': 'input', 'position': 10}, {'token': 'Ġme', 'token_id': 162, 'type': 'input', 'position': 11}, {'token': ',', 'token_id': 6, 'type': 'input', 'position': 12}, {'token': 'Ġits', 'token_id': 63, 'type': 'input', 'position': 13}, {'token': 'Ġcalm', 'token_id': 6327, 'type': 'input', 'position': 14}, {'token': 'Ġwaters', 'token_id': 5794, 'type': 'input', 'position': 15}, {'token': 'Ġreflecting', 'token_id': 10811, 'type': 'input', 'position': 16}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 17}, {'token': 'Ġsky', 'token_id': 6360, 'type': 'input', 'position': 18}, {'token': 'Ġabove', 'token_id': 1065, 'type': 'input', 'position': 19}, {'token': '.', 'token_id': 4, 'type': 'input', 'position': 20}, {'token': 'Ċ', 'token_id': 50118, 'type': 'input', 'position': 21}, {'token': 'Its', 'token_id': 30872, 'type': 'input', 'position': 22}, {'token': 'Ġtranquil', 'token_id': 33535, 'type': 'input', 'position': 23}, {'token': 'Ġsurface', 'token_id': 4084, 'type': 'input', 'position': 24}, {'token': ',', 'token_id': 6, 'type': 'input', 'position': 25}, {'token': 'Ġund', 'token_id': 2432, 'type': 'input', 'position': 26}, {'token': 'ist', 'token_id': 661, 'type': 'input', 'position': 27}, {'token': 'urbed', 'token_id': 35250, 'type': 'input', 'position': 28}, {'token': 'Ġby', 'token_id': 30, 'type': 'input', 'position': 29}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 30}, {'token': 'Ġchaos', 'token_id': 7724, 'type': 'input', 'position': 31}, {'token': 'Ġof', 'token_id': 9, 'type': 'input', 'position': 32}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 33}, {'token': 'Ġworld', 'token_id': 232, 'type': 'input', 'position': 34}, {'token': ',', 'token_id': 6, 'type': 'input', 'position': 35}, {'token': 'Ġinspires', 'token_id': 25362, 'type': 'input', 'position': 36}, {'token': 'Ġa', 'token_id': 10, 'type': 'input', 'position': 37}, {'token': 'Ġsense', 'token_id': 1472, 'type': 'input', 'position': 38}, {'token': 'Ġof', 'token_id': 9, 'type': 'input', 'position': 39}, {'token': 'Ġse', 'token_id': 842, 'type': 'input', 'position': 40}, {'token': 'rene', 'token_id': 20962, 'type': 'input', 'position': 41}, {'token': 'Ġtranqu', 'token_id': 36474, 'type': 'input', 'position': 42}, {'token': 'ility', 'token_id': 13005, 'type': 'input', 'position': 43}, {'token': '.', 'token_id': 4, 'type': 'input', 'position': 44}, {'token': 'Ċ', 'token_id': 50118, 'type': 'input', 'position': 45}, {'token': 'The', 'token_id': 133, 'type': 'input', 'position': 46}, {'token': 'Ġsoft', 'token_id': 3793, 'type': 'input', 'position': 47}, {'token': 'Ġwhisper', 'token_id': 37539, 'type': 'input', 'position': 48}, {'token': 'Ġof', 'token_id': 9, 'type': 'input', 'position': 49}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 50}, {'token': 'Ġwaves', 'token_id': 6995, 'type': 'input', 'position': 51}, {'token': 'Ġkissing', 'token_id': 20479, 'type': 'input', 'position': 52}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 53}, {'token': 'Ġshore', 'token_id': 8373, 'type': 'input', 'position': 54}, {'token': 'line', 'token_id': 1902, 'type': 'input', 'position': 55}, {'token': 'Ġis', 'token_id': 16, 'type': 'input', 'position': 56}, {'token': 'Ġa', 'token_id': 10, 'type': 'input', 'position': 57}, {'token': 'Ġpeaceful', 'token_id': 7053, 'type': 'input', 'position': 58}, {'token': 'Ġsym', 'token_id': 22850, 'type': 'input', 'position': 59}, {'token': 'phony', 'token_id': 29788, 'type': 'input', 'position': 60}, {'token': ',', 'token_id': 6, 'type': 'input', 'position': 61}, {'token': 'Ġa', 'token_id': 10, 'type': 'input', 'position': 62}, {'token': 'Ġmel', 'token_id': 15352, 'type': 'input', 'position': 63}, {'token': 'od', 'token_id': 1630, 'type': 'input', 'position': 64}, {'token': 'ious', 'token_id': 6514, 'type': 'input', 'position': 65}, {'token': 'Ġsong', 'token_id': 2214, 'type': 'input', 'position': 66}, {'token': 'Ġof', 'token_id': 9, 'type': 'input', 'position': 67}, {'token': 'Ċ', 'token_id': 50118, 'type': 'input', 'position': 68}, {'token': 'nature', 'token_id': 43102, 'type': 'input', 'position': 69}, {'token': 'Ġthat', 'token_id': 14, 'type': 'input', 'position': 70}, {'token': 'Ġso', 'token_id': 98, 'type': 'input', 'position': 71}, {'token': 'othes', 'token_id': 32915, 'type': 'input', 'position': 72}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 73}, {'token': 'Ġsoul', 'token_id': 7047, 'type': 'input', 'position': 74}, {'token': '.', 'token_id': 4, 'type': 'input', 'position': 75}, {'token': 'ĠThe', 'token_id': 20, 'type': 'input', 'position': 76}, {'token': 'Ġgentle', 'token_id': 16634, 'type': 'input', 'position': 77}, {'token': ',', 'token_id': 6, 'type': 'input', 'position': 78}, {'token': 'Ġrhyth', 'token_id': 41063, 'type': 'input', 'position': 79}, {'token': 'mic', 'token_id': 15796, 'type': 'input', 'position': 80}, {'token': 'Ġmotion', 'token_id': 4298, 'type': 'input', 'position': 81}, {'token': 'Ġof', 'token_id': 9, 'type': 'input', 'position': 82}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 83}, {'token': 'Ġocean', 'token_id': 6444, 'type': 'input', 'position': 84}, {'token': 'âĢ', 'token_id': 17, 'type': 'input', 'position': 85}, {'token': 'Ļ', 'token_id': 27, 'type': 'input', 'position': 86}, {'token': 's', 'token_id': 29, 'type': 'input', 'position': 87}, {'token': 'Ġcurrents', 'token_id': 28124, 'type': 'input', 'position': 88}, {'token': 'Ġembodies', 'token_id': 33434, 'type': 'input', 'position': 89}, {'token': 'Ġa', 'token_id': 10, 'type': 'input', 'position': 90}, {'token': 'Ġsilent', 'token_id': 8454, 'type': 'input', 'position': 91}, {'token': 'Ċ', 'token_id': 50118, 'type': 'input', 'position': 92}, {'token': 'strength', 'token_id': 33687, 'type': 'input', 'position': 93}, {'token': ',', 'token_id': 6, 'type': 'input', 'position': 94}, {'token': 'Ġa', 'token_id': 10, 'type': 'input', 'position': 95}, {'token': 'Ġsense', 'token_id': 1472, 'type': 'input', 'position': 96}, {'token': 'Ġof', 'token_id': 9, 'type': 'input', 'position': 97}, {'token': 'Ġenduring', 'token_id': 16480, 'type': 'input', 'position': 98}, {'token': 'Ġse', 'token_id': 842, 'type': 'input', 'position': 99}, {'token': 'ren', 'token_id': 2558, 'type': 'input', 'position': 100}, {'token': 'ity', 'token_id': 1571, 'type': 'input', 'position': 101}, {'token': 'Ġthat', 'token_id': 14, 'type': 'input', 'position': 102}, {'token': 'Ġis', 'token_id': 16, 'type': 'input', 'position': 103}, {'token': 'Ġprofoundly', 'token_id': 27301, 'type': 'input', 'position': 104}, {'token': 'Ġpeaceful', 'token_id': 7053, 'type': 'input', 'position': 105}, {'token': '.', 'token_id': 4, 'type': 'input', 'position': 106}, {'token': 'ĠThe', 'token_id': 20, 'type': 'input', 'position': 107}, {'token': 'Ġbound', 'token_id': 8191, 'type': 'input', 'position': 108}, {'token': 'less', 'token_id': 1672, 'type': 'input', 'position': 109}, {'token': 'Ġbeauty', 'token_id': 4002, 'type': 'input', 'position': 110}, {'token': 'Ġof', 'token_id': 9, 'type': 'input', 'position': 111}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 112}, {'token': 'Ġsea', 'token_id': 3342, 'type': 'input', 'position': 113}, {'token': ',', 'token_id': 6, 'type': 'input', 'position': 114}, {'token': 'Ċ', 'token_id': 50118, 'type': 'input', 'position': 115}, {'token': 'with', 'token_id': 5632, 'type': 'input', 'position': 116}, {'token': 'Ġits', 'token_id': 63, 'type': 'input', 'position': 117}, {'token': 'Ġcapt', 'token_id': 13363, 'type': 'input', 'position': 118}, {'token': 'ivating', 'token_id': 28191, 'type': 'input', 'position': 119}, {'token': 'Ġcalm', 'token_id': 6327, 'type': 'input', 'position': 120}, {'token': 'ness', 'token_id': 1825, 'type': 'input', 'position': 121}, {'token': ',', 'token_id': 6, 'type': 'input', 'position': 122}, {'token': 'Ġoffers', 'token_id': 1523, 'type': 'input', 'position': 123}, {'token': 'Ġa', 'token_id': 10, 'type': 'input', 'position': 124}, {'token': 'Ġsanctuary', 'token_id': 13488, 'type': 'input', 'position': 125}, {'token': 'Ġfrom', 'token_id': 31, 'type': 'input', 'position': 126}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 127}, {'token': 'Ġbustling', 'token_id': 29019, 'type': 'input', 'position': 128}, {'token': 'Ġnoise', 'token_id': 6496, 'type': 'input', 'position': 129}, {'token': 'Ġof', 'token_id': 9, 'type': 'input', 'position': 130}, {'token': 'Ġeveryday', 'token_id': 7476, 'type': 'input', 'position': 131}, {'token': 'Ġlife', 'token_id': 301, 'type': 'input', 'position': 132}, {'token': ',', 'token_id': 6, 'type': 'input', 'position': 133}, {'token': 'Ġa', 'token_id': 10, 'type': 'input', 'position': 134}, {'token': 'Ġhaven', 'token_id': 2220, 'type': 'input', 'position': 135}, {'token': 'Ċ', 'token_id': 50118, 'type': 'input', 'position': 136}, {'token': 'where', 'token_id': 8569, 'type': 'input', 'position': 137}, {'token': 'Ġone', 'token_id': 65, 'type': 'input', 'position': 138}, {'token': 'Ġcan', 'token_id': 64, 'type': 'input', 'position': 139}, {'token': 'Ġfind', 'token_id': 465, 'type': 'input', 'position': 140}, {'token': 'Ġpeace', 'token_id': 1987, 'type': 'input', 'position': 141}, {'token': 'Ġand', 'token_id': 8, 'type': 'input', 'position': 142}, {'token': 'Ġsolitude', 'token_id': 41813, 'type': 'input', 'position': 143}, {'token': '.', 'token_id': 4, 'type': 'input', 'position': 144}, {'token': 'ĠIt', 'token_id': 85, 'type': 'input', 'position': 145}, {'token': 'âĢ', 'token_id': 17, 'type': 'input', 'position': 146}, {'token': 'Ļ', 'token_id': 27, 'type': 'input', 'position': 147}, {'token': 's', 'token_id': 29, 'type': 'input', 'position': 148}, {'token': 'Ġa', 'token_id': 10, 'type': 'input', 'position': 149}, {'token': 'Ġplace', 'token_id': 317, 'type': 'input', 'position': 150}, {'token': 'Ġwhere', 'token_id': 147, 'type': 'input', 'position': 151}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 152}, {'token': 'Ġmind', 'token_id': 1508, 'type': 'input', 'position': 153}, {'token': 'Ġcan', 'token_id': 64, 'type': 'input', 'position': 154}, {'token': 'Ġrest', 'token_id': 1079, 'type': 'input', 'position': 155}, {'token': ',', 'token_id': 6, 'type': 'input', 'position': 156}, {'token': 'Ġallowing', 'token_id': 2455, 'type': 'input', 'position': 157}, {'token': 'Ġthoughts', 'token_id': 4312, 'type': 'input', 'position': 158}, {'token': 'Ġto', 'token_id': 7, 'type': 'input', 'position': 159}, {'token': 'Ċ', 'token_id': 50118, 'type': 'input', 'position': 160}, {'token': 'flow', 'token_id': 19322, 'type': 'input', 'position': 161}, {'token': 'Ġas', 'token_id': 25, 'type': 'input', 'position': 162}, {'token': 'Ġfreely', 'token_id': 13215, 'type': 'input', 'position': 163}, {'token': 'Ġas', 'token_id': 25, 'type': 'input', 'position': 164}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 165}, {'token': 'Ġwaters', 'token_id': 5794, 'type': 'input', 'position': 166}, {'token': 'Ġthemselves', 'token_id': 1235, 'type': 'input', 'position': 167}, {'token': '.', 'token_id': 4, 'type': 'input', 'position': 168}, {'token': 'ĠThe', 'token_id': 20, 'type': 'input', 'position': 169}, {'token': 'Ġocean', 'token_id': 6444, 'type': 'input', 'position': 170}, {'token': ',', 'token_id': 6, 'type': 'input', 'position': 171}, {'token': 'Ġin', 'token_id': 11, 'type': 'input', 'position': 172}, {'token': 'Ġits', 'token_id': 63, 'type': 'input', 'position': 173}, {'token': 'Ġinfinite', 'token_id': 32952, 'type': 'input', 'position': 174}, {'token': 'Ġtranqu', 'token_id': 36474, 'type': 'input', 'position': 175}, {'token': 'ility', 'token_id': 13005, 'type': 'input', 'position': 176}, {'token': ',', 'token_id': 6, 'type': 'input', 'position': 177}, {'token': 'Ġserves', 'token_id': 4542, 'type': 'input', 'position': 178}, {'token': 'Ġas', 'token_id': 25, 'type': 'input', 'position': 179}, {'token': 'Ġa', 'token_id': 10, 'type': 'input', 'position': 180}, {'token': 'Ġreminder', 'token_id': 8306, 'type': 'input', 'position': 181}, {'token': 'Ċ', 'token_id': 50118, 'type': 'input', 'position': 182}, {'token': 'of', 'token_id': 1116, 'type': 'input', 'position': 183}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 184}, {'token': 'Ġsimple', 'token_id': 2007, 'type': 'input', 'position': 185}, {'token': 'Ġand', 'token_id': 8, 'type': 'input', 'position': 186}, {'token': 'Ġquiet', 'token_id': 5128, 'type': 'input', 'position': 187}, {'token': 'Ġbeauty', 'token_id': 4002, 'type': 'input', 'position': 188}, {'token': 'Ġthat', 'token_id': 14, 'type': 'input', 'position': 189}, {'token': 'Ġexists', 'token_id': 8785, 'type': 'input', 'position': 190}, {'token': 'Âł', 'token_id': 50141, 'type': 'input', 'position': 191}, {'token': 'in', 'token_id': 179, 'type': 'input', 'position': 192}, {'token': 'Âł', 'token_id': 50141, 'type': 'input', 'position': 193}, {'token': 'our', 'token_id': 2126, 'type': 'input', 'position': 194}, {'token': 'Âł', 'token_id': 50141, 'type': 'input', 'position': 195}, {'token': 'world', 'token_id': 8331, 'type': 'input', 'position': 196}, {'token': '.', 'token_id': 4, 'type': 'input', 'position': 197}, {'token': 'Ċ', 'token_id': 50118, 'type': 'input', 'position': 198}, {'token': '</s>', 'token_id': 2, 'type': 'input', 'position': 199}], 'factors': [[[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.004648796282708645, 0.014092928729951382, 0.0073826611042022705, 0.0, 0.0, 0.0, 0.0, 4.727337000076659e-05, 0.0, 0.0, 0.001952331862412393, 6.68579626083374, 0.0, 0.003913078922778368, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0015404843725264072, 0.0, 0.0, 0.0, 0.0, 0.0, 0.00034395160037092865, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 6.688753128051758, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0038419298361986876, 0.0032701848540455103, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.00020269516971893609, 0.0, 0.0, 0.0, 0.0, 0.0, 0.004119517747312784, 0.004500196315348148, 6.676978588104248, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0013449823018163443, 0.0009959939634427428, 0.0, 0.0, 0.0008940346888266504, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.002061585895717144, 6.68698787689209, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0019004668574780226, 0.0, 0.007367130368947983, 0.0, 0.001126352814026177, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.003133761929348111, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.003739438485354185, 0.0, 0.0, 0.0, 0.015212965197861195, 0.004262582398951054, 0.0, 0.0, 0.0, 0.0, 0.0004808996163774282, 6.689610004425049, 0.005186339840292931, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.004867335315793753, 0.0012876938562840223, 0.0007715860847383738, 0.0, 0.0, 0.001807166961953044, 0.003453334793448448, 0.0, 0.0, 0.013501428067684174, 0.0, 0.01776731386780739, 0.0, 0.0, 0.0014760720077902079, 6.683019161224365, 0.0, 0.0, 0.0, 0.003013096284121275, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0025207928847521544, 0.0, 0.0, 0.0, 0.051939353346824646, 0.0, 8.308097676490434e-06, 0.0006249349098652601, 0.001512704766355455, 0.0, 0.0, 0.0, 0.0, 0.0061887893825769424, 0.0, 0.008597326464951038, 0.0, 0.0, 0.008180086500942707, 0.0, 0.003487830748781562], [6.17616081237793, 0.02591400407254696, 0.0, 0.0, 0.002406845800578594, 0.0, 0.0, 0.0, 0.0008911903132684529, 0.0, 0.0021316036581993103, 0.013329360634088516, 0.0, 0.0, 0.0, 0.0, 0.010013130493462086, 0.0, 0.0, 0.0, 0.0035248722415417433, 0.0, 0.00503079779446125, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0071013318374753, 0.007953646592795849, 0.0, 0.0, 0.0, 0.0, 0.0, 0.006526174955070019, 0.0056260982528328896, 0.0, 0.0, 0.0, 0.0, 0.0, 0.000999337062239647, 0.0, 0.0009020520374178886, 0.0, 0.009632818400859833, 0.0, 0.0007657031528651714, 0.0, 0.0, 0.0, 0.005550000816583633, 0.0, 0.0, 0.00942216720432043, 0.010654046200215816, 0.003327356418594718, 0.003346042474731803, 0.0017019560327753425, 0.0, 0.0, 0.0, 0.0, 0.0, 0.001024286262691021, 0.0, 0.017458615824580193, 0.043096646666526794, 0.0596059150993824, 0.010041052475571632, 0.0, 0.0036383483093231916, 0.0, 0.0, 0.008359451778233051, 0.0030621092300862074, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.00347569165751338, 0.0, 0.003946063574403524, 0.021237285807728767, 0.03638532757759094, 0.023855997249484062, 0.04723073169589043, 0.06762278079986572, 0.0, 0.0, 0.0, 0.0022133325692266226, 0.002823628019541502, 0.0, 0.0, 0.0, 0.00041088886791840196, 0.0037552916910499334, 0.0, 0.0020638033747673035, 0.0, 0.003298115450888872, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.01954547129571438, 0.04185473173856735, 0.0905885174870491, 0.00713107455521822, 0.0, 0.004537325352430344, 0.0, 0.0, 0.010622027330100536, 0.0, 0.0, 0.0, 0.0, 0.0, 0.019399380311369896, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0007873725262470543, 0.0, 0.0, 0.004521781112998724, 0.0, 0.012072904035449028, 0.005118569824844599, 0.001958775334060192, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.023034783080220222, 0.07680859416723251, 0.05116443336009979, 0.007651969790458679, 0.013870607130229473, 0.0, 0.0, 0.0, 0.0, 0.003754627425223589, 0.0, 0.0, 0.0, 0.0, 0.006097834557294846, 0.0, 0.0028545523528009653, 0.0, 0.00423883693292737, 0.0, 0.0, 0.0, 0.0, 0.0, 0.03539086505770683, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0053322031162679195, 0.0, 0.0, 0.004890564363449812, 0.0, 0.0, 0.0, 0.0, 0.00549444230273366, 0.005964728072285652, 0.07660020142793655], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.18634627759456635, 0.10318005830049515, 0.058862846344709396, 0.0017131055938079953, 0.0, 0.02670523151755333, 0.0, 0.0, 0.08158159255981445, 0.03162313997745514, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.08094938099384308, 0.03709215670824051, 0.0, 0.06029241532087326, 0.0, 0.0, 0.0, 0.0, 0.0, 0.07558602094650269, 0.3259393572807312, 0.660822868347168, 0.37009137868881226, 0.3425831198692322, 0.053378425538539886, 0.0, 0.0, 0.004151645116508007, 0.0, 0.0, 0.002604310167953372, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.43216457962989807, 0.6726109981536865, 0.0, 0.0277737844735384, 0.07425198704004288, 0.07437944412231445, 0.6525806784629822, 0.022986555472016335, 0.0, 0.0, 0.06828376650810242, 0.06397975981235504, 0.0, 0.0, 0.3371535837650299, 0.32663694024086, 0.18514478206634521, 0.11569849401712418, 0.02221236750483513, 0.0, 0.05864676088094711, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.01950724609196186, 0.0, 0.0, 0.0, 0.28335389494895935, 0.5443544983863831, 0.0, 0.03947078436613083, 0.0, 0.04972367733716965, 0.7106717228889465, 0.37577125430107117, 0.32217931747436523, 0.020793801173567772, 0.03219148889183998, 0.0, 0.05128146708011627, 0.32580557465553284, 0.47289982438087463, 0.10473611205816269, 0.0, 0.0, 0.09709656238555908, 0.0, 0.006715112365782261, 0.006354608107358217, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0010489084525033832, 0.04074496403336525, 0.0, 0.022651053965091705, 0.0, 0.026429861783981323, 0.02550433948636055, 0.5184130072593689, 0.7425103783607483, 0.1769542247056961, 0.24885022640228271, 0.06373697519302368, 0.0, 0.0, 0.0, 0.00827841181308031, 0.031020546332001686, 0.07424837350845337, 0.7593725323677063, 0.20671811699867249, 0.0, 0.24156808853149414, 0.2473987191915512, 0.4428512752056122, 0.3683472275733948, 0.054656147956848145, 0.14780379831790924, 0.04847564175724983, 0.0071441400796175, 0.29853153228759766, 0.5506295561790466, 0.0, 0.6164371371269226, 0.8373505473136902, 0.5208625197410583, 0.4983071982860565, 0.13341809809207916, 0.059950653463602066, 0.4517427086830139, 0.15175718069076538, 0.09950340539216995, 0.31660977005958557, 0.05775389447808266, 0.2739335000514984, 0.03452448919415474, 0.0, 0.2435479611158371, 0.04300575330853462, 0.18787844479084015, 0.005434165708720684, 0.0, 0.055908143520355225, 0.005754332523792982, 0.05799860507249832, 0.0, 0.0, 0.07865896075963974, 0.055687692016363144, 0.0, 0.0, 0.0, 0.10726498812437057, 0.4792323410511017, 0.6364669799804688, 0.7491480112075806, 0.21036705374717712, 0.0, 0.09590161591768265, 0.1980905383825302, 0.0, 0.017664313316345215, 0.0, 0.07394067198038101, 0.2315947264432907, 0.27352675795555115, 0.0, 0.11496303230524063, 0.0, 0.0, 0.0, 0.044016283005476, 0.002916201250627637, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0025032164994627237, 0.011331542395055294, 0.21472758054733276, 0.0, 0.0, 0.0, 0.016233904287219048, 0.0441899411380291, 0.0, 0.0, 0.0, 0.0, 0.0, 0.008451979607343674, 0.029438888654112816, 0.0, 0.02834969200193882, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.016491152346134186, 0.03790609538555145, 0.02865949273109436, 0.158051535487175, 0.10146709531545639, 0.036274202167987823, 0.333109587430954, 0.10472646355628967, 0.07366599142551422, 0.0, 0.0, 0.0, 0.014878595247864723, 0.14340005815029144, 0.0, 0.0, 0.0, 0.007576132193207741, 0.0, 0.0, 0.0, 0.0, 0.0064433119259774685, 0.26057395339012146, 0.024025660008192062, 0.0, 0.02303703874349594, 0.012314673513174057, 0.0, 0.00962237548083067, 0.0, 0.0, 0.0, 0.0, 0.0007079701754264534, 0.0, 0.0, 0.0, 0.012906430289149284, 0.006400765851140022, 0.00707369577139616, 0.1835324615240097, 0.004328847862780094, 0.0, 0.030127445235848427, 0.005872257985174656, 0.0, 0.009544468484818935, 0.0, 0.0, 0.0, 0.0, 0.03552456572651863, 0.012827151454985142, 0.03150787204504013, 0.03547972813248634, 0.26352620124816895, 0.014716320671141148, 0.0, 0.0680156797170639, 0.0, 0.06265317648649216, 0.0, 0.0, 0.0, 0.0, 0.003745229681953788, 0.0, 0.0, 0.0, 0.010989446192979813, 0.1403169482946396, 0.0, 0.0, 0.0, 0.009509735740721226, 0.04449267312884331, 0.014522278681397438, 0.0, 0.0, 3.810287671512924e-05, 0.0, 0.0, 0.0182472113519907, 0.01988537795841694, 0.2725265622138977, 0.009080074727535248, 0.0, 0.0, 0.0, 0.025050239637494087, 0.0, 0.0, 0.005396793130785227, 0.0, 0.02130143903195858, 0.0, 0.0, 0.0, 0.0, 0.12292271107435226, 0.09808433800935745, 0.023867344483733177, 0.05443769320845604, 0.3311344087123871, 0.07640819251537323, 0.09269354492425919, 0.0, 0.0, 0.0, 0.0, 0.004920582752674818, 0.015849605202674866, 0.010995996184647083, 0.022414622828364372, 0.0, 0.05818759649991989, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.037683673202991486, 0.0, 0.0, 0.01591677777469158, 0.0020323365461081266, 0.0, 0.007736402563750744, 0.005832817405462265, 0.038744281977415085, 0.0, 0.0, 0.035138435661792755, 0.0, 0.09848365187644958, 0.02122596837580204, 0.0, 0.04346488416194916, 0.0, 0.0, 0.0, 0.0, 0.037471216171979904, 0.0, 0.0, 0.0, 0.0051619550213217735, 0.0, 0.0, 0.013098673895001411, 0.0, 0.0, 0.0, 0.09723178297281265, 0.0023838700726628304, 0.0, 0.0450218990445137, 0.0, 0.008478470146656036, 0.111615389585495, 0.026337958872318268, 0.0, 0.10424882918596268, 0.0, 0.021953828632831573, 0.0, 0.03606140986084938, 0.0031339298002421856, 0.0, 0.0], [0.0, 0.0, 0.005199238657951355, 0.0, 0.0039032725617289543, 0.0, 0.0, 0.0, 0.016187041997909546, 0.02274908684194088, 0.019795268774032593, 0.005805880296975374, 0.21173237264156342, 0.040341977030038834, 0.0, 0.001417560619302094, 0.008313218131661415, 0.0022880209144204855, 0.0, 0.007025647442787886, 0.0, 0.0, 0.020197246223688126, 0.0, 0.008322058245539665, 0.24448737502098083, 0.031023086979985237, 0.014067532494664192, 0.006966469809412956, 0.012310696765780449, 0.0, 0.0, 0.0, 0.0, 0.0, 0.22131408751010895, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.006090117618441582, 0.0011109282495453954, 0.0, 0.0028718928806483746, 0.0, 0.0, 0.011846971698105335, 0.0, 0.0, 0.006767247803509235, 0.05538985878229141, 0.0, 0.0, 0.0, 0.0004262102011125535, 0.2499270886182785, 0.02472318895161152, 0.001479682861827314, 0.0004139249213039875, 0.010055575519800186, 0.0, 0.0, 0.0, 0.0, 0.0799146369099617, 0.006011414807289839, 0.0, 0.0, 0.0, 0.0012261895462870598, 0.0167545173317194, 0.0005375862237997353, 0.1532091498374939, 0.0011140145361423492, 0.00618448993191123, 0.0, 0.003752264194190502, 0.0, 0.0, 0.017298346385359764, 0.0, 0.017663205042481422, 0.0, 0.0013447938254103065, 0.0, 0.0, 0.0, 0.0, 0.22654275596141815, 0.008844045922160149, 0.0, 0.0, 0.0, 0.0, 0.0, 0.00176666968036443, 0.08207841217517853, 0.03174319490790367, 0.004086172673851252, 0.0, 0.0005936476518400013, 0.011770609766244888, 0.0013686013408005238, 0.0134224658831954, 0.002090823370963335, 0.0007396757719106972, 0.0, 0.0, 0.20960146188735962, 0.021672014147043228, 0.031036365777254105, 0.0374111570417881, 0.0024692893493920565, 0.008614185266196728, 0.0, 0.02055356465280056, 0.22338137030601501, 0.0008468886371701956, 0.0, 0.0, 0.0007108075660653412, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.2394397109746933, 0.004331694450229406, 0.0, 0.008779685944318771, 0.0, 0.005575021728873253, 0.009278066456317902, 0.0, 0.0, 0.05270109698176384, 0.0, 0.0, 0.03379342705011368, 0.035908136516809464, 0.0, 0.026347629725933075, 0.0, 0.0, 0.007270431146025658, 0.0, 0.0, 0.008582968264818192, 0.0, 0.20927204191684723, 0.012970222160220146, 0.0, 0.019389787688851357, 0.0, 0.0, 0.026799146085977554, 0.00045609023072756827, 0.028413765132427216, 0.0, 0.0, 0.02271570824086666, 0.00010043437214335427, 0.003431492717936635, 0.0, 0.2178056389093399, 0.06275495141744614, 0.035486187785863876, 0.0, 0.0, 0.010464022867381573, 0.202194482088089, 0.006523216608911753, 0.00029837561305612326, 0.0, 0.0, 0.0038586631417274475, 0.0, 0.0, 0.0, 0.07607459276914597, 0.0, 0.0, 0.049299269914627075, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.05027700588107109, 0.0, 0.004612948279827833], [0.0, 0.0, 0.16077156364917755, 0.10176524519920349, 0.07650768756866455, 0.0, 0.0, 0.0, 0.018895508721470833, 0.0, 0.0, 0.0, 0.0, 0.030758263543248177, 0.256217360496521, 0.0, 0.05287139490246773, 0.0, 0.0, 0.0, 0.0, 0.0, 0.017672576010227203, 0.25564131140708923, 0.034919776022434235, 0.0, 0.21024364233016968, 0.20180585980415344, 0.1808871328830719, 0.0555698424577713, 0.013172779232263565, 0.16205525398254395, 0.0, 0.0, 0.00965844001621008, 0.0, 0.05810125172138214, 0.03562942147254944, 0.1426788866519928, 0.07931430637836456, 0.30337271094322205, 0.3307613730430603, 0.3119403123855591, 0.306816965341568, 0.001923835021443665, 0.0, 0.005912443622946739, 0.24300915002822876, 0.12053005397319794, 0.0, 0.0, 0.0, 0.060262784361839294, 0.0, 0.03525910526514053, 0.016204793006181717, 0.0, 0.0, 0.2985732853412628, 0.15141867101192474, 0.16308407485485077, 0.0, 0.0, 0.25778716802597046, 0.27733883261680603, 0.22599220275878906, 0.10460269451141357, 0.0, 0.017918433994054794, 0.07434433698654175, 0.0035946310963481665, 0.17425313591957092, 0.2031206339597702, 0.01263013482093811, 0.10128078609704971, 0.0, 0.0, 0.2665615975856781, 0.10579787939786911, 0.22991159558296204, 0.2616924047470093, 0.15338630974292755, 0.0, 0.0, 0.0, 0.01709417626261711, 0.0, 0.018121497705578804, 0.04939374700188637, 0.046317316591739655, 0.00903680082410574, 0.22358672320842743, 0.041929904371500015, 0.11774744838476181, 0.0, 0.0, 0.11847468465566635, 0.05816319212317467, 0.1983155459165573, 0.28167903423309326, 0.33175423741340637, 0.2726210653781891, 0.029267897829413414, 0.060095466673374176, 0.20157746970653534, 0.2961629331111908, 0.005250868387520313, 0.003971881698817015, 0.3220447897911072, 0.2675456404685974, 0.20735281705856323, 0.0, 0.0, 0.0, 0.0, 0.0, 0.031045010313391685, 0.047587502747774124, 0.2735513746738434, 0.2536011338233948, 0.33310991525650024, 0.24780498445034027, 0.0, 0.020275291055440903, 0.0, 0.1772930920124054, 0.06022779271006584, 0.011190321296453476, 0.16417057812213898, 0.14902876317501068, 0.0, 0.13095517456531525, 0.06732460856437683, 0.0, 0.0, 0.10691892355680466, 0.0, 0.0, 0.011800134554505348, 0.013980558142066002, 0.06715578585863113, 0.2888597548007965, 0.06645012646913528, 0.24526210129261017, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.012836265377700329, 0.010471758432686329, 0.0, 0.04121893271803856, 0.04075305908918381, 0.17004746198654175, 0.0, 0.06313806772232056, 0.043604299426078796, 0.00466138543561101, 0.0, 0.05241738259792328, 0.03317729011178017, 0.1527978628873825, 0.005952303763478994, 0.0, 0.0, 0.028291849419474602, 0.0, 0.0, 0.0, 0.0, 0.02950475737452507, 0.061299800872802734, 0.19123071432113647, 0.3072957992553711, 0.2863091826438904, 0.0, 0.034140851348638535, 0.0, 0.0, 0.08087828755378723, 0.0, 0.0, 0.025563005357980728, 0.22016295790672302, 0.12744417786598206, 0.3017180860042572, 0.21234376728534698, 0.008609162643551826, 0.06189293786883354, 0.018422860652208328, 0.006075313314795494, 0.0017616236582398415, 0.013322355225682259, 0.019831715151667595, 0.01068040356040001, 0.0, 0.0, 0.0], [0.0, 0.4246164560317993, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.023940986022353172, 0.0, 0.2668117880821228, 0.0, 0.0, 0.0, 0.0, 0.09723255783319473, 0.0, 0.0, 0.0322192944586277, 0.0, 1.5454226732254028, 0.161586195230484, 0.012984762899577618, 0.10073897242546082, 0.0, 0.0, 0.031409792602062225, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.10876070708036423, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.4648895263671875, 0.1401567906141281, 0.0, 0.0881643071770668, 0.0, 0.0055723609402775764, 0.0, 0.2465987205505371, 0.010076016187667847, 0.022834019735455513, 0.0675756186246872, 0.04263310879468918, 0.0, 0.014119052328169346, 0.1632159948348999, 0.005675566382706165, 0.0, 0.0, 0.08210243284702301, 0.0, 0.095090351998806, 0.1133197546005249, 0.37753552198410034, 1.339428186416626, 0.5286679863929749, 0.10502006858587265, 0.06389807909727097, 0.004833540879189968, 0.044542815536260605, 0.05284348130226135, 0.024908538907766342, 0.11894874274730682, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.035328980535268784, 0.2666192352771759, 0.08807993680238724, 0.24197611212730408, 1.2027782201766968, 0.5571668148040771, 0.01274817530065775, 0.0, 0.0, 0.02140389196574688, 0.29428425431251526, 0.07679411768913269, 0.0, 0.0, 0.02033868618309498, 0.0, 0.17188797891139984, 0.01693657413125038, 0.0011868256842717528, 0.15166929364204407, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.21219046413898468, 1.6120991706848145, 0.597234845161438, 0.0, 0.08550742268562317, 0.006870606914162636, 0.0, 0.0, 0.017929673194885254, 0.04935111105442047, 0.0, 0.006081726867705584, 0.0, 0.0, 0.14314289391040802, 0.0, 0.0, 0.04973752796649933, 0.0, 0.0, 0.04435635358095169, 0.1963600069284439, 1.8516850471496582, 0.5148048400878906, 0.1017863005399704, 0.04715844988822937, 0.00229961727745831, 0.0, 0.08289571106433868, 0.0, 0.0, 0.11993616074323654, 7.77364220994059e-06, 0.0, 0.0, 0.0, 0.0, 0.0, 0.008113019168376923, 0.032326821237802505, 0.0, 0.0008641811436973512, 0.0, 0.1255025714635849, 0.17250283062458038, 0.2390543818473816, 1.3153836727142334, 0.4893098473548889, 0.199379563331604, 0.23688706755638123, 0.09659919142723083, 0.0, 0.0, 0.06583506613969803, 0.0388512946665287, 0.21692384779453278, 0.0, 0.08278844505548477, 0.2395644336938858, 0.0, 0.2213844656944275, 0.0, 0.0, 0.01720220409333706, 0.04479086026549339, 0.0, 0.0, 0.213208869099617, 1.9187411069869995, 0.5953587889671326, 0.05226815491914749, 0.22062772512435913, 0.13569588959217072, 0.03827515244483948, 0.017735591158270836, 0.04074913263320923, 0.08344028890132904, 1.4114515781402588, 0.37919262051582336, 1.5210884809494019, 0.45701760053634644, 1.421481966972351, 0.25954511761665344, 0.8294389247894287, 1.3870704174041748, 0.7933921217918396], [0.0, 0.1537497192621231, 0.02668178640305996, 0.0054672472178936005, 0.0, 0.023323647677898407, 0.20004990696907043, 0.0, 0.0, 0.0, 0.0, 0.04593272879719734, 0.006264425814151764, 0.1560000777244568, 0.0009902138262987137, 0.0, 0.0, 0.17831416428089142, 0.0, 0.0, 0.0, 0.022328678518533707, 0.15667757391929626, 0.01309029571712017, 0.006226259283721447, 0.010986500419676304, 0.0, 0.0, 0.0, 0.021482503041625023, 0.19136233627796173, 0.0, 0.0, 0.1832013726234436, 0.0, 0.0, 0.0, 0.04740500450134277, 0.0, 0.0, 0.00479890126734972, 0.0, 0.0, 0.0, 0.0, 0.025373738259077072, 0.20853886008262634, 0.02969255857169628, 0.0, 0.018723132088780403, 0.20478957891464233, 0.0, 0.003421006491407752, 0.2007034420967102, 0.0012191724963486195, 0.0, 0.009461592882871628, 0.050339099019765854, 0.0016489026602357626, 0.0, 0.0, 0.0, 0.037553559988737106, 0.005791469477117062, 0.0, 0.006141477730125189, 0.0, 0.0, 0.020635802298784256, 0.0, 0.0, 0.0, 0.0, 0.20542660355567932, 0.01761479303240776, 0.005262976046651602, 0.2222416251897812, 0.023044724017381668, 0.006128990091383457, 0.007905373349785805, 0.0167441014200449, 0.0, 0.034913673996925354, 0.22871309518814087, 0.0032337759621441364, 0.12596702575683594, 0.0, 0.11302359402179718, 0.0, 0.0, 0.04670475050806999, 0.0, 0.007010330446064472, 0.0, 0.0, 0.04048396274447441, 0.0, 0.0, 7.400931644951925e-05, 0.0008990043425001204, 0.0, 0.0, 0.0, 0.007792077027261257, 0.0, 0.0, 0.0028148714918643236, 0.20057323575019836, 0.0008791894069872797, 0.01726383902132511, 0.01278148777782917, 0.020581422373652458, 0.2319851964712143, 0.0, 0.0, 0.0, 0.012544012628495693, 0.1569535732269287, 0.0, 0.0009724608389660716, 0.0, 0.007006318774074316, 0.0, 0.0, 0.01902312971651554, 0.0, 0.0, 0.17403081059455872, 0.0, 0.0, 0.0, 0.0016142396489158273, 0.0, 0.0, 0.0036678193137049675, 0.0, 0.0, 0.0, 0.031933002173900604, 0.0, 0.0, 0.0, 0.010829517617821693, 0.0, 0.0, 0.07356424629688263, 0.0, 0.0, 0.0, 0.0068755862303078175, 0.0, 0.0, 0.19617746770381927, 0.008794909343123436, 0.0, 0.0, 0.0, 0.0, 0.004138386808335781, 0.005686957389116287, 0.0027483783196657896, 0.0, 0.009160272777080536, 0.0, 0.0, 0.20172704756259918, 0.004471467807888985, 0.008512812666594982, 0.0013000466860830784, 0.20816701650619507, 0.008975288830697536, 0.015479795634746552, 0.04195559024810791, 0.17965281009674072, 0.013094885274767876, 0.0, 0.002450095023959875, 0.005255723372101784, 0.0, 0.0, 0.006859437562525272, 0.0, 0.0, 0.0, 0.17602001130580902, 0.0023948708549141884, 0.008800208568572998, 0.0, 0.0018653739243745804, 0.0019415776478126645, 0.0, 0.0, 0.01610032469034195, 0.0, 0.07107516378164291, 0.0, 0.0, 0.015529969707131386, 0.006945841480046511, 0.02403203584253788], [0.0, 0.03858160600066185, 0.39768186211586, 0.6561291813850403, 0.6665815711021423, 0.2633908987045288, 0.3973658084869385, 1.0896811485290527, 0.6515664458274841, 0.658297598361969, 0.6793559193611145, 0.4235447645187378, 0.08241816610097885, 0.15064316987991333, 0.23985472321510315, 0.9957021474838257, 0.4757479727268219, 0.33122557401657104, 1.0030652284622192, 0.9393747448921204, 0.021279143169522285, 0.0, 0.021225055679678917, 0.04923933371901512, 0.688789963722229, 0.009336226619780064, 0.0, 0.0, 0.1433599293231964, 0.04277746379375458, 0.0, 0.3175475597381592, 0.0, 0.15421611070632935, 0.8513637781143188, 0.07391317188739777, 0.14001208543777466, 0.0, 0.13777539134025574, 0.0, 0.022775180637836456, 0.0, 0.0, 0.0390905998647213, 0.011557277292013168, 0.0, 0.0, 0.08157287538051605, 0.4207363426685333, 0.02220829762518406, 0.12431133538484573, 1.0008294582366943, 0.33478888869285583, 0.17924827337265015, 0.8131757378578186, 0.7740539908409119, 0.04510651156306267, 0.0, 0.0, 0.2745705246925354, 0.38132116198539734, 0.0, 0.0, 0.0, 0.0, 0.0, 0.3744315803050995, 0.0, 0.0, 0.23386476933956146, 0.0, 0.007529302034527063, 0.05322844535112381, 0.0, 0.4633949398994446, 0.0, 0.0, 0.025769704952836037, 0.0, 0.15900801122188568, 0.01689174212515354, 0.43940412998199463, 0.0, 0.22926536202430725, 1.0400903224945068, 0.12727558612823486, 0.0, 0.0930902361869812, 0.7414314150810242, 0.10272110253572464, 0.0, 0.0, 0.0, 0.08260796219110489, 0.0, 0.0, 0.06741897761821747, 0.0, 0.005803607404232025, 0.0, 0.0, 0.0, 0.0, 0.0, 0.005226875655353069, 0.0, 0.0, 0.0, 0.0, 0.0, 0.23896092176437378, 0.0, 0.18782877922058105, 1.0452690124511719, 0.0, 0.0, 0.0060064103454351425, 0.0, 0.0, 0.0, 0.013835513032972813, 0.04820146784186363, 0.0, 0.15920960903167725, 0.0, 0.16189150512218475, 0.0, 0.0, 0.14596253633499146, 0.3005443513393402, 0.0, 0.2927682101726532, 0.4843810796737671, 0.0, 0.0, 0.2151849865913391, 0.0, 0.12447194755077362, 0.21137620508670807, 0.11978907883167267, 0.09379778057336807, 0.03807484731078148, 0.0, 0.0957813560962677, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.3334900438785553, 0.09899771958589554, 0.03757210448384285, 0.6538937091827393, 0.07736511528491974, 0.17193153500556946, 0.0, 0.08346977829933167, 0.43610328435897827, 0.07443397492170334, 0.010037125088274479, 0.3210291862487793, 0.0, 0.12866275012493134, 0.032939255237579346, 0.1742139458656311, 0.9247562289237976, 0.36441105604171753, 0.0, 0.0, 0.966672420501709, 0.02128838561475277, 0.0, 0.0, 0.08251120150089264, 0.0, 0.05694828927516937, 0.024189339950680733, 0.1262609213590622, 0.0, 0.0, 0.11511299759149551, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.16413985192775726, 0.03918224200606346, 0.2680206894874573, 0.027308732271194458, 0.127594456076622, 0.0, 0.13365674018859863, 0.0, 0.594844400882721, 0.01980706863105297, 0.0, 0.031531136482954025], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0724114254117012, 0.0, 0.028376024216413498, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0027906778268516064, 0.0, 0.0, 0.03375750035047531, 0.0, 0.0, 0.0, 0.0035887276753783226, 0.043657947331666946, 0.0010883315699175, 0.0, 0.0, 0.0, 0.0, 0.0, 0.00809242483228445, 0.0, 0.0, 0.0, 0.0, 0.0, 0.011908135376870632, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.011917009949684143, 0.0, 0.0, 0.003033831249922514, 0.0, 0.0, 0.0, 0.0033862567506730556, 0.03116469457745552, 0.007833329029381275, 0.0, 0.0, 0.0, 0.0, 0.0, 0.007253776304423809, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.04820142686367035, 0.012643156573176384, 0.0100115817040205, 0.0, 0.0, 0.0, 0.007970125414431095, 0.1479167342185974, 0.9815592765808105, 0.15720586478710175, 0.0011571022914722562, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.007530855014920235, 0.0008834878681227565, 0.0, 0.010451375506818295, 0.017471129074692726, 0.0, 0.0, 0.0, 0.0, 0.021957295015454292, 0.0038720951415598392, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.01159222237765789, 0.037771232426166534, 0.006681761704385281, 0.0027556309942156076, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.040221139788627625, 0.011873886920511723, 0.0, 0.0, 0.04297895357012749, 0.0, 0.010754534043371677, 0.07239188998937607, 0.15496110916137695, 0.9859082102775574, 0.11547122895717621, 0.0, 0.0, 0.0, 0.004117783159017563, 0.01852729544043541, 0.025513680651783943, 0.0, 0.0, 0.0, 0.0, 0.002207912504673004, 0.0, 0.0, 0.002684050938114524, 0.0, 0.0043909745290875435, 0.0, 0.0018229279667139053, 0.05326513946056366, 0.0, 0.0, 0.0007718947599641979, 0.007007696200162172, 0.01155481394380331, 0.06848929077386856, 0.013932831585407257, 0.0047070337459445, 0.006011100020259619, 0.006770594976842403, 0.02803964726626873, 0.026892978698015213, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.04400228336453438, 0.0, 0.0, 0.01522895973175764, 0.0, 0.13635151088237762, 0.0, 0.1368221789598465, 0.04120657965540886, 0.14833667874336243, 0.0, 0.019060183316469193, 0.0, 0.08265969157218933]]]},\n            {\n            'hltrCFG': {'tokenization_config': {\"token_prefix\": \"\\u0120\", \"partial_token_prefix\": \"\"}\n                }\n            })\n         }, function (err) {\n            console.log(err);\n        })"},"metadata":{}}]},{"cell_type":"markdown","source":"> - Above visualization shows how a pretrained RoBERTa model's prediction activated by input sentence tokens using dimention reduction.","metadata":{}},{"cell_type":"code","source":"from bertviz.transformers_neuron_view import RobertaModel, RobertaTokenizer\nfrom bertviz.neuron_view import show\nmodel_type = 'roberta'\nmodel_version = 'roberta-base'\nmodel = RobertaModel.from_pretrained(model_version)\ntokenizer = RobertaTokenizer.from_pretrained(model_version)\nsentence_a = \"I hope for a better future\"\nsentence_b = \"Oh!! GOD! You should always obey me!!\"\nmodel_D_neuron_attention = show(model, model_type, tokenizer, sentence_a, sentence_b, layer=4, head=8,html_action='return')\nwith open('model_D_neuron_attention.html', 'w') as file:\n    file.write(model_D_neuron_attention.data)\nhtml_file_path = ['model_D_neuron_attention.html']","metadata":{"execution":{"iopub.status.busy":"2023-05-31T22:22:54.588732Z","iopub.execute_input":"2023-05-31T22:22:54.589681Z","iopub.status.idle":"2023-05-31T22:23:00.391088Z","shell.execute_reply.started":"2023-05-31T22:22:54.589633Z","shell.execute_reply":"2023-05-31T22:23:00.389807Z"},"trusted":true},"execution_count":9,"outputs":[]},{"cell_type":"code","source":"from IPython.display import IFrame\ndisplay(IFrame(src=html_file_path[0], width='100%', height='800px'))","metadata":{"execution":{"iopub.status.busy":"2023-05-31T22:23:02.624687Z","iopub.execute_input":"2023-05-31T22:23:02.625055Z","iopub.status.idle":"2023-05-31T22:23:02.631356Z","shell.execute_reply.started":"2023-05-31T22:23:02.625025Z","shell.execute_reply":"2023-05-31T22:23:02.630472Z"},"trusted":true},"execution_count":10,"outputs":[{"output_type":"display_data","data":{"text/plain":"<IPython.lib.display.IFrame at 0x7db008434a30>","text/html":"\n        <iframe\n            width=\"100%\"\n            height=\"800px\"\n            src=\"model_D_neuron_attention.html\"\n            frameborder=\"0\"\n            allowfullscreen\n            \n        ></iframe>\n        "},"metadata":{}}]},{"cell_type":"markdown","source":"## Our Finetuned Model D Activations Factorization:","metadata":{}},{"cell_type":"code","source":"from transformers import AutoTokenizer, RobertaForSequenceClassification\nmodel_D = RobertaForSequenceClassification.from_pretrained(\"/kaggle/input/model-new-up/MODELs/MODEL_D\",output_attentions=True)\ntokenizer_D = AutoTokenizer.from_pretrained(\"/kaggle/input/model-new-up/MODELs/MODEL_D\")\nmodel_D","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-31T22:26:14.322094Z","iopub.execute_input":"2023-05-31T22:26:14.322493Z","iopub.status.idle":"2023-05-31T22:26:20.315244Z","shell.execute_reply.started":"2023-05-31T22:26:14.322462Z","shell.execute_reply":"2023-05-31T22:26:20.314319Z"},"trusted":true},"execution_count":13,"outputs":[{"execution_count":13,"output_type":"execute_result","data":{"text/plain":"RobertaForSequenceClassification(\n  (roberta): RobertaModel(\n    (embeddings): RobertaEmbeddings(\n      (word_embeddings): Embedding(50266, 768, padding_idx=1)\n      (position_embeddings): Embedding(514, 768, padding_idx=1)\n      (token_type_embeddings): Embedding(1, 768)\n      (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n      (dropout): Dropout(p=0.1, inplace=False)\n    )\n    (encoder): RobertaEncoder(\n      (layer): ModuleList(\n        (0-11): 12 x RobertaLayer(\n          (attention): RobertaAttention(\n            (self): RobertaSelfAttention(\n              (query): Linear(in_features=768, out_features=768, bias=True)\n              (key): Linear(in_features=768, out_features=768, bias=True)\n              (value): Linear(in_features=768, out_features=768, bias=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n            (output): RobertaSelfOutput(\n              (dense): Linear(in_features=768, out_features=768, bias=True)\n              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n          )\n          (intermediate): RobertaIntermediate(\n            (dense): Linear(in_features=768, out_features=3072, bias=True)\n            (intermediate_act_fn): GELUActivation()\n          )\n          (output): RobertaOutput(\n            (dense): Linear(in_features=3072, out_features=768, bias=True)\n            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n  )\n  (classifier): RobertaClassificationHead(\n    (dense): Linear(in_features=768, out_features=768, bias=True)\n    (dropout): Dropout(p=0.1, inplace=False)\n    (out_proj): Linear(in_features=768, out_features=1, bias=True)\n  )\n)"},"metadata":{}}]},{"cell_type":"code","source":"model_config_D = {\n    'embedding': \"embeddings.word_embeddings\",\n    'type': 'mlm',\n    'activations':['\\d+\\.output\\.dense'],\n    'token_prefix': 'Ġ',\n    'partial_token_prefix': '##'\n}\nD_model_id = \"/kaggle/input/model-new-up/MODELs/MODEL_D\"","metadata":{"execution":{"iopub.status.busy":"2023-05-31T18:45:35.546494Z","iopub.execute_input":"2023-05-31T18:45:35.546897Z","iopub.status.idle":"2023-05-31T18:45:35.552837Z","shell.execute_reply.started":"2023-05-31T18:45:35.546858Z","shell.execute_reply":"2023-05-31T18:45:35.551700Z"},"trusted":true},"execution_count":71,"outputs":[]},{"cell_type":"code","source":"ecco_model = ecco.from_pretrained(D_model_id, \n                                  activations=True,\n                                  model_config=model_config_D)","metadata":{"execution":{"iopub.status.busy":"2023-05-31T18:45:35.554373Z","iopub.execute_input":"2023-05-31T18:45:35.554908Z","iopub.status.idle":"2023-05-31T18:45:37.582144Z","shell.execute_reply.started":"2023-05-31T18:45:35.554861Z","shell.execute_reply":"2023-05-31T18:45:37.581004Z"},"_kg_hide-output":true,"trusted":true},"execution_count":72,"outputs":[{"name":"stderr","text":"Some weights of the model checkpoint at /kaggle/input/model-new-up/MODELs/MODEL_D were not used when initializing RobertaModel: ['classifier.dense.weight', 'classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight']\n- This IS expected if you are initializing RobertaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n- This IS NOT expected if you are initializing RobertaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\nSome weights of RobertaModel were not initialized from the model checkpoint at /kaggle/input/model-new-up/MODELs/MODEL_D and are newly initialized: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight']\nYou should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n","output_type":"stream"}]},{"cell_type":"markdown","source":"### Factorizing Activations of a All Layer Layers","metadata":{}},{"cell_type":"code","source":"# Factorize the activations of all layers\ninputs = ecco_model.tokenizer([text], return_tensors=\"pt\")\noutput = ecco_model(inputs)\nnmf_1 = output.run_nmf(n_components=10)                           #non-negative matrix factorization on 10 components\nnmf_1.explore()","metadata":{"execution":{"iopub.status.busy":"2023-05-31T18:45:37.583873Z","iopub.execute_input":"2023-05-31T18:45:37.584350Z","iopub.status.idle":"2023-05-31T18:45:48.348030Z","shell.execute_reply.started":"2023-05-31T18:45:37.584307Z","shell.execute_reply":"2023-05-31T18:45:48.346943Z"},"trusted":true},"execution_count":73,"outputs":[{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"<html lang=\"en\">\n<script src=\"https://requirejs.org/docs/release/2.3.6/minified/require.js\"></script>\n<script>\n    var ecco_url = 'https://storage.googleapis.com/ml-intro/ecco/'\n    //var ecco_url = 'http://localhost:8000/'\n\n    if (window.ecco === undefined) window.ecco = {}\n\n    // Setup the paths of the script we'll be using\n    requirejs.config({\n        urlArgs: \"bust=\" + (new Date()).getTime(),\n        nodeRequire: require,\n        paths: {\n            d3: \"https://d3js.org/d3.v6.min\", // This is only for use in setup.html and basic.html\n            \"d3-array\": \"https://d3js.org/d3-array.v2.min\",\n            jquery: \"https://code.jquery.com/jquery-3.5.1.min\",\n            ecco: ecco_url + 'js/0.0.6/ecco-bundle.min',\n            xregexp: 'https://cdnjs.cloudflare.com/ajax/libs/xregexp/3.2.0/xregexp-all.min'\n        }\n    });\n\n    // Add the css file\n    //requirejs(['d3'],\n    //    function (d3) {\n    //        d3.select('#css').attr('href', ecco_url + 'html/styles.css')\n    //    })\n\n    console.log('Ecco initialize!!')\n\n    // returns a 'basic' object. basic.init() selects the html div we'll be\n    // rendering the html into, adds styles.css to the document.\n    define('basic', ['d3'],\n        function (d3) {\n            return {\n                init: function (viz_id = null) {\n                    if (viz_id == null) {\n                        viz_id = \"viz_\" + Math.round(Math.random() * 10000000)\n                    }\n                    // Select the div rendered below, change its id\n                    const div = d3.select('#basic').attr('id', viz_id),\n                        div_parent = d3.select('#' + viz_id).node().parentNode\n\n                    // Link to CSS file\n                    d3.select(div_parent).insert('link')\n                        .attr('rel', 'stylesheet')\n                        .attr('type', 'text/css')\n                        .attr('href', ecco_url + 'html/0.0.2/styles.css')\n\n                    return viz_id\n                }\n            }\n        }, function (err) {\n            console.log(err);\n        }\n    )\n</script>\n\n<head>\n    <link id='css' rel=\"stylesheet\" type=\"text/css\">\n</head>\n<div id=\"basic\"></div>\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.Javascript object>","application/javascript":"\n         requirejs(['basic', 'ecco'], function(basic, ecco){\n            const viz_id = basic.init()\n            \n            ecco.interactiveTokensAndFactorSparklines(viz_id, {'tokens': [{'token': '<s>', 'token_id': 0, 'type': 'input', 'position': 0}, {'token': 'ĠThe', 'token_id': 20, 'type': 'input', 'position': 1}, {'token': 'Ġvast', 'token_id': 4714, 'type': 'input', 'position': 2}, {'token': 'Ġexp', 'token_id': 4553, 'type': 'input', 'position': 3}, {'token': 'anse', 'token_id': 31327, 'type': 'input', 'position': 4}, {'token': 'Ġof', 'token_id': 9, 'type': 'input', 'position': 5}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 6}, {'token': 'Ġocean', 'token_id': 6444, 'type': 'input', 'position': 7}, {'token': 'Ġstretches', 'token_id': 15706, 'type': 'input', 'position': 8}, {'token': 'Ġout', 'token_id': 66, 'type': 'input', 'position': 9}, {'token': 'Ġbefore', 'token_id': 137, 'type': 'input', 'position': 10}, {'token': 'Ġme', 'token_id': 162, 'type': 'input', 'position': 11}, {'token': ',', 'token_id': 6, 'type': 'input', 'position': 12}, {'token': 'Ġits', 'token_id': 63, 'type': 'input', 'position': 13}, {'token': 'Ġcalm', 'token_id': 6327, 'type': 'input', 'position': 14}, {'token': 'Ġwaters', 'token_id': 5794, 'type': 'input', 'position': 15}, {'token': 'Ġreflecting', 'token_id': 10811, 'type': 'input', 'position': 16}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 17}, {'token': 'Ġsky', 'token_id': 6360, 'type': 'input', 'position': 18}, {'token': 'Ġabove', 'token_id': 1065, 'type': 'input', 'position': 19}, {'token': '.', 'token_id': 4, 'type': 'input', 'position': 20}, {'token': 'Ċ', 'token_id': 50118, 'type': 'input', 'position': 21}, {'token': 'Its', 'token_id': 30872, 'type': 'input', 'position': 22}, {'token': 'Ġtranquil', 'token_id': 33535, 'type': 'input', 'position': 23}, {'token': 'Ġsurface', 'token_id': 4084, 'type': 'input', 'position': 24}, {'token': ',', 'token_id': 6, 'type': 'input', 'position': 25}, {'token': 'Ġund', 'token_id': 2432, 'type': 'input', 'position': 26}, {'token': 'ist', 'token_id': 661, 'type': 'input', 'position': 27}, {'token': 'urbed', 'token_id': 35250, 'type': 'input', 'position': 28}, {'token': 'Ġby', 'token_id': 30, 'type': 'input', 'position': 29}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 30}, {'token': 'Ġchaos', 'token_id': 7724, 'type': 'input', 'position': 31}, {'token': 'Ġof', 'token_id': 9, 'type': 'input', 'position': 32}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 33}, {'token': 'Ġworld', 'token_id': 232, 'type': 'input', 'position': 34}, {'token': ',', 'token_id': 6, 'type': 'input', 'position': 35}, {'token': 'Ġinspires', 'token_id': 25362, 'type': 'input', 'position': 36}, {'token': 'Ġa', 'token_id': 10, 'type': 'input', 'position': 37}, {'token': 'Ġsense', 'token_id': 1472, 'type': 'input', 'position': 38}, {'token': 'Ġof', 'token_id': 9, 'type': 'input', 'position': 39}, {'token': 'Ġse', 'token_id': 842, 'type': 'input', 'position': 40}, {'token': 'rene', 'token_id': 20962, 'type': 'input', 'position': 41}, {'token': 'Ġtranqu', 'token_id': 36474, 'type': 'input', 'position': 42}, {'token': 'ility', 'token_id': 13005, 'type': 'input', 'position': 43}, {'token': '.', 'token_id': 4, 'type': 'input', 'position': 44}, {'token': 'Ċ', 'token_id': 50118, 'type': 'input', 'position': 45}, {'token': 'The', 'token_id': 133, 'type': 'input', 'position': 46}, {'token': 'Ġsoft', 'token_id': 3793, 'type': 'input', 'position': 47}, {'token': 'Ġwhisper', 'token_id': 37539, 'type': 'input', 'position': 48}, {'token': 'Ġof', 'token_id': 9, 'type': 'input', 'position': 49}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 50}, {'token': 'Ġwaves', 'token_id': 6995, 'type': 'input', 'position': 51}, {'token': 'Ġkissing', 'token_id': 20479, 'type': 'input', 'position': 52}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 53}, {'token': 'Ġshore', 'token_id': 8373, 'type': 'input', 'position': 54}, {'token': 'line', 'token_id': 1902, 'type': 'input', 'position': 55}, {'token': 'Ġis', 'token_id': 16, 'type': 'input', 'position': 56}, {'token': 'Ġa', 'token_id': 10, 'type': 'input', 'position': 57}, {'token': 'Ġpeaceful', 'token_id': 7053, 'type': 'input', 'position': 58}, {'token': 'Ġsym', 'token_id': 22850, 'type': 'input', 'position': 59}, {'token': 'phony', 'token_id': 29788, 'type': 'input', 'position': 60}, {'token': ',', 'token_id': 6, 'type': 'input', 'position': 61}, {'token': 'Ġa', 'token_id': 10, 'type': 'input', 'position': 62}, {'token': 'Ġmel', 'token_id': 15352, 'type': 'input', 'position': 63}, {'token': 'od', 'token_id': 1630, 'type': 'input', 'position': 64}, {'token': 'ious', 'token_id': 6514, 'type': 'input', 'position': 65}, {'token': 'Ġsong', 'token_id': 2214, 'type': 'input', 'position': 66}, {'token': 'Ġof', 'token_id': 9, 'type': 'input', 'position': 67}, {'token': 'Ċ', 'token_id': 50118, 'type': 'input', 'position': 68}, {'token': 'nature', 'token_id': 43102, 'type': 'input', 'position': 69}, {'token': 'Ġthat', 'token_id': 14, 'type': 'input', 'position': 70}, {'token': 'Ġso', 'token_id': 98, 'type': 'input', 'position': 71}, {'token': 'othes', 'token_id': 32915, 'type': 'input', 'position': 72}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 73}, {'token': 'Ġsoul', 'token_id': 7047, 'type': 'input', 'position': 74}, {'token': '.', 'token_id': 4, 'type': 'input', 'position': 75}, {'token': 'ĠThe', 'token_id': 20, 'type': 'input', 'position': 76}, {'token': 'Ġgentle', 'token_id': 16634, 'type': 'input', 'position': 77}, {'token': ',', 'token_id': 6, 'type': 'input', 'position': 78}, {'token': 'Ġrhyth', 'token_id': 41063, 'type': 'input', 'position': 79}, {'token': 'mic', 'token_id': 15796, 'type': 'input', 'position': 80}, {'token': 'Ġmotion', 'token_id': 4298, 'type': 'input', 'position': 81}, {'token': 'Ġof', 'token_id': 9, 'type': 'input', 'position': 82}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 83}, {'token': 'Ġocean', 'token_id': 6444, 'type': 'input', 'position': 84}, {'token': 'âĢ', 'token_id': 17, 'type': 'input', 'position': 85}, {'token': 'Ļ', 'token_id': 27, 'type': 'input', 'position': 86}, {'token': 's', 'token_id': 29, 'type': 'input', 'position': 87}, {'token': 'Ġcurrents', 'token_id': 28124, 'type': 'input', 'position': 88}, {'token': 'Ġembodies', 'token_id': 33434, 'type': 'input', 'position': 89}, {'token': 'Ġa', 'token_id': 10, 'type': 'input', 'position': 90}, {'token': 'Ġsilent', 'token_id': 8454, 'type': 'input', 'position': 91}, {'token': 'Ċ', 'token_id': 50118, 'type': 'input', 'position': 92}, {'token': 'strength', 'token_id': 33687, 'type': 'input', 'position': 93}, {'token': ',', 'token_id': 6, 'type': 'input', 'position': 94}, {'token': 'Ġa', 'token_id': 10, 'type': 'input', 'position': 95}, {'token': 'Ġsense', 'token_id': 1472, 'type': 'input', 'position': 96}, {'token': 'Ġof', 'token_id': 9, 'type': 'input', 'position': 97}, {'token': 'Ġenduring', 'token_id': 16480, 'type': 'input', 'position': 98}, {'token': 'Ġse', 'token_id': 842, 'type': 'input', 'position': 99}, {'token': 'ren', 'token_id': 2558, 'type': 'input', 'position': 100}, {'token': 'ity', 'token_id': 1571, 'type': 'input', 'position': 101}, {'token': 'Ġthat', 'token_id': 14, 'type': 'input', 'position': 102}, {'token': 'Ġis', 'token_id': 16, 'type': 'input', 'position': 103}, {'token': 'Ġprofoundly', 'token_id': 27301, 'type': 'input', 'position': 104}, {'token': 'Ġpeaceful', 'token_id': 7053, 'type': 'input', 'position': 105}, {'token': '.', 'token_id': 4, 'type': 'input', 'position': 106}, {'token': 'ĠThe', 'token_id': 20, 'type': 'input', 'position': 107}, {'token': 'Ġbound', 'token_id': 8191, 'type': 'input', 'position': 108}, {'token': 'less', 'token_id': 1672, 'type': 'input', 'position': 109}, {'token': 'Ġbeauty', 'token_id': 4002, 'type': 'input', 'position': 110}, {'token': 'Ġof', 'token_id': 9, 'type': 'input', 'position': 111}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 112}, {'token': 'Ġsea', 'token_id': 3342, 'type': 'input', 'position': 113}, {'token': ',', 'token_id': 6, 'type': 'input', 'position': 114}, {'token': 'Ċ', 'token_id': 50118, 'type': 'input', 'position': 115}, {'token': 'with', 'token_id': 5632, 'type': 'input', 'position': 116}, {'token': 'Ġits', 'token_id': 63, 'type': 'input', 'position': 117}, {'token': 'Ġcapt', 'token_id': 13363, 'type': 'input', 'position': 118}, {'token': 'ivating', 'token_id': 28191, 'type': 'input', 'position': 119}, {'token': 'Ġcalm', 'token_id': 6327, 'type': 'input', 'position': 120}, {'token': 'ness', 'token_id': 1825, 'type': 'input', 'position': 121}, {'token': ',', 'token_id': 6, 'type': 'input', 'position': 122}, {'token': 'Ġoffers', 'token_id': 1523, 'type': 'input', 'position': 123}, {'token': 'Ġa', 'token_id': 10, 'type': 'input', 'position': 124}, {'token': 'Ġsanctuary', 'token_id': 13488, 'type': 'input', 'position': 125}, {'token': 'Ġfrom', 'token_id': 31, 'type': 'input', 'position': 126}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 127}, {'token': 'Ġbustling', 'token_id': 29019, 'type': 'input', 'position': 128}, {'token': 'Ġnoise', 'token_id': 6496, 'type': 'input', 'position': 129}, {'token': 'Ġof', 'token_id': 9, 'type': 'input', 'position': 130}, {'token': 'Ġeveryday', 'token_id': 7476, 'type': 'input', 'position': 131}, {'token': 'Ġlife', 'token_id': 301, 'type': 'input', 'position': 132}, {'token': ',', 'token_id': 6, 'type': 'input', 'position': 133}, {'token': 'Ġa', 'token_id': 10, 'type': 'input', 'position': 134}, {'token': 'Ġhaven', 'token_id': 2220, 'type': 'input', 'position': 135}, {'token': 'Ċ', 'token_id': 50118, 'type': 'input', 'position': 136}, {'token': 'where', 'token_id': 8569, 'type': 'input', 'position': 137}, {'token': 'Ġone', 'token_id': 65, 'type': 'input', 'position': 138}, {'token': 'Ġcan', 'token_id': 64, 'type': 'input', 'position': 139}, {'token': 'Ġfind', 'token_id': 465, 'type': 'input', 'position': 140}, {'token': 'Ġpeace', 'token_id': 1987, 'type': 'input', 'position': 141}, {'token': 'Ġand', 'token_id': 8, 'type': 'input', 'position': 142}, {'token': 'Ġsolitude', 'token_id': 41813, 'type': 'input', 'position': 143}, {'token': '.', 'token_id': 4, 'type': 'input', 'position': 144}, {'token': 'ĠIt', 'token_id': 85, 'type': 'input', 'position': 145}, {'token': 'âĢ', 'token_id': 17, 'type': 'input', 'position': 146}, {'token': 'Ļ', 'token_id': 27, 'type': 'input', 'position': 147}, {'token': 's', 'token_id': 29, 'type': 'input', 'position': 148}, {'token': 'Ġa', 'token_id': 10, 'type': 'input', 'position': 149}, {'token': 'Ġplace', 'token_id': 317, 'type': 'input', 'position': 150}, {'token': 'Ġwhere', 'token_id': 147, 'type': 'input', 'position': 151}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 152}, {'token': 'Ġmind', 'token_id': 1508, 'type': 'input', 'position': 153}, {'token': 'Ġcan', 'token_id': 64, 'type': 'input', 'position': 154}, {'token': 'Ġrest', 'token_id': 1079, 'type': 'input', 'position': 155}, {'token': ',', 'token_id': 6, 'type': 'input', 'position': 156}, {'token': 'Ġallowing', 'token_id': 2455, 'type': 'input', 'position': 157}, {'token': 'Ġthoughts', 'token_id': 4312, 'type': 'input', 'position': 158}, {'token': 'Ġto', 'token_id': 7, 'type': 'input', 'position': 159}, {'token': 'Ċ', 'token_id': 50118, 'type': 'input', 'position': 160}, {'token': 'flow', 'token_id': 19322, 'type': 'input', 'position': 161}, {'token': 'Ġas', 'token_id': 25, 'type': 'input', 'position': 162}, {'token': 'Ġfreely', 'token_id': 13215, 'type': 'input', 'position': 163}, {'token': 'Ġas', 'token_id': 25, 'type': 'input', 'position': 164}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 165}, {'token': 'Ġwaters', 'token_id': 5794, 'type': 'input', 'position': 166}, {'token': 'Ġthemselves', 'token_id': 1235, 'type': 'input', 'position': 167}, {'token': '.', 'token_id': 4, 'type': 'input', 'position': 168}, {'token': 'ĠThe', 'token_id': 20, 'type': 'input', 'position': 169}, {'token': 'Ġocean', 'token_id': 6444, 'type': 'input', 'position': 170}, {'token': ',', 'token_id': 6, 'type': 'input', 'position': 171}, {'token': 'Ġin', 'token_id': 11, 'type': 'input', 'position': 172}, {'token': 'Ġits', 'token_id': 63, 'type': 'input', 'position': 173}, {'token': 'Ġinfinite', 'token_id': 32952, 'type': 'input', 'position': 174}, {'token': 'Ġtranqu', 'token_id': 36474, 'type': 'input', 'position': 175}, {'token': 'ility', 'token_id': 13005, 'type': 'input', 'position': 176}, {'token': ',', 'token_id': 6, 'type': 'input', 'position': 177}, {'token': 'Ġserves', 'token_id': 4542, 'type': 'input', 'position': 178}, {'token': 'Ġas', 'token_id': 25, 'type': 'input', 'position': 179}, {'token': 'Ġa', 'token_id': 10, 'type': 'input', 'position': 180}, {'token': 'Ġreminder', 'token_id': 8306, 'type': 'input', 'position': 181}, {'token': 'Ċ', 'token_id': 50118, 'type': 'input', 'position': 182}, {'token': 'of', 'token_id': 1116, 'type': 'input', 'position': 183}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 184}, {'token': 'Ġsimple', 'token_id': 2007, 'type': 'input', 'position': 185}, {'token': 'Ġand', 'token_id': 8, 'type': 'input', 'position': 186}, {'token': 'Ġquiet', 'token_id': 5128, 'type': 'input', 'position': 187}, {'token': 'Ġbeauty', 'token_id': 4002, 'type': 'input', 'position': 188}, {'token': 'Ġthat', 'token_id': 14, 'type': 'input', 'position': 189}, {'token': 'Ġexists', 'token_id': 8785, 'type': 'input', 'position': 190}, {'token': 'Âł', 'token_id': 50141, 'type': 'input', 'position': 191}, {'token': 'in', 'token_id': 179, 'type': 'input', 'position': 192}, {'token': 'Âł', 'token_id': 50141, 'type': 'input', 'position': 193}, {'token': 'our', 'token_id': 2126, 'type': 'input', 'position': 194}, {'token': 'Âł', 'token_id': 50141, 'type': 'input', 'position': 195}, {'token': 'world', 'token_id': 8331, 'type': 'input', 'position': 196}, {'token': '.', 'token_id': 4, 'type': 'input', 'position': 197}, {'token': 'Ċ', 'token_id': 50118, 'type': 'input', 'position': 198}, {'token': '</s>', 'token_id': 2, 'type': 'input', 'position': 199}], 'factors': [[[0.0035361810587346554, 0.0, 0.0, 0.020792609080672264, 0.04108893871307373, 0.42964980006217957, 0.0, 0.0, 0.0, 0.040093790739774704, 0.08122817426919937, 0.0, 0.00808766670525074, 0.0, 0.0, 0.00887211598455906, 0.03857743367552757, 0.026162708178162575, 0.0, 0.02337517775595188, 0.0, 0.0, 0.0, 0.0, 0.0005390521255321801, 0.0, 0.0, 0.0, 0.0, 0.18887430429458618, 0.02731953002512455, 0.011759213171899319, 0.5147596001625061, 0.04763694480061531, 0.0, 0.0, 0.0, 0.0, 0.021983616054058075, 0.2840948700904846, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.01695079728960991, 0.4850414991378784, 0.03220218047499657, 0.006109337788075209, 0.05515505000948906, 0.05240119248628616, 0.0, 0.02402854710817337, 0.04531220719218254, 0.0, 0.0, 0.004718354903161526, 0.012945190072059631, 0.038509394973516464, 0.0, 0.0, 0.01764330081641674, 0.019078878685832024, 0.022553853690624237, 0.32601040601730347, 0.0, 0.0, 0.0874447301030159, 0.03589866682887077, 0.01739577017724514, 0.02054494246840477, 0.0, 0.0009446903131902218, 0.0, 0.0, 0.02067708596587181, 0.005050791427493095, 0.03716467320919037, 0.07556000351905823, 0.546319305896759, 0.054925840348005295, 0.0, 0.06076178327202797, 0.030281972140073776, 0.057766105979681015, 0.0044393474236130714, 0.015346687287092209, 0.01382719911634922, 0.0, 0.013379364274442196, 0.0, 0.025089332833886147, 0.0, 0.03330530598759651, 0.24486768245697021, 0.0, 0.0039084795862436295, 0.0, 0.03550974652171135, 0.10352661460638046, 0.030049271881580353, 0.0, 0.0, 0.0, 0.0, 0.009021435864269733, 0.0404999703168869, 0.046627532690763474, 0.5230283737182617, 0.036228302866220474, 0.007634864188730717, 0.0, 0.0, 0.0397779680788517, 0.0, 0.0, 0.009003032930195332, 0.0, 0.02924979478120804, 0.0, 0.0, 0.0, 0.0, 0.18092328310012817, 0.0, 0.0, 0.043219100683927536, 0.3900614380836487, 0.0, 0.0031363163143396378, 0.0054818009957671165, 0.0, 0.0, 0.005035036709159613, 0.011785756796598434, 0.0, 0.0, 0.02449091151356697, 0.0, 0.05614953115582466, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.06787341088056564, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.036128461360931396, 0.0, 0.0, 0.013680041767656803, 0.0, 0.09392360597848892, 0.0, 0.0, 0.01434206124395132, 0.0, 0.0, 0.0, 0.0, 0.1123528778553009, 0.0, 0.005787168629467487, 0.0, 0.0, 0.0, 0.0, 0.026393940672278404, 0.0, 0.0, 0.018130820244550705, 0.20531438291072845, 0.0, 0.0, 0.042054682970047, 0.0, 0.0, 0.11798553913831711, 0.0, 0.0, 0.05224870145320892, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08640600740909576, 0.40340858697891235, 1.1984789371490479, 0.9781271815299988, 1.1275320053100586, 0.5014640688896179, 0.48066702485084534, 0.34413349628448486, 1.5121182203292847, 1.1378841400146484, 1.0008412599563599, 0.5994250774383545, 0.47776931524276733, 1.1111171245574951, 0.8526929020881653, 0.5430008172988892, 1.2400752305984497, 0.5898318886756897, 0.7702411413192749, 0.9811338782310486, 0.3476639986038208, 0.3071478307247162, 0.7847773432731628, 0.5997228622436523, 0.7469450831413269, 0.20879854261875153, 0.21232913434505463, 0.16599228978157043, 0.4198036193847656, 0.2899167239665985, 0.010965323075652122, 0.3741605579853058, 0.0, 0.0, 0.13110151886940002, 0.027254479005932808, 0.5444333553314209, 0.1909116953611374, 0.3102450370788574, 0.0, 0.0749225839972496, 0.1781729757785797, 0.0, 0.0, 0.06401117146015167, 0.17206470668315887, 0.38146188855171204, 0.8342610597610474, 0.9516343474388123, 0.38102084398269653, 0.3120419979095459, 0.4545346200466156, 1.0111913681030273, 0.3100586533546448, 0.3253933787345886, 0.3743848204612732, 0.3496541976928711, 0.2547626495361328, 0.4137634336948395, 0.4471285939216614, 0.45498234033584595, 0.031921301037073135, 0.2158108502626419, 0.04906143620610237, 0.0, 0.04831996187567711, 0.5298997163772583, 0.0, 0.05358058586716652, 0.10533491522073746, 0.09731912612915039, 0.2627919614315033, 0.4590207636356354, 0.0, 0.06678743660449982, 0.03173194080591202, 0.13423974812030792, 0.5331905484199524, 0.013709497638046741, 0.6845324635505676, 0.42190539836883545, 0.7421107888221741, 0.06482173502445221, 0.0, 0.0, 0.0, 0.0, 0.0, 0.29135820269584656, 0.30476394295692444, 0.0, 0.27771490812301636, 0.0, 0.16405148804187775, 0.0, 0.0, 0.10764826089143753, 0.0, 0.19274692237377167, 0.0, 0.0, 0.0, 0.0, 0.0, 0.11786585301160812, 0.06383014470338821, 0.0, 0.0, 0.16128945350646973, 0.016639607027173042, 0.1796693503856659, 0.0, 0.0, 0.0, 0.0, 0.1077045425772667, 0.3143996298313141, 0.24690334498882294, 0.09163343906402588, 0.12503023445606232, 0.1122516393661499, 0.04195356369018555, 0.0, 0.38422951102256775, 0.0, 0.06069980561733246, 0.008632451295852661, 0.0, 0.30118975043296814, 0.4261574447154999, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.06886778026819229, 0.0, 0.08103777468204498, 0.13685713708400726, 0.062298763543367386, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0654878169298172, 0.13888902962207794, 0.30709272623062134, 0.0, 0.20959815382957458, 0.2129092961549759, 0.0, 0.07591192424297333, 0.3038554787635803, 0.00802428089082241, 0.2534346282482147, 0.0, 0.0, 0.0, 0.12875716388225555, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.22661930322647095, 0.0, 0.0, 0.06304571777582169, 0.0, 0.0, 0.0, 0.0, 0.0, 0.08120070397853851, 0.0, 0.0, 0.26661360263824463, 0.029057174921035767, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0007586907595396042, 0.11666747182607651, 0.16546213626861572, 0.0422699935734272, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.009476385079324245, 0.23809944093227386, 0.0, 0.011348426342010498, 0.0, 0.0, 0.0, 0.0, 0.0, 0.02504613809287548, 0.2535344362258911, 0.004236103501170874, 0.0105015579611063, 0.23605754971504211, 0.291931688785553, 0.24107736349105835, 0.05770550295710564, 0.04384835064411163, 0.2059280276298523, 0.0, 0.011506790295243263, 0.02065371349453926, 0.0, 0.07660134881734848, 0.0032411047723144293, 0.10095755755901337, 0.05288945510983467, 0.3175467550754547, 0.37986132502555847, 0.3712118864059448, 0.29308226704597473, 0.0, 0.0, 0.0, 0.2016507238149643, 0.10317213833332062, 0.0, 0.0, 0.0, 0.039273206144571304, 0.0, 0.046192098408937454, 0.0, 0.0, 0.0, 0.332546204328537, 0.16516728699207306, 0.13993242383003235, 0.0, 0.0, 0.30772456526756287, 0.3398284614086151, 0.2735825181007385, 0.04096965491771698, 0.0, 0.009623963385820389, 0.040907859802246094, 0.0, 0.1683982014656067, 0.17406313121318817, 0.0018566998187452555, 0.12907898426055908, 0.0, 0.0, 0.25061705708503723, 0.08069244027137756, 0.26343122124671936, 0.2874613106250763, 0.11852219700813293, 0.0, 0.0, 0.0, 0.0, 0.013595224358141422, 0.026530863717198372, 0.019577959552407265, 0.07578124105930328, 0.018165703862905502, 0.2832675874233246, 0.05239404737949371, 0.15813560783863068, 0.013249417766928673, 0.03024030663073063, 0.13331055641174316, 0.05858627334237099, 0.2798331081867218, 0.316333532333374, 0.4200943410396576, 0.2861673831939697, 0.024932773783802986, 0.05722324922680855, 0.28068649768829346, 0.35315507650375366, 0.005277412943542004, 0.008498495444655418, 0.37484440207481384, 0.3311186730861664, 0.14805416762828827, 0.0, 0.0, 0.0, 0.0, 0.0, 0.039406631141901016, 0.042094454169273376, 0.33296459913253784, 0.28502458333969116, 0.34911760687828064, 0.22479186952114105, 0.0, 0.007016634102910757, 0.0, 0.1570882350206375, 0.0017821566434577107, 0.002374667674303055, 0.22986292839050293, 0.11321914941072464, 0.0005563956801779568, 0.1439415067434311, 0.0415809266269207, 0.0, 0.0, 0.057039882987737656, 0.0, 0.015189464204013348, 0.0, 0.0, 0.03277374431490898, 0.26035866141319275, 0.03901040181517601, 0.22364719212055206, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.048758819699287415, 0.0, 0.09279544651508331, 0.0, 0.031916387379169464, 0.024779047816991806, 0.0, 0.005959205329418182, 0.02375151589512825, 0.01369917206466198, 0.10842271894216537, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.03675233945250511, 0.10070821642875671, 0.30679851770401, 0.4015676975250244, 0.29233482480049133, 0.0, 0.008106958121061325, 0.0, 0.0, 0.0904330387711525, 0.0, 0.011628473177552223, 0.0, 0.22413791716098785, 0.06540079414844513, 0.31217801570892334, 0.12588194012641907, 0.0, 0.04044456034898758, 0.053852491080760956, 0.0135507108643651, 0.03225414827466011, 0.03454913944005966, 0.03900153934955597, 0.016866227611899376, 0.0, 0.0, 0.0], [0.011605659499764442, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0003179467748850584, 0.0005173212266527116, 0.0, 0.00038343569030985236, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.003258991753682494, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.00027851550839841366, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.5657922625541687, 0.0021730675362050533, 0.0005690088728442788, 0.00034800986759364605, 0.000398021045839414, 0.00018709573487285525, 0.0, 0.0, 0.0, 0.0003175176098011434, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 6.0980299167567864e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0014958318788558245, 0.000916283403057605, 0.00012051799421897158, 0.5173013210296631, 0.001210394431836903, 0.0008240093011409044, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0003011080261785537, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.00012988231901545078, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0001321741146966815, 0.0013812001561746001, 0.000397591560613364, 0.0, 0.5714530944824219, 0.0011315691517665982, 0.0, 9.2054542619735e-05, 0.0, 0.0, 0.0, 2.7619955744739855e-06, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.00042794697219505906, 0.0, 0.0004979583900421858, 0.0, 0.0, 0.0, 0.0006054965197108686, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0003784533473663032, 0.0, 0.0, 0.0, 0.0, 0.0, 0.00039745058165863156, 0.0, 0.0, 0.0, 0.0, 0.0, 0.5721882581710815, 0.0011658560251817107, 0.00018991615797858685, 0.0, 0.0, 0.0, 0.0004350291274022311, 0.0023356499150395393, 0.0013044382212683558, 0.0012037663254886866, 0.0005989198107272387, 0.00029739912133663893, 6.202567601576447e-05, 0.0, 0.0, 0.0005352781736291945, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.000492303108330816, 0.5032380819320679, 0.0, 0.0, 0.0, 0.001435233629308641, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.00017899509111884981, 0.00011344490485498682, 0.0, 0.0, 0.0, 0.0009182734647765756, 0.0001464070228394121, 0.0, 0.0008401652448810637, 0.0, 0.0, 0.0009017800912261009, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.004193447530269623, 0.001232888549566269, 0.00606337608769536], [0.14606045186519623, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.18202310800552368, 0.14535099267959595, 0.18044081330299377, 0.16095319390296936, 0.0, 0.008983696810901165, 0.0, 0.007499690633267164, 0.22355738282203674, 0.0, 0.02544071339070797, 0.12029485404491425, 0.08037839084863663, 0.0, 0.10401821881532669, 0.0, 0.033359117805957794, 0.0, 0.21063025295734406, 0.1660287082195282, 0.1630978286266327, 0.20107750594615936, 0.006469557527452707, 0.0, 0.03011246770620346, 0.027174655348062515, 0.0007700928836129606, 0.032065775245428085, 0.3916381597518921, 0.07583476603031158, 0.145252987742424, 0.09171503037214279, 0.034053415060043335, 0.060005683451890945, 0.0504894033074379, 0.10681280493736267, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.161360964179039, 0.0, 0.0, 0.02561849355697632, 0.24857497215270996, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.003714183112606406, 0.0, 0.02283918857574463, 0.0, 0.08031328022480011, 0.0450015664100647, 0.15285848081111908, 0.24667717516422272, 0.05046803876757622, 0.030235137790441513, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.10948110371828079, 0.057765934616327286, 0.10148332267999649, 0.06138363853096962, 0.34210309386253357, 0.0, 0.0, 0.0, 0.096776582300663, 0.0, 0.007091627921909094, 0.09762075543403625, 0.06140512600541115, 0.1148688942193985, 0.0, 0.0, 0.0, 0.06043699011206627, 0.30179551243782043, 0.11922610551118851, 0.01733517460525036, 0.0, 0.0711059644818306, 0.05483906343579292, 0.046042781323194504, 0.04177425429224968, 0.05338345468044281, 0.0, 0.0, 0.0, 0.0, 0.29416748881340027, 0.14935439825057983, 0.1284521371126175, 0.1482260823249817, 0.010001765564084053, 0.07378999888896942, 0.05907292664051056, 0.5566102266311646, 0.2106155902147293, 0.24848927557468414, 0.33604729175567627, 0.10283773392438889, 0.13423052430152893, 0.06221947818994522, 0.1767403781414032, 0.21683762967586517, 0.24768871068954468, 0.16963085532188416, 0.21924936771392822, 0.3035547733306885, 0.06697403639554977, 0.40594446659088135, 0.42579880356788635, 0.5964483022689819, 0.5598035454750061, 0.14500834047794342, 0.3233937919139862, 0.19392579793930054, 0.00857479590922594, 0.4392666220664978, 0.5555021166801453, 0.28221645951271057, 0.6130795478820801, 0.3449348211288452, 0.4239813983440399, 0.5673792362213135, 0.34274497628211975, 0.30525702238082886, 0.624272882938385, 0.42189598083496094, 0.21421600878238678, 0.5798012614250183, 0.30678892135620117, 0.45441150665283203, 0.2343684881925583, 0.3049859404563904, 0.4831295907497406, 0.3994380831718445, 0.5074568390846252, 0.29483598470687866, 0.23972338438034058, 0.45780453085899353, 0.03675449267029762, 0.18051987886428833, 0.07756238430738449, 0.08749192208051682, 0.32261738181114197, 0.1833927482366562, 0.12249187380075455, 0.012490673922002316, 0.12521307170391083, 0.07365849614143372, 0.619141161441803, 0.4918367862701416, 0.20087453722953796, 0.2967234253883362, 0.0, 0.20322728157043457, 0.03704241290688515, 0.06477394700050354, 0.014601409435272217, 0.0, 0.06652709096670151, 0.13747809827327728, 0.4177452027797699, 0.01932639256119728, 0.25695526599884033, 0.0, 0.14598585665225983, 0.0, 0.04455329105257988, 0.05350680649280548, 0.001255353563465178, 0.029989076778292656], [0.022223496809601784, 0.0, 0.04429454356431961, 0.061570312827825546, 0.18613207340240479, 0.0, 0.0, 0.5468668937683105, 0.0, 0.0, 0.0, 0.12211401760578156, 0.0, 0.0, 0.06615367531776428, 0.4260331094264984, 0.0, 0.0, 0.3358655273914337, 0.15134117007255554, 0.0, 0.0, 0.0, 0.08363494277000427, 0.36784741282463074, 0.0, 0.0, 0.0, 0.02118087001144886, 0.0, 0.0, 0.22750049829483032, 0.0280077513307333, 0.050291936844587326, 0.49925553798675537, 0.05472855269908905, 0.0, 0.0, 0.11147216707468033, 0.0, 0.011955374851822853, 0.0, 0.028605690225958824, 0.10573448985815048, 0.0, 0.0, 0.0, 0.025190522894263268, 0.1597130000591278, 0.0, 0.002406288171187043, 0.48406460881233215, 0.029142817482352257, 0.0, 0.37299349904060364, 0.396345317363739, 0.0, 0.0, 0.038852911442518234, 0.09890187531709671, 0.2090388387441635, 0.0, 0.0, 0.0, 0.0, 0.0, 0.230550155043602, 0.0, 0.0, 0.1647813469171524, 0.022389274090528488, 0.0, 0.0, 0.0, 0.32808640599250793, 0.0, 0.0, 0.07666517049074173, 0.0, 0.0, 0.0, 0.1696578711271286, 0.0, 0.042747143656015396, 0.6022328734397888, 0.035510364919900894, 0.033723730593919754, 0.033554498106241226, 0.39469239115715027, 0.0, 0.0, 0.033018939197063446, 0.0, 0.12308180332183838, 0.0, 0.00220291456207633, 0.10636887699365616, 0.0, 0.0, 0.002903927583247423, 0.0, 0.08345427364110947, 0.05361540988087654, 0.0, 0.0, 0.10474972426891327, 0.0, 0.0, 0.0, 0.0, 0.25923576951026917, 0.0020885118283331394, 0.04818301275372505, 0.5754918456077576, 0.02389109507203102, 0.0, 0.0, 0.0, 0.0, 0.0, 0.10098142176866531, 0.16171248257160187, 0.003517358098179102, 0.0, 0.0, 0.15156231820583344, 0.0, 0.0, 0.08947780728340149, 0.24325262010097504, 0.0, 0.14912953972816467, 0.2836979329586029, 0.0, 0.0, 0.20737123489379883, 0.0, 0.09182723611593246, 0.09936512261629105, 0.015192240476608276, 0.0, 0.13284067809581757, 0.0, 0.15401072800159454, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.2099253237247467, 0.015543865971267223, 0.0, 0.2743268311023712, 0.0, 0.044097453355789185, 0.0, 0.0, 0.19282636046409607, 0.0, 0.0, 0.15565341711044312, 0.0, 0.0414227694272995, 0.0, 0.0, 0.41820552945137024, 0.09395670145750046, 0.0, 0.0, 0.5221292972564697, 0.0010243103606626391, 0.0, 0.0, 0.009678013622760773, 0.029375266283750534, 0.11368092894554138, 0.0, 0.0, 0.0, 0.0, 0.0918872207403183, 0.0, 0.0, 0.0, 0.03648423030972481, 0.0, 0.06136131286621094, 0.23203766345977783, 0.03743362799286842, 0.004566431511193514, 0.0, 0.0, 0.0, 0.030426787212491035, 0.0, 0.30921921133995056, 0.0, 0.0013823113404214382, 0.0252449419349432], [0.0, 0.07337938994169235, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.020420797169208527, 0.0, 0.0, 0.0, 0.11034400761127472, 0.0, 0.007978366687893867, 0.0, 0.043033163994550705, 0.025875605642795563, 0.0, 0.0, 0.0, 0.06749524921178818, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.061604324728250504, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0005424991250038147, 0.47279083728790283, 0.1266089826822281, 0.15523146092891693, 0.014697270467877388, 0.0, 0.0, 0.0, 0.0, 0.0034077924210578203, 0.14594173431396484, 0.06857740134000778, 0.010744591243565083, 0.006921544671058655, 0.001538067706860602, 0.003343453397974372, 0.0, 0.0, 0.0, 0.0, 0.11335251480340958, 0.5092518925666809, 0.025584278628230095, 0.05698324367403984, 0.0517057403922081, 0.0749903991818428, 0.5377638339996338, 0.08143696933984756, 0.06248866394162178, 0.06645916402339935, 0.10557360202074051, 0.12029074877500534, 0.05237838253378868, 0.0, 0.0920850858092308, 0.0220407173037529, 0.0, 0.04525592178106308, 0.008342665620148182, 0.0018693970050662756, 0.1508486419916153, 0.048073187470436096, 0.05308690667152405, 0.0022429723758250475, 0.03269350901246071, 0.0, 0.0, 0.0, 0.0, 0.09662716090679169, 0.0, 0.1059243381023407, 0.006101205013692379, 0.02696894109249115, 0.5540883541107178, 0.07306152582168579, 0.08339811116456985, 0.008889700286090374, 0.08579941838979721, 0.5140876173973083, 0.1666347086429596, 0.19421082735061646, 0.04394082725048065, 0.049136485904455185, 0.0, 0.02105635032057762, 0.09251866489648819, 0.12305020540952682, 0.06582363694906235, 0.0, 0.0018697528867051005, 0.11945059150457382, 0.0, 0.005289106629788876, 0.006918823346495628, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.14714834094047546, 0.019016092643141747, 0.034810811281204224, 0.0, 0.006767177022993565, 0.0, 0.008703550323843956, 0.42123499512672424, 0.019162777811288834, 0.03816310688853264, 0.12270371615886688, 0.0, 0.004258650355041027, 0.025278868153691292, 0.0, 0.0, 0.0, 0.37406525015830994, 0.0, 0.0, 0.0, 0.0, 0.0, 0.004361770581454039, 0.0, 0.04107137396931648, 0.0, 0.0, 0.02391914650797844, 0.06983508169651031, 0.0, 0.06323341280221939, 0.3193914592266083, 0.04922498017549515, 0.006156308576464653, 0.022406166419386864, 0.0, 0.005692028906196356, 0.0, 0.0, 0.009938056580722332, 0.020425966009497643, 0.019391220062971115, 0.0, 0.0, 0.020695878192782402, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.025925807654857635, 0.12135378271341324, 0.0, 0.0, 0.0, 0.0, 0.0, 0.15346769988536835, 0.4231683611869812, 0.012039017863571644, 0.007896999828517437, 0.009390484541654587, 0.19666841626167297, 0.05069227144122124, 0.13681234419345856, 0.026195060461759567, 0.026074590161442757, 0.04651140794157982, 0.0, 0.0, 0.022519893944263458, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.030434569343924522, 0.7510085105895996, 0.21793746948242188, 0.06448006629943848, 0.0, 0.12113841623067856, 1.4952657222747803, 0.00404721312224865, 0.0, 0.0, 0.006903966423124075, 0.24252033233642578, 0.0, 0.40750953555107117, 0.0, 0.0, 0.0, 1.112167239189148, 0.0, 0.0, 0.10533370077610016, 0.1403769999742508, 0.41398730874061584, 0.06561003625392914, 0.0, 0.04994625225663185, 0.0, 0.02839205600321293, 0.0, 0.14836205542087555, 1.3657323122024536, 0.0, 0.07983159273862839, 1.4496504068374634, 0.18455299735069275, 0.0, 0.0, 0.025525780394673347, 0.0, 0.0, 0.14748568832874298, 0.01216861791908741, 0.0, 0.0, 0.0, 0.14418426156044006, 0.8432008624076843, 0.03631266951560974, 0.0, 0.0, 1.402811050415039, 0.014857744798064232, 0.0, 1.4046052694320679, 0.1642962247133255, 0.02188277430832386, 0.109036386013031, 0.0855708196759224, 0.0, 0.0, 0.0, 0.0, 0.018700748682022095, 0.0, 0.0, 0.0, 0.0, 0.0, 0.12721005082130432, 0.0, 0.0, 0.0, 0.0, 1.1957283020019531, 0.03581157326698303, 0.024494923651218414, 0.9450125098228455, 0.0, 0.06469863653182983, 0.0, 0.037865348160266876, 0.0, 0.07735986262559891, 1.5571389198303223, 0.08223924040794373, 0.4504290819168091, 0.10522743314504623, 0.482852578163147, 0.0, 0.0, 0.020117703825235367, 0.0, 0.0075312103144824505, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.00946158915758133, 0.0, 0.0, 0.0, 0.10409317910671234, 0.003409144701436162, 0.0, 0.0, 0.8561471700668335, 0.03213604912161827, 0.1637144833803177, 0.0, 0.09083836525678635, 1.6318751573562622, 0.1279844492673874, 0.029201587662100792, 0.07485201209783554, 0.0, 0.40953513979911804, 0.0, 0.0, 0.0038650603964924812, 0.0, 0.0, 0.0, 0.012114928103983402, 0.0, 0.03036263771355152, 1.1441057920455933, 0.0497637540102005, 0.0, 0.0, 0.1305168718099594, 0.03339557722210884, 0.0, 0.05188511312007904, 0.0, 0.00013000352191738784, 0.0, 0.1440654993057251, 0.07052035629749298, 0.0, 0.0, 0.08770211040973663, 0.0, 0.0, 0.167805016040802, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.8677391409873962, 0.016161486506462097, 0.0, 0.0, 0.0, 0.0, 0.0, 0.03197193518280983, 0.08244912326335907, 0.0, 0.11572178453207016, 0.0, 0.07781554013490677, 0.9929354786872864, 0.0, 0.020929813385009766, 0.0024220759514719248, 0.9969769716262817, 0.011166352778673172, 0.0, 0.11075453460216522, 0.5028024315834045, 0.08133435994386673, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.8086915612220764, 0.0, 0.010449683293700218, 0.0, 0.0, 0.0, 0.0, 0.0, 0.004143587779253721, 0.0, 0.06094047799706459, 0.0, 0.0, 0.0, 0.02816825546324253, 0.07717560976743698], [0.19402866065502167, 0.07139421999454498, 0.0, 0.0032638246193528175, 0.0, 0.0, 0.0, 0.0, 0.0, 0.015665071085095406, 0.019156184047460556, 0.04475130885839462, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.008159277960658073, 0.03270398825407028, 0.14822633564472198, 0.2197050005197525, 0.02581295371055603, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.025167711079120636, 0.0, 0.02150610275566578, 0.0, 0.0, 0.23326629400253296, 0.020967630669474602, 0.0, 0.010770564898848534, 0.0, 0.011323575861752033, 0.0, 0.01830163039267063, 0.0, 0.02228015474975109, 0.011143689043819904, 0.0, 0.0, 0.0, 0.05691388621926308, 0.014891975559294224, 0.0, 0.0, 0.05762895196676254, 0.04021815210580826, 0.07376077026128769, 0.030124187469482422, 0.07264810055494308, 0.25400054454803467, 0.16085031628608704, 0.10517626255750656, 0.06188645958900452, 0.005952347069978714, 0.05766044557094574, 0.03470627963542938, 0.008932594209909439, 0.03559884801506996, 0.0, 0.0, 0.02778223156929016, 0.00964529812335968, 0.0, 0.0, 0.0, 0.0, 0.05664198473095894, 0.23469695448875427, 0.050728943198919296, 0.02702018804848194, 0.0376739539206028, 0.0, 0.02554723247885704, 0.20395343005657196, 0.07193979620933533, 0.010246108286082745, 0.0, 0.0, 0.008692966774106026, 0.01669810712337494, 0.06123824790120125, 0.0254029743373394, 0.02794969081878662, 0.063988596200943, 0.0, 0.00023295063874684274, 0.0, 0.0, 0.04339549317955971, 0.0, 0.0, 0.0, 0.0, 5.7850793382385746e-05, 0.0, 0.02006392553448677, 0.23552966117858887, 0.049102477729320526, 0.0, 0.0, 0.0, 0.0, 0.0, 0.027933111414313316, 0.0, 0.0, 0.0017046838765963912, 0.0, 0.0, 0.0, 0.0, 0.013783535920083523, 0.0258365198969841, 0.020166903734207153, 0.0, 0.012400772422552109, 0.04308166354894638, 0.26710981130599976, 0.06373942643404007, 0.04159398749470711, 0.0, 0.0, 0.0, 0.014723486267030239, 0.0, 0.0, 0.06573443859815598, 0.01667429320514202, 0.20078493654727936, 0.0, 0.0, 0.0, 0.0, 0.005374654196202755, 0.0, 0.0, 0.0, 0.00011046290455851704, 0.0, 0.02811959572136402, 0.04698067530989647, 0.17657671868801117, 0.03830321505665779, 0.005986538715660572, 0.0, 0.0, 0.016297344118356705, 0.0009141197078861296, 0.03341440111398697, 0.019511910155415535, 0.0742667019367218, 0.025335272774100304, 0.023807531222701073, 0.0016073384322226048, 0.011450609192252159, 0.026093991473317146, 0.019171644002199173, 0.0, 0.046896617859601974, 0.00019907658861484379, 0.0, 0.01129501685500145, 0.06313109397888184, 0.2882687449455261, 0.10856178402900696, 0.06810500472784042, 0.09505273401737213, 0.10375266522169113, 0.05078205466270447, 0.0765175148844719, 0.10699789971113205, 0.06304673850536346, 0.28270092606544495, 0.16277289390563965, 0.3115142583847046, 0.21804597973823547, 0.3013743758201599, 0.14285610616207123, 0.2769629657268524, 0.3175087571144104, 0.27534517645835876], [0.019482553005218506, 0.04894077032804489, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.006231330800801516, 0.06529644876718521, 0.045975349843502045, 0.028861617669463158, 0.548455536365509, 0.036600932478904724, 0.0, 0.020762775093317032, 0.0002435527858324349, 0.011650432832539082, 0.0, 0.0, 0.1551251858472824, 0.09102904796600342, 0.013978823088109493, 0.0, 0.006271384656429291, 0.5998114347457886, 0.05284499377012253, 0.01981145143508911, 0.0, 0.03446308895945549, 0.0009421982103958726, 0.0, 0.0, 0.0, 0.014307251200079918, 0.5978204011917114, 0.0, 0.0, 0.0, 0.009730234742164612, 0.023617086932063103, 0.0, 0.0, 0.020452387630939484, 0.0, 0.09568740427494049, 0.045709725469350815, 0.0, 0.0, 0.0028490587137639523, 0.006703207269310951, 0.0, 0.0, 0.00498091708868742, 0.0, 0.05285939574241638, 0.10448426753282547, 0.0, 0.0, 0.0, 0.0, 0.552777111530304, 0.003148508258163929, 0.0, 0.0, 0.0, 0.0, 0.0035340976901352406, 0.023302726447582245, 0.0, 0.17178110778331757, 0.017547478899359703, 0.0, 0.0, 0.0, 0.025860097259283066, 0.04465994983911514, 0.0, 0.46774229407310486, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.07842171937227249, 0.007876627147197723, 0.038834404200315475, 0.0, 0.0, 0.0, 0.0, 0.07079979032278061, 0.0, 0.509480893611908, 0.04015938192605972, 0.0, 0.005013055168092251, 0.0, 0.014066137373447418, 0.0, 0.03533696010708809, 0.1738981455564499, 0.07294977456331253, 0.012694945558905602, 0.0030643087811768055, 0.0, 0.0618455596268177, 0.00500192865729332, 0.034236714243888855, 0.0256967730820179, 0.0, 0.0, 0.011284167878329754, 0.627376914024353, 0.151597797870636, 0.08812949061393738, 0.08307117968797684, 0.01706392876803875, 0.01793467439711094, 0.02027561515569687, 0.06411416083574295, 0.5795279145240784, 0.0, 0.0, 0.0, 0.07688068598508835, 0.020097747445106506, 0.0, 0.0, 0.0, 0.0, 0.019198698922991753, 0.5250990986824036, 0.009457932785153389, 0.0, 0.07246951758861542, 0.0, 0.013879921287298203, 0.014812066219747066, 0.0, 0.0, 0.13239064812660217, 0.0, 0.0, 0.03609374538064003, 0.03567028418183327, 0.0, 0.015041994862258434, 0.0, 0.0, 0.026958541944622993, 0.0011112819192931056, 0.0034918771125376225, 0.018410151824355125, 0.0, 0.49318927526474, 0.0, 0.0, 0.06723111867904663, 0.02844279818236828, 0.0, 0.08572126179933548, 0.0, 0.045171596109867096, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.5681443810462952, 0.1143999993801117, 0.015524080954492092, 0.0, 0.0, 0.0, 0.5270103216171265, 0.0, 0.0037885019555687904, 0.0, 0.0, 0.07258697599172592, 0.0, 0.0, 0.0, 0.1611885279417038, 0.0, 0.0, 0.11819115281105042, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.061001136898994446, 0.014357292093336582, 0.026383068412542343]]]},\n            {\n            'hltrCFG': {'tokenization_config': {\"token_prefix\": \"\\u0120\", \"partial_token_prefix\": \"##\"}\n                }\n            })\n         }, function (err) {\n            console.log(err);\n        })"},"metadata":{}}]},{"cell_type":"markdown","source":"### Factorizing Activations of a Single Layer","metadata":{}},{"cell_type":"code","source":"# Factorize the activations of only the first layer\nnmf_2 = output.run_nmf(n_components=10, from_layer=0, to_layer=1)\nnmf_2.explore()","metadata":{"execution":{"iopub.status.busy":"2023-05-31T18:45:48.349500Z","iopub.execute_input":"2023-05-31T18:45:48.350279Z","iopub.status.idle":"2023-05-31T18:45:48.593156Z","shell.execute_reply.started":"2023-05-31T18:45:48.350236Z","shell.execute_reply":"2023-05-31T18:45:48.591671Z"},"trusted":true},"execution_count":74,"outputs":[{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"<html lang=\"en\">\n<script src=\"https://requirejs.org/docs/release/2.3.6/minified/require.js\"></script>\n<script>\n    var ecco_url = 'https://storage.googleapis.com/ml-intro/ecco/'\n    //var ecco_url = 'http://localhost:8000/'\n\n    if (window.ecco === undefined) window.ecco = {}\n\n    // Setup the paths of the script we'll be using\n    requirejs.config({\n        urlArgs: \"bust=\" + (new Date()).getTime(),\n        nodeRequire: require,\n        paths: {\n            d3: \"https://d3js.org/d3.v6.min\", // This is only for use in setup.html and basic.html\n            \"d3-array\": \"https://d3js.org/d3-array.v2.min\",\n            jquery: \"https://code.jquery.com/jquery-3.5.1.min\",\n            ecco: ecco_url + 'js/0.0.6/ecco-bundle.min',\n            xregexp: 'https://cdnjs.cloudflare.com/ajax/libs/xregexp/3.2.0/xregexp-all.min'\n        }\n    });\n\n    // Add the css file\n    //requirejs(['d3'],\n    //    function (d3) {\n    //        d3.select('#css').attr('href', ecco_url + 'html/styles.css')\n    //    })\n\n    console.log('Ecco initialize!!')\n\n    // returns a 'basic' object. basic.init() selects the html div we'll be\n    // rendering the html into, adds styles.css to the document.\n    define('basic', ['d3'],\n        function (d3) {\n            return {\n                init: function (viz_id = null) {\n                    if (viz_id == null) {\n                        viz_id = \"viz_\" + Math.round(Math.random() * 10000000)\n                    }\n                    // Select the div rendered below, change its id\n                    const div = d3.select('#basic').attr('id', viz_id),\n                        div_parent = d3.select('#' + viz_id).node().parentNode\n\n                    // Link to CSS file\n                    d3.select(div_parent).insert('link')\n                        .attr('rel', 'stylesheet')\n                        .attr('type', 'text/css')\n                        .attr('href', ecco_url + 'html/0.0.2/styles.css')\n\n                    return viz_id\n                }\n            }\n        }, function (err) {\n            console.log(err);\n        }\n    )\n</script>\n\n<head>\n    <link id='css' rel=\"stylesheet\" type=\"text/css\">\n</head>\n<div id=\"basic\"></div>\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.Javascript object>","application/javascript":"\n         requirejs(['basic', 'ecco'], function(basic, ecco){\n            const viz_id = basic.init()\n            \n            ecco.interactiveTokensAndFactorSparklines(viz_id, {'tokens': [{'token': '<s>', 'token_id': 0, 'type': 'input', 'position': 0}, {'token': 'ĠThe', 'token_id': 20, 'type': 'input', 'position': 1}, {'token': 'Ġvast', 'token_id': 4714, 'type': 'input', 'position': 2}, {'token': 'Ġexp', 'token_id': 4553, 'type': 'input', 'position': 3}, {'token': 'anse', 'token_id': 31327, 'type': 'input', 'position': 4}, {'token': 'Ġof', 'token_id': 9, 'type': 'input', 'position': 5}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 6}, {'token': 'Ġocean', 'token_id': 6444, 'type': 'input', 'position': 7}, {'token': 'Ġstretches', 'token_id': 15706, 'type': 'input', 'position': 8}, {'token': 'Ġout', 'token_id': 66, 'type': 'input', 'position': 9}, {'token': 'Ġbefore', 'token_id': 137, 'type': 'input', 'position': 10}, {'token': 'Ġme', 'token_id': 162, 'type': 'input', 'position': 11}, {'token': ',', 'token_id': 6, 'type': 'input', 'position': 12}, {'token': 'Ġits', 'token_id': 63, 'type': 'input', 'position': 13}, {'token': 'Ġcalm', 'token_id': 6327, 'type': 'input', 'position': 14}, {'token': 'Ġwaters', 'token_id': 5794, 'type': 'input', 'position': 15}, {'token': 'Ġreflecting', 'token_id': 10811, 'type': 'input', 'position': 16}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 17}, {'token': 'Ġsky', 'token_id': 6360, 'type': 'input', 'position': 18}, {'token': 'Ġabove', 'token_id': 1065, 'type': 'input', 'position': 19}, {'token': '.', 'token_id': 4, 'type': 'input', 'position': 20}, {'token': 'Ċ', 'token_id': 50118, 'type': 'input', 'position': 21}, {'token': 'Its', 'token_id': 30872, 'type': 'input', 'position': 22}, {'token': 'Ġtranquil', 'token_id': 33535, 'type': 'input', 'position': 23}, {'token': 'Ġsurface', 'token_id': 4084, 'type': 'input', 'position': 24}, {'token': ',', 'token_id': 6, 'type': 'input', 'position': 25}, {'token': 'Ġund', 'token_id': 2432, 'type': 'input', 'position': 26}, {'token': 'ist', 'token_id': 661, 'type': 'input', 'position': 27}, {'token': 'urbed', 'token_id': 35250, 'type': 'input', 'position': 28}, {'token': 'Ġby', 'token_id': 30, 'type': 'input', 'position': 29}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 30}, {'token': 'Ġchaos', 'token_id': 7724, 'type': 'input', 'position': 31}, {'token': 'Ġof', 'token_id': 9, 'type': 'input', 'position': 32}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 33}, {'token': 'Ġworld', 'token_id': 232, 'type': 'input', 'position': 34}, {'token': ',', 'token_id': 6, 'type': 'input', 'position': 35}, {'token': 'Ġinspires', 'token_id': 25362, 'type': 'input', 'position': 36}, {'token': 'Ġa', 'token_id': 10, 'type': 'input', 'position': 37}, {'token': 'Ġsense', 'token_id': 1472, 'type': 'input', 'position': 38}, {'token': 'Ġof', 'token_id': 9, 'type': 'input', 'position': 39}, {'token': 'Ġse', 'token_id': 842, 'type': 'input', 'position': 40}, {'token': 'rene', 'token_id': 20962, 'type': 'input', 'position': 41}, {'token': 'Ġtranqu', 'token_id': 36474, 'type': 'input', 'position': 42}, {'token': 'ility', 'token_id': 13005, 'type': 'input', 'position': 43}, {'token': '.', 'token_id': 4, 'type': 'input', 'position': 44}, {'token': 'Ċ', 'token_id': 50118, 'type': 'input', 'position': 45}, {'token': 'The', 'token_id': 133, 'type': 'input', 'position': 46}, {'token': 'Ġsoft', 'token_id': 3793, 'type': 'input', 'position': 47}, {'token': 'Ġwhisper', 'token_id': 37539, 'type': 'input', 'position': 48}, {'token': 'Ġof', 'token_id': 9, 'type': 'input', 'position': 49}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 50}, {'token': 'Ġwaves', 'token_id': 6995, 'type': 'input', 'position': 51}, {'token': 'Ġkissing', 'token_id': 20479, 'type': 'input', 'position': 52}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 53}, {'token': 'Ġshore', 'token_id': 8373, 'type': 'input', 'position': 54}, {'token': 'line', 'token_id': 1902, 'type': 'input', 'position': 55}, {'token': 'Ġis', 'token_id': 16, 'type': 'input', 'position': 56}, {'token': 'Ġa', 'token_id': 10, 'type': 'input', 'position': 57}, {'token': 'Ġpeaceful', 'token_id': 7053, 'type': 'input', 'position': 58}, {'token': 'Ġsym', 'token_id': 22850, 'type': 'input', 'position': 59}, {'token': 'phony', 'token_id': 29788, 'type': 'input', 'position': 60}, {'token': ',', 'token_id': 6, 'type': 'input', 'position': 61}, {'token': 'Ġa', 'token_id': 10, 'type': 'input', 'position': 62}, {'token': 'Ġmel', 'token_id': 15352, 'type': 'input', 'position': 63}, {'token': 'od', 'token_id': 1630, 'type': 'input', 'position': 64}, {'token': 'ious', 'token_id': 6514, 'type': 'input', 'position': 65}, {'token': 'Ġsong', 'token_id': 2214, 'type': 'input', 'position': 66}, {'token': 'Ġof', 'token_id': 9, 'type': 'input', 'position': 67}, {'token': 'Ċ', 'token_id': 50118, 'type': 'input', 'position': 68}, {'token': 'nature', 'token_id': 43102, 'type': 'input', 'position': 69}, {'token': 'Ġthat', 'token_id': 14, 'type': 'input', 'position': 70}, {'token': 'Ġso', 'token_id': 98, 'type': 'input', 'position': 71}, {'token': 'othes', 'token_id': 32915, 'type': 'input', 'position': 72}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 73}, {'token': 'Ġsoul', 'token_id': 7047, 'type': 'input', 'position': 74}, {'token': '.', 'token_id': 4, 'type': 'input', 'position': 75}, {'token': 'ĠThe', 'token_id': 20, 'type': 'input', 'position': 76}, {'token': 'Ġgentle', 'token_id': 16634, 'type': 'input', 'position': 77}, {'token': ',', 'token_id': 6, 'type': 'input', 'position': 78}, {'token': 'Ġrhyth', 'token_id': 41063, 'type': 'input', 'position': 79}, {'token': 'mic', 'token_id': 15796, 'type': 'input', 'position': 80}, {'token': 'Ġmotion', 'token_id': 4298, 'type': 'input', 'position': 81}, {'token': 'Ġof', 'token_id': 9, 'type': 'input', 'position': 82}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 83}, {'token': 'Ġocean', 'token_id': 6444, 'type': 'input', 'position': 84}, {'token': 'âĢ', 'token_id': 17, 'type': 'input', 'position': 85}, {'token': 'Ļ', 'token_id': 27, 'type': 'input', 'position': 86}, {'token': 's', 'token_id': 29, 'type': 'input', 'position': 87}, {'token': 'Ġcurrents', 'token_id': 28124, 'type': 'input', 'position': 88}, {'token': 'Ġembodies', 'token_id': 33434, 'type': 'input', 'position': 89}, {'token': 'Ġa', 'token_id': 10, 'type': 'input', 'position': 90}, {'token': 'Ġsilent', 'token_id': 8454, 'type': 'input', 'position': 91}, {'token': 'Ċ', 'token_id': 50118, 'type': 'input', 'position': 92}, {'token': 'strength', 'token_id': 33687, 'type': 'input', 'position': 93}, {'token': ',', 'token_id': 6, 'type': 'input', 'position': 94}, {'token': 'Ġa', 'token_id': 10, 'type': 'input', 'position': 95}, {'token': 'Ġsense', 'token_id': 1472, 'type': 'input', 'position': 96}, {'token': 'Ġof', 'token_id': 9, 'type': 'input', 'position': 97}, {'token': 'Ġenduring', 'token_id': 16480, 'type': 'input', 'position': 98}, {'token': 'Ġse', 'token_id': 842, 'type': 'input', 'position': 99}, {'token': 'ren', 'token_id': 2558, 'type': 'input', 'position': 100}, {'token': 'ity', 'token_id': 1571, 'type': 'input', 'position': 101}, {'token': 'Ġthat', 'token_id': 14, 'type': 'input', 'position': 102}, {'token': 'Ġis', 'token_id': 16, 'type': 'input', 'position': 103}, {'token': 'Ġprofoundly', 'token_id': 27301, 'type': 'input', 'position': 104}, {'token': 'Ġpeaceful', 'token_id': 7053, 'type': 'input', 'position': 105}, {'token': '.', 'token_id': 4, 'type': 'input', 'position': 106}, {'token': 'ĠThe', 'token_id': 20, 'type': 'input', 'position': 107}, {'token': 'Ġbound', 'token_id': 8191, 'type': 'input', 'position': 108}, {'token': 'less', 'token_id': 1672, 'type': 'input', 'position': 109}, {'token': 'Ġbeauty', 'token_id': 4002, 'type': 'input', 'position': 110}, {'token': 'Ġof', 'token_id': 9, 'type': 'input', 'position': 111}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 112}, {'token': 'Ġsea', 'token_id': 3342, 'type': 'input', 'position': 113}, {'token': ',', 'token_id': 6, 'type': 'input', 'position': 114}, {'token': 'Ċ', 'token_id': 50118, 'type': 'input', 'position': 115}, {'token': 'with', 'token_id': 5632, 'type': 'input', 'position': 116}, {'token': 'Ġits', 'token_id': 63, 'type': 'input', 'position': 117}, {'token': 'Ġcapt', 'token_id': 13363, 'type': 'input', 'position': 118}, {'token': 'ivating', 'token_id': 28191, 'type': 'input', 'position': 119}, {'token': 'Ġcalm', 'token_id': 6327, 'type': 'input', 'position': 120}, {'token': 'ness', 'token_id': 1825, 'type': 'input', 'position': 121}, {'token': ',', 'token_id': 6, 'type': 'input', 'position': 122}, {'token': 'Ġoffers', 'token_id': 1523, 'type': 'input', 'position': 123}, {'token': 'Ġa', 'token_id': 10, 'type': 'input', 'position': 124}, {'token': 'Ġsanctuary', 'token_id': 13488, 'type': 'input', 'position': 125}, {'token': 'Ġfrom', 'token_id': 31, 'type': 'input', 'position': 126}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 127}, {'token': 'Ġbustling', 'token_id': 29019, 'type': 'input', 'position': 128}, {'token': 'Ġnoise', 'token_id': 6496, 'type': 'input', 'position': 129}, {'token': 'Ġof', 'token_id': 9, 'type': 'input', 'position': 130}, {'token': 'Ġeveryday', 'token_id': 7476, 'type': 'input', 'position': 131}, {'token': 'Ġlife', 'token_id': 301, 'type': 'input', 'position': 132}, {'token': ',', 'token_id': 6, 'type': 'input', 'position': 133}, {'token': 'Ġa', 'token_id': 10, 'type': 'input', 'position': 134}, {'token': 'Ġhaven', 'token_id': 2220, 'type': 'input', 'position': 135}, {'token': 'Ċ', 'token_id': 50118, 'type': 'input', 'position': 136}, {'token': 'where', 'token_id': 8569, 'type': 'input', 'position': 137}, {'token': 'Ġone', 'token_id': 65, 'type': 'input', 'position': 138}, {'token': 'Ġcan', 'token_id': 64, 'type': 'input', 'position': 139}, {'token': 'Ġfind', 'token_id': 465, 'type': 'input', 'position': 140}, {'token': 'Ġpeace', 'token_id': 1987, 'type': 'input', 'position': 141}, {'token': 'Ġand', 'token_id': 8, 'type': 'input', 'position': 142}, {'token': 'Ġsolitude', 'token_id': 41813, 'type': 'input', 'position': 143}, {'token': '.', 'token_id': 4, 'type': 'input', 'position': 144}, {'token': 'ĠIt', 'token_id': 85, 'type': 'input', 'position': 145}, {'token': 'âĢ', 'token_id': 17, 'type': 'input', 'position': 146}, {'token': 'Ļ', 'token_id': 27, 'type': 'input', 'position': 147}, {'token': 's', 'token_id': 29, 'type': 'input', 'position': 148}, {'token': 'Ġa', 'token_id': 10, 'type': 'input', 'position': 149}, {'token': 'Ġplace', 'token_id': 317, 'type': 'input', 'position': 150}, {'token': 'Ġwhere', 'token_id': 147, 'type': 'input', 'position': 151}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 152}, {'token': 'Ġmind', 'token_id': 1508, 'type': 'input', 'position': 153}, {'token': 'Ġcan', 'token_id': 64, 'type': 'input', 'position': 154}, {'token': 'Ġrest', 'token_id': 1079, 'type': 'input', 'position': 155}, {'token': ',', 'token_id': 6, 'type': 'input', 'position': 156}, {'token': 'Ġallowing', 'token_id': 2455, 'type': 'input', 'position': 157}, {'token': 'Ġthoughts', 'token_id': 4312, 'type': 'input', 'position': 158}, {'token': 'Ġto', 'token_id': 7, 'type': 'input', 'position': 159}, {'token': 'Ċ', 'token_id': 50118, 'type': 'input', 'position': 160}, {'token': 'flow', 'token_id': 19322, 'type': 'input', 'position': 161}, {'token': 'Ġas', 'token_id': 25, 'type': 'input', 'position': 162}, {'token': 'Ġfreely', 'token_id': 13215, 'type': 'input', 'position': 163}, {'token': 'Ġas', 'token_id': 25, 'type': 'input', 'position': 164}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 165}, {'token': 'Ġwaters', 'token_id': 5794, 'type': 'input', 'position': 166}, {'token': 'Ġthemselves', 'token_id': 1235, 'type': 'input', 'position': 167}, {'token': '.', 'token_id': 4, 'type': 'input', 'position': 168}, {'token': 'ĠThe', 'token_id': 20, 'type': 'input', 'position': 169}, {'token': 'Ġocean', 'token_id': 6444, 'type': 'input', 'position': 170}, {'token': ',', 'token_id': 6, 'type': 'input', 'position': 171}, {'token': 'Ġin', 'token_id': 11, 'type': 'input', 'position': 172}, {'token': 'Ġits', 'token_id': 63, 'type': 'input', 'position': 173}, {'token': 'Ġinfinite', 'token_id': 32952, 'type': 'input', 'position': 174}, {'token': 'Ġtranqu', 'token_id': 36474, 'type': 'input', 'position': 175}, {'token': 'ility', 'token_id': 13005, 'type': 'input', 'position': 176}, {'token': ',', 'token_id': 6, 'type': 'input', 'position': 177}, {'token': 'Ġserves', 'token_id': 4542, 'type': 'input', 'position': 178}, {'token': 'Ġas', 'token_id': 25, 'type': 'input', 'position': 179}, {'token': 'Ġa', 'token_id': 10, 'type': 'input', 'position': 180}, {'token': 'Ġreminder', 'token_id': 8306, 'type': 'input', 'position': 181}, {'token': 'Ċ', 'token_id': 50118, 'type': 'input', 'position': 182}, {'token': 'of', 'token_id': 1116, 'type': 'input', 'position': 183}, {'token': 'Ġthe', 'token_id': 5, 'type': 'input', 'position': 184}, {'token': 'Ġsimple', 'token_id': 2007, 'type': 'input', 'position': 185}, {'token': 'Ġand', 'token_id': 8, 'type': 'input', 'position': 186}, {'token': 'Ġquiet', 'token_id': 5128, 'type': 'input', 'position': 187}, {'token': 'Ġbeauty', 'token_id': 4002, 'type': 'input', 'position': 188}, {'token': 'Ġthat', 'token_id': 14, 'type': 'input', 'position': 189}, {'token': 'Ġexists', 'token_id': 8785, 'type': 'input', 'position': 190}, {'token': 'Âł', 'token_id': 50141, 'type': 'input', 'position': 191}, {'token': 'in', 'token_id': 179, 'type': 'input', 'position': 192}, {'token': 'Âł', 'token_id': 50141, 'type': 'input', 'position': 193}, {'token': 'our', 'token_id': 2126, 'type': 'input', 'position': 194}, {'token': 'Âł', 'token_id': 50141, 'type': 'input', 'position': 195}, {'token': 'world', 'token_id': 8331, 'type': 'input', 'position': 196}, {'token': '.', 'token_id': 4, 'type': 'input', 'position': 197}, {'token': 'Ċ', 'token_id': 50118, 'type': 'input', 'position': 198}, {'token': '</s>', 'token_id': 2, 'type': 'input', 'position': 199}], 'factors': [[[0.0, 0.13278378546237946, 0.26212409138679504, 0.356606125831604, 0.1796877086162567, 0.2500709295272827, 0.1712968796491623, 0.07714884728193283, 0.5250862836837769, 0.3681563436985016, 0.40387359261512756, 0.4038686454296112, 0.16234703361988068, 0.5493131875991821, 0.3106575608253479, 0.1470707505941391, 0.6262395977973938, 0.19043949246406555, 0.24718914926052094, 0.6757633090019226, 0.13922707736492157, 0.028096137568354607, 0.3363933265209198, 0.20055098831653595, 0.31857454776763916, 0.08053791522979736, 0.09184930473566055, 0.0, 0.08870801329612732, 0.14799532294273376, 0.0, 0.1523151993751526, 0.032192815095186234, 0.0, 0.0, 0.0, 0.21598225831985474, 0.0, 0.04757178947329521, 0.0, 0.0, 0.1037604957818985, 0.00010446396481711417, 0.009447518736124039, 0.039244409650564194, 0.0, 0.06632725149393082, 0.2433377355337143, 0.35085582733154297, 0.17992381751537323, 0.10775965452194214, 0.13132858276367188, 0.41905051469802856, 0.14662426710128784, 0.1435103565454483, 0.20132611691951752, 0.12129838019609451, 0.023308392614126205, 0.045860614627599716, 0.09927155822515488, 0.10606292635202408, 0.007653013803064823, 0.08503836393356323, 0.0, 0.0, 0.0, 0.11141043901443481, 0.0, 0.0, 0.07256810367107391, 0.0, 0.04031288996338844, 0.17615912854671478, 0.0, 0.0, 0.04885110259056091, 0.02627795934677124, 0.09524070471525192, 0.020353443920612335, 0.1543515920639038, 0.08780738711357117, 0.2524307668209076, 0.04747259244322777, 0.0, 0.0, 0.0, 0.0, 0.0, 0.1512775719165802, 0.1320994794368744, 0.0, 0.08757922053337097, 0.0, 0.07862897962331772, 0.0, 0.0, 0.0019402948673814535, 0.0, 0.0, 0.0, 0.03847048431634903, 0.0, 0.0, 0.0, 0.007290931418538094, 0.0, 0.0, 0.0, 0.10015850514173508, 0.008727719075977802, 0.08387584984302521, 0.0, 0.0, 0.0, 0.0, 0.0, 0.13211697340011597, 0.22366948425769806, 0.011328866705298424, 0.04438154399394989, 0.0, 0.004039115738123655, 0.0, 0.22896654903888702, 0.0, 0.0, 0.034421470016241074, 0.0, 0.017642544582486153, 0.07721357047557831, 0.0, 0.0, 0.0, 0.0, 0.0, 0.01334349811077118, 0.0, 0.061752717941999435, 0.010484443046152592, 0.041241999715566635, 0.01826140470802784, 0.0, 0.0, 0.0012732421746477485, 0.0, 0.06033090502023697, 0.0, 0.0, 0.0, 0.0, 0.0, 0.11509035527706146, 0.0, 0.0, 0.06353519856929779, 0.07759293913841248, 0.0, 0.13234253227710724, 0.07043508440256119, 0.0, 0.0, 0.11265911906957626, 0.016719890758395195, 0.10347390174865723, 0.027704648673534393, 0.0, 0.0, 0.15736530721187592, 0.0, 0.0, 0.0, 0.0, 0.06098335236310959, 0.13340412080287933, 0.0, 0.0, 0.0, 0.0, 0.14813962578773499, 0.0, 0.0, 0.06476908177137375, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0579143762588501, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.06887037307024002, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.1269489824771881, 0.17092914879322052, 0.06891942024230957, 1.0853883028030396, 0.026441635563969612, 0.0, 0.0, 0.0076301549561321735, 0.023400919511914253, 0.0, 0.0, 0.2067456692457199, 0.0494009368121624, 0.0, 0.0, 0.0, 0.999954104423523, 0.0, 0.021629145368933678, 0.0, 0.12872987985610962, 0.0, 0.0, 0.0, 0.0, 0.10115594416856766, 0.9955722093582153, 0.017244847491383553, 0.01431487500667572, 0.029629386961460114, 0.01982027105987072, 0.0, 0.0, 0.0, 0.0282636359333992, 0.17157024145126343, 0.041003115475177765, 0.03843844681978226, 0.029585517942905426, 0.0, 0.009295079857110977, 0.03833021596074104, 0.0, 0.0, 0.06734450161457062, 0.0, 0.03797519579529762, 0.1125025600194931, 0.0, 0.0, 0.020945874974131584, 0.03442929685115814, 1.0121936798095703, 0.038648828864097595, 0.0, 0.01223684847354889, 0.02855139970779419, 0.011790338903665543, 0.0, 0.011570092290639877, 0.0, 0.06376320868730545, 0.07144512981176376, 0.009303226135671139, 1.589341445651371e-05, 0.04260597378015518, 0.18831327557563782, 0.03676629438996315, 0.02087186835706234, 0.985172688961029, 0.0007049543200992048, 0.0, 0.03334515914320946, 0.0, 0.00375529401935637, 0.0, 0.03348901495337486, 0.0, 0.0, 0.0, 0.0, 0.05385948717594147, 0.0, 0.0823967382311821, 0.010149543173611164, 0.9101302027702332, 0.012669089250266552, 0.023755384609103203, 0.0, 0.0005288292304612696, 0.0, 0.00394820561632514, 0.07456838339567184, 0.09131195396184921, 0.014381038025021553, 0.0, 0.0, 0.1438719928264618, 0.014736403711140156, 0.0, 0.033312030136585236, 0.05851772800087929, 0.0026635434478521347, 0.0, 0.0, 0.9541134834289551, 0.033760376274585724, 0.013450074009597301, 0.03317955136299133, 0.0, 0.0, 0.01993497647345066, 0.17100681364536285, 0.9743090867996216, 0.0, 0.0, 0.0, 0.04301077872514725, 0.0, 0.0, 0.04091508314013481, 0.0, 0.0, 0.04764631763100624, 0.8234854340553284, 0.0, 0.060379575937986374, 0.0, 0.0, 0.0, 0.0028345950413495302, 0.0, 0.0, 0.26451587677001953, 0.0, 0.07441823929548264, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.03607653081417084, 0.011326177045702934, 0.0, 0.8815006017684937, 0.0, 0.0, 0.0, 0.0, 0.0, 0.017257805913686752, 0.0, 0.056979838758707047, 0.0, 0.0, 0.0022301916033029556, 0.12387999892234802, 0.0, 0.0, 0.8829333782196045, 0.037201326340436935, 0.0, 0.0, 0.0, 0.0, 0.8120481967926025, 0.0, 0.052783723920583725, 0.0, 0.0, 0.020949212834239006, 0.0, 0.0, 0.0, 0.36279040575027466, 0.0, 0.02543043904006481, 0.047876328229904175, 0.0, 0.04193280637264252, 0.0, 0.031184719875454903, 0.0, 0.012839404866099358, 0.0, 0.012403957545757294, 0.0, 0.03907786309719086], [0.02476179413497448, 0.0, 0.0, 0.21440108120441437, 0.1879688799381256, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.00887210387736559, 0.0, 0.018141862004995346, 0.023190148174762726, 0.5488049387931824, 0.529609203338623, 0.08833145350217819, 0.08094139397144318, 0.004396539181470871, 0.013398881070315838, 0.0, 0.0, 0.0, 0.0, 0.0, 0.015118969604372978, 0.0, 0.06808318942785263, 0.6910041570663452, 0.39212745428085327, 0.08101287484169006, 0.00044384720968082547, 0.01067374274134636, 0.0, 0.013300961814820766, 0.1078074499964714, 0.10940425097942352, 0.0, 0.005748534109443426, 0.07441280782222748, 0.06032966077327728, 0.0, 0.037018682807683945, 0.09260369837284088, 0.0, 0.004510001745074987, 0.0, 0.2529209852218628, 0.1647859513759613, 0.0, 0.0, 0.5665619373321533, 1.062696099281311, 0.13094021379947662, 0.15283364057540894, 0.0, 0.02731706202030182, 0.059892687946558, 0.03891725838184357, 0.5733210444450378, 0.11474709957838058, 0.0, 0.0075593385845422745, 0.02151172049343586, 0.036130212247371674, 0.0, 0.047380175441503525, 0.3069911003112793, 0.1915922313928604, 0.08542922884225845, 0.0, 0.0, 0.0, 0.0, 0.014101477339863777, 0.0, 0.07173763960599899, 0.011695575900375843, 0.029498737305402756, 0.040483493357896805, 0.009828545153141022, 0.0, 0.0, 0.023418491706252098, 0.0, 0.0661182701587677, 0.03841081261634827, 0.6312713623046875, 0.6598029732704163, 0.06303095072507858, 0.026271414011716843, 0.06811871379613876, 0.20512083172798157, 0.0, 0.03190097212791443, 0.034485869109630585, 0.5528063774108887, 0.08421983569860458, 0.0, 0.00028536387253552675, 0.023735670372843742, 0.0, 0.0, 0.004290014039725065, 0.05830550938844681, 0.012830209918320179, 0.6503215432167053, 0.13279852271080017, 0.039863813668489456, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.014347556047141552, 0.012094837613403797, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0335288904607296, 0.0, 0.012416106648743153, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.1268300712108612, 0.0, 0.025702668353915215, 0.0, 0.0013279387494549155, 0.0, 0.0, 0.03238555043935776, 0.0, 0.03577012941241264, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.05030781775712967, 0.019760193303227425, 0.03515920788049698, 0.0, 0.0, 0.011100053787231445, 0.0, 0.0, 0.0, 0.0, 0.02833389677107334, 0.0, 0.0, 0.1000605970621109, 0.0, 0.0, 0.0, 0.0, 0.1165679320693016, 0.12674950063228607, 0.10005563497543335, 0.07959624379873276, 0.09238698333501816, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.24064743518829346, 0.11170311272144318, 0.055592190474271774, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.07744954526424408, 0.0, 0.05384298413991928, 0.44626176357269287, 0.011422770097851753, 0.13400717079639435, 0.0, 0.04900119826197624, 0.0, 0.0, 0.0, 0.03760770708322525, 0.3465205729007721, 0.014581472612917423, 0.0, 0.0, 0.0, 0.31488874554634094, 0.020530790090560913, 0.0, 0.5587190985679626, 0.0, 0.0039205183275043964, 0.3190418779850006, 0.007557469420135021, 0.1543891876935959, 0.06366276741027832, 0.35135960578918457, 0.09158269315958023, 0.0, 0.27670446038246155, 0.43937766551971436, 0.7086942195892334, 0.0, 0.0, 0.0053788465447723866, 0.3533398509025574, 0.3707493543624878, 0.0, 0.0, 0.054636333137750626, 0.06468352675437927, 0.0, 0.0032682938035577536, 0.07389579713344574, 0.0, 0.010506085120141506, 0.6332706809043884, 0.16675886511802673, 0.4127807319164276, 0.0, 0.0, 0.11896029859781265, 0.0, 0.24954424798488617, 0.2556573748588562, 0.0, 0.0017856440972536802, 0.2563090920448303, 0.03419381007552147, 0.0, 0.21827182173728943, 0.012298212386667728, 0.39201876521110535, 0.0, 0.049198880791664124, 0.438732385635376, 0.019752489402890205, 0.17077672481536865, 0.3336241841316223, 0.37439119815826416, 0.0, 0.0, 0.0, 0.0, 0.03953416272997856, 0.0, 0.21545930206775665, 0.1808146834373474, 0.1128697544336319, 0.5497341156005859, 0.04152451455593109, 0.49962717294692993, 0.0, 0.05027034506201744, 0.38767826557159424, 0.10601811856031418, 0.5152539610862732, 0.0, 0.09318360686302185, 0.4317503571510315, 0.11559214442968369, 0.11530494689941406, 0.38709738850593567, 0.7601490616798401, 0.032810769975185394, 0.0727192610502243, 0.25265195965766907, 0.33536529541015625, 0.47077304124832153, 0.0163998082280159, 0.024093670770525932, 0.0, 0.02735247276723385, 0.0218631811439991, 0.08119428902864456, 0.10089635848999023, 0.24440746009349823, 0.31982773542404175, 0.7314355969429016, 0.5420873165130615, 0.0, 0.07231888175010681, 0.0, 0.310800164937973, 0.0, 0.0, 0.38062408566474915, 0.4515111744403839, 0.0, 0.40835314989089966, 0.2813667356967926, 0.0, 0.0, 0.15383166074752808, 0.0, 0.07996557652950287, 0.0878315195441246, 0.0, 0.06708705425262451, 0.586751401424408, 0.0, 0.43333742022514343, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.11823561042547226, 0.0, 0.0, 0.20277677476406097, 0.0, 0.17508840560913086, 0.0, 0.2163981795310974, 0.22895167768001556, 0.0, 0.0, 0.2796376645565033, 0.03794572502374649, 0.3875495493412018, 0.026556313037872314, 0.0, 0.05398965999484062, 0.07124273478984833, 0.0, 0.0, 0.0, 0.0, 0.0, 0.11336026340723038, 0.44847527146339417, 0.4024244248867035, 0.5988922119140625, 0.0, 0.15649710595607758, 0.01257910206913948, 0.0, 0.3038111627101898, 0.0, 0.0, 0.0, 0.5209652781486511, 0.004321007523685694, 0.6304095387458801, 0.42049622535705566, 0.051159270107746124, 0.1716020256280899, 0.08460274338722229, 0.0, 0.050168994814157486, 0.0932764932513237, 0.03953665494918823, 0.13378754258155823, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.037747032940387726, 0.0, 0.013416698202490807, 0.014085797592997551, 0.165364608168602, 0.0, 0.0, 0.0, 0.029760248959064484, 0.008200930431485176, 0.0, 0.003132481127977371, 0.0, 0.07786974310874939, 0.016391616314649582, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.021594800055027008, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.7182362079620361, 0.01335205603390932, 0.042379043996334076, 0.0, 0.0, 0.0, 0.0, 0.015395419672131538, 0.0, 0.027779236435890198, 0.08017205446958542, 0.0, 0.003431516932323575, 0.0, 0.03360437974333763, 0.0, 0.0, 0.0, 0.0, 0.05661972612142563, 0.7261818647384644, 0.03737298399209976, 0.0, 0.009620586410164833, 0.030763985589146614, 0.7425814867019653, 0.00023058657825458795, 0.018925167620182037, 0.026762109249830246, 0.0, 0.0, 0.0, 0.0, 0.07032348960638046, 0.018176095560193062, 0.0, 0.024317508563399315, 0.0008970434428192675, 0.0, 0.006594094913452864, 0.01956651359796524, 0.0131461750715971, 0.0, 0.02122737467288971, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.009478899650275707, 0.0, 0.00778324855491519, 0.7556726336479187, 0.014956945553421974, 0.04929342493414879, 0.0023443521931767464, 0.027991218492388725, 0.7836634516716003, 0.015783311799168587, 0.032787494361400604, 0.02149093896150589, 0.0, 0.0, 0.0, 0.04775092750787735, 0.07604507356882095, 0.03700828552246094, 0.01684609055519104, 0.0005503959255293012, 0.0011438416549935937, 0.0, 0.0410943441092968, 0.0, 0.0, 0.0, 0.01439193356782198, 0.0, 0.029255211353302002, 0.0, 0.12124838680028915, 0.0, 0.015753747895359993, 0.0, 0.0, 0.0, 0.0, 0.6567580699920654, 0.0, 0.0, 0.010642996057868004, 0.0, 0.0, 0.0, 0.019056163728237152, 0.0, 0.0, 0.5942744612693787, 0.0, 0.0, 0.0, 0.041723668575286865, 0.0, 0.0071759894490242004, 0.0006369537441059947, 0.02669978328049183, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.6286287903785706, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.006133498623967171, 0.0, 0.017952660098671913, 0.0, 0.0, 0.018812168389558792, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.034849848598241806, 0.09065479785203934, 0.001996951177716255, 0.0, 0.0, 0.0, 0.0, 0.09260968863964081, 0.6226373910903931, 0.0, 0.01280460599809885, 0.0, 0.023818014189600945, 0.0, 0.06060250103473663, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.048048969358205795, 0.0, 0.0, 0.0, 0.030985437333583832, 0.0, 0.0, 0.0, 0.0, 0.07549455761909485, 0.10449453443288803, 0.12143615633249283, 0.0, 0.0, 0.0, 0.044566746801137924, 0.0, 0.0, 0.0, 0.014169946312904358, 0.14809733629226685, 0.32703250646591187, 0.05527487397193909, 0.0, 0.021562278270721436, 0.0, 0.02659738063812256, 0.03658544644713402, 0.0, 0.09488998353481293, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.07318593561649323, 0.0, 0.051195867359638214, 0.0, 0.010464470833539963, 0.02579120360314846, 0.0, 0.0, 0.1584341675043106, 0.3285596966743469, 0.031785476952791214, 0.0, 0.0, 0.0, 0.0, 0.0, 0.11444659531116486, 0.0, 0.0, 0.02531273290514946, 0.1271267533302307, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.06535399705171585, 0.018420008942484856, 0.039337389171123505, 0.2951054871082306, 0.09518148750066757, 0.25745660066604614, 0.15205276012420654, 0.08671187609434128, 0.06413806974887848, 0.03163846209645271, 0.14063270390033722, 0.036356519907712936, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.08595619350671768, 0.0, 0.0, 0.29790905117988586, 0.09241639077663422, 0.006219041533768177, 0.0, 0.05236674100160599, 0.017099278047680855, 0.0966833308339119, 0.025832857936620712, 0.012658655643463135, 0.048218075186014175, 0.23913466930389404, 0.24883806705474854, 0.039798036217689514, 0.01978985220193863, 0.1684802621603012, 0.07869195938110352, 0.0, 0.05979808792471886, 0.0, 0.007641317788511515, 0.0, 0.011656138114631176, 0.0, 0.3074216842651367, 0.16883637011051178, 0.042038485407829285, 0.0, 0.04113388806581497, 0.0, 0.023692404851317406, 0.029233217239379883, 0.25698062777519226, 0.08759044110774994, 0.11138687282800674, 0.2695833742618561, 0.09634935110807419, 0.061494551599025726, 0.02744394540786743, 0.08697786927223206, 0.18064524233341217, 0.1912316530942917, 0.109203040599823, 0.1383887231349945, 0.24008344113826752, 0.4020046889781952, 0.28907597064971924, 0.3708096146583557, 0.3736747205257416, 0.34100979566574097, 0.10041482001543045, 0.2773461937904358, 0.10880960524082184, 0.26655733585357666, 0.2773858308792114, 0.12300575524568558, 0.11185360699892044, 0.13924472033977509, 0.13539451360702515, 0.3325534164905548, 0.3436483144760132, 0.18540272116661072, 0.1974520981311798, 0.3495948016643524, 0.27244797348976135, 0.12268433719873428, 0.2747529447078705, 0.21993735432624817, 0.41468459367752075, 0.42347604036331177, 0.13151094317436218, 0.4706096351146698, 0.192174032330513, 0.42778268456459045, 0.18048575520515442, 0.13889293372631073, 0.21230074763298035, 0.30517035722732544, 0.19207054376602173, 0.09420468658208847, 0.12742745876312256, 0.3763723075389862, 0.150748610496521, 0.12777729332447052, 0.026706306263804436, 0.0682959258556366, 0.13749122619628906, 0.32488495111465454, 0.37901028990745544, 0.14010240137577057, 0.18550454080104828, 0.4082367420196533, 0.23024432361125946, 0.20973451435565948, 0.1872231364250183, 0.32540470361709595, 0.1292242556810379, 0.10538776963949203, 0.36497464776039124, 0.21064630150794983, 0.1596022993326187, 0.27712908387184143, 0.1726289689540863, 0.2363440841436386, 0.19211015105247498, 0.24931831657886505, 0.3465018570423126, 0.423959881067276, 0.3132511079311371], [0.0, 0.0, 0.0, 0.0, 0.0, 1.5067719221115112, 0.0, 0.0, 0.0, 0.26798999309539795, 0.16800959408283234, 0.0049475450068712234, 0.008164260536432266, 0.04177943989634514, 0.0, 0.0, 0.0, 0.07459528744220734, 0.0, 0.05784169211983681, 0.014070224948227406, 0.0, 0.0, 0.0, 0.0, 0.018912730738520622, 0.0, 0.12335850298404694, 0.05377030745148659, 0.38970762491226196, 0.03172832354903221, 0.0, 1.6235781908035278, 0.0, 0.058667175471782684, 0.0, 0.0, 0.06558258831501007, 0.05160655081272125, 1.5444285869598389, 0.0, 0.0, 0.0, 0.00981967244297266, 0.06340883672237396, 0.0, 0.049247801303863525, 0.011630203574895859, 0.0061905356124043465, 1.6249206066131592, 0.014488696120679379, 0.04609574005007744, 0.0, 0.031417250633239746, 0.0, 0.12132386118173599, 0.2195841521024704, 0.07377546280622482, 0.0, 0.0, 0.04255741462111473, 0.12930771708488464, 0.070682592689991, 0.0, 0.0, 0.07725543528795242, 0.3112761378288269, 1.4914653301239014, 0.0, 0.0, 0.4582832455635071, 0.09549950808286667, 0.0, 0.11106163263320923, 0.07215465605258942, 0.06796339899301529, 0.026417439803481102, 0.04263881966471672, 0.0, 0.0, 0.01918068528175354, 0.023057086393237114, 1.5821975469589233, 0.0, 0.0, 0.11183217167854309, 0.0, 0.038141120225191116, 0.019343428313732147, 0.0, 0.09644903987646103, 0.0, 0.011416223831474781, 0.0367591418325901, 0.08556582033634186, 0.041538503021001816, 0.06594426184892654, 1.4049092531204224, 0.0, 0.0, 0.08267766237258911, 0.15861424803733826, 0.3653261959552765, 0.07282939553260803, 0.0, 0.025761539116501808, 0.04678637161850929, 0.0, 0.10576055943965912, 0.12916956841945648, 0.0, 1.6430071592330933, 0.0, 0.0, 0.014653102494776249, 0.0, 0.1538408249616623, 0.0292552188038826, 0.0, 0.045105352997779846, 0.0, 0.13179822266101837, 0.0, 0.0, 0.0, 0.0, 0.4105921685695648, 0.008462407626211643, 0.0, 0.0, 1.4607535600662231, 0.0, 0.09446795284748077, 0.0003327990125399083, 0.0, 0.011310544796288013, 0.0, 0.0, 0.09014221280813217, 0.0, 0.017204802483320236, 0.014158234931528568, 0.13600507378578186, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.022365087643265724, 0.2171878069639206, 0.0, 0.0, 0.039594680070877075, 0.0, 0.0, 0.0, 0.009180138818919659, 0.07449840009212494, 0.0, 0.0, 0.08597195148468018, 0.0, 0.17881405353546143, 0.0, 0.0, 0.058796316385269165, 0.0, 0.0, 0.0, 0.0, 0.2706868350505829, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.11014378070831299, 0.0, 0.0, 0.0, 0.6838807463645935, 0.0, 0.0, 0.0313916951417923, 0.0, 0.0, 0.3072642683982849, 0.0, 0.0, 0.06803709268569946, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.009958110749721527, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.07054418325424194, 0.01934760808944702, 0.0, 0.023035869002342224, 0.0, 0.02666444703936577, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.04338439553976059, 0.0, 0.06484389305114746, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0022527149412781, 0.014124798588454723, 0.0, 0.0, 0.0, 0.0, 0.018409626558423042, 0.06801340728998184, 0.0, 0.09811755269765854, 0.0, 0.0, 0.0, 0.0, 0.0, 0.020054321736097336, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.04481678456068039, 0.35627058148384094, 0.03759847953915596, 0.004926938097923994, 0.07535012066364288, 0.04729153960943222, 0.020627181977033615, 0.0, 0.013839132152497768, 0.0, 0.09870783239603043, 0.03702717274427414, 0.045121025294065475, 0.03130943700671196, 0.020970622077584267, 0.01790463924407959, 0.0, 0.033372312784194946, 0.0, 0.013148020952939987, 0.11432576924562454, 0.08363933116197586, 0.039504509419202805, 0.03137792646884918, 0.011114634573459625, 0.0340452715754509, 0.0, 0.05953444540500641, 0.1246173307299614, 0.07790198177099228, 1.1393605470657349, 1.2024104595184326, 0.9038493037223816, 0.02563975378870964, 0.14460811018943787, 0.018982885405421257, 0.06176010146737099, 0.05883818864822388, 0.010412097908556461, 0.04496179521083832, 0.002795429900288582, 0.16038481891155243, 0.0227438323199749, 0.027342913672327995, 0.0, 0.0, 0.032740987837314606, 0.019650064408779144, 0.283687025308609, 0.0, 0.0, 0.08752025663852692, 0.04449279606342316, 0.0, 0.051549796015024185, 0.0, 0.019992854446172714, 0.0577094741165638, 0.004472836386412382, 0.01656094565987587, 0.00019967209664173424, 0.03746494650840759, 0.09699627012014389, 0.0, 0.006522510200738907, 0.0, 0.0006157378084026277, 0.0, 0.052244774997234344, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.03211819753050804, 0.040120549499988556, 0.0, 0.0, 0.002155736554414034, 0.0, 0.0, 0.0, 0.09408648312091827, 0.0, 0.0, 0.13535018265247345, 0.029527870938181877, 0.15628409385681152, 0.3203728497028351, 0.9057579040527344, 0.9719058871269226, 0.7880311012268066, 0.0028493949212133884, 0.0029002008959650993, 0.0, 0.0, 0.0, 0.15282312035560608, 0.002766729099676013, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.10811028629541397, 0.015110877342522144, 0.0, 0.0, 0.0, 0.0, 0.01821945793926716, 0.0, 0.0, 0.03325536474585533, 0.014002084732055664, 0.1237378865480423, 0.09229038655757904, 0.021034294739365578, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.05631024017930031, 0.12048449367284775, 0.08054477721452713, 0.1079036071896553, 0.028234822675585747, 0.1014828085899353, 0.0, 0.0, 0.0, 0.032010819762945175], [0.01016330812126398, 0.0, 0.07994617521762848, 0.2737716734409332, 0.24585199356079102, 0.0, 0.0490071102976799, 1.388919711112976, 0.234025239944458, 0.0, 0.0, 0.06012550741434097, 0.0, 0.0, 0.45220932364463806, 0.9705877304077148, 0.0, 0.0, 0.9075078964233398, 0.27726322412490845, 0.0, 0.0, 0.0, 0.1741354614496231, 0.8558353185653687, 0.03493652120232582, 0.013591919094324112, 0.0, 0.05981569364666939, 0.0, 0.0, 0.18169790506362915, 0.04874543473124504, 0.0883931815624237, 0.8097697496414185, 0.10346196591854095, 0.0, 0.0, 0.0962202325463295, 0.0, 0.4236951172351837, 0.0, 0.009158974513411522, 0.0, 0.04300661012530327, 0.029824335128068924, 0.0, 0.005941347684711218, 0.0, 0.0, 0.025385404005646706, 0.9878846406936646, 0.0, 0.04659784212708473, 0.9791423678398132, 0.42833688855171204, 0.0, 0.009761795401573181, 0.10439509898424149, 0.10066504031419754, 0.08122561126947403, 0.0, 0.0, 0.0, 0.0, 0.061080120503902435, 0.32207342982292175, 0.0, 0.018852300941944122, 0.04248485714197159, 0.02562183514237404, 0.0, 0.0, 0.0, 0.46612104773521423, 0.025674160569906235, 0.0, 0.1566755324602127, 0.0, 0.0, 0.06314647942781448, 0.2380582094192505, 0.0, 0.05587567016482353, 1.358317255973816, 0.08442851901054382, 0.004936151206493378, 0.04163653030991554, 0.5568544268608093, 0.0, 0.0, 0.05423395335674286, 0.0534709133207798, 0.0, 0.0, 0.0, 0.08500916510820389, 0.0, 0.0, 0.3210012912750244, 0.0, 0.0, 0.04564077407121658, 0.0, 0.0, 0.018864259123802185, 0.0043527004308998585, 0.0, 0.01021871529519558, 0.0, 0.0, 0.0, 0.04396020248532295, 1.434731125831604, 0.08804678916931152, 0.04195740446448326, 0.0, 0.0, 0.0, 0.0, 0.09683507680892944, 0.0, 0.0, 0.0, 0.0, 0.08007291704416275, 0.0, 0.0, 0.11819665879011154, 0.13022074103355408, 0.0, 0.09959490597248077, 0.3059696555137634, 0.0, 0.0, 0.15944790840148926, 0.0, 0.0011147373588755727, 0.08450313657522202, 0.021993141621351242, 0.0, 0.04834914207458496, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.3776468336582184, 0.020564820617437363, 0.0, 0.5098364949226379, 0.0, 0.0, 0.0, 0.0, 0.1455649584531784, 0.0, 0.0, 0.07371487468481064, 0.0, 0.0, 0.0, 0.0, 0.7648847103118896, 0.16136491298675537, 0.0, 0.0, 1.0087690353393555, 0.0005362307420000434, 0.0, 0.0, 0.06368616968393326, 0.05075998604297638, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.03650754317641258, 0.0, 0.0, 0.0, 0.0, 0.0, 0.003334980458021164, 0.027172813192009926, 0.0319097675383091, 0.0035695405676960945, 0.07974450290203094, 0.38449791073799133, 0.0, 0.0, 0.009954140521585941], [0.0, 0.41067928075790405, 0.11004066467285156, 0.0, 4.4379616156220436e-05, 0.012642036192119122, 0.6498333215713501, 0.003156297840178013, 0.0, 0.005707526579499245, 0.011853815987706184, 0.06743104010820389, 0.0, 0.11758365482091904, 0.0, 0.0, 0.0, 0.5623472929000854, 0.0, 0.0, 0.01751571334898472, 0.009694779291749, 0.05981304869055748, 0.021477900445461273, 0.0039039647672325373, 0.012599236331880093, 0.0, 0.03131648898124695, 0.00012296343629714102, 0.030130255967378616, 0.6289082765579224, 0.0, 0.009427014738321304, 0.6659696698188782, 0.035426653921604156, 0.006480798590928316, 0.0, 0.021167578175663948, 0.0, 0.0060516465455293655, 0.00046563189243897796, 0.0, 0.0, 0.0, 0.02407536283135414, 0.014088384807109833, 0.36092084646224976, 0.05119406431913376, 0.0, 0.0, 0.6249324679374695, 0.0, 0.0, 0.5307695269584656, 0.014502523466944695, 0.0, 0.02301466464996338, 0.007017237599939108, 0.054561324417591095, 0.02534303441643715, 0.0, 0.0, 0.002017674036324024, 0.0, 0.0014062491245567799, 0.015294679440557957, 0.0, 0.0, 0.011156370863318443, 0.0, 0.025536904111504555, 0.04359478875994682, 0.0, 0.5491288304328918, 0.006294705905020237, 0.027151964604854584, 0.38357001543045044, 0.027112387120723724, 0.024789730086922646, 0.00041114483610726893, 0.028396155685186386, 0.003357799956575036, 0.0, 0.628644585609436, 0.0, 0.0, 0.0, 0.040722329169511795, 0.0, 0.0, 0.0, 0.017213724553585052, 0.0, 0.0, 0.005399975460022688, 0.008086255751550198, 0.0, 0.0, 0.02154960110783577, 0.0, 0.0, 0.0, 0.019980980083346367, 0.05396033823490143, 0.029425054788589478, 0.009832898154854774, 0.026064105331897736, 0.32179537415504456, 0.0, 0.036608900874853134, 0.0, 0.0, 0.6235849857330322, 0.0, 0.002566863317042589, 0.020776385441422462, 0.0, 0.12106309086084366, 0.0, 0.0, 0.0, 0.005738129373639822, 0.0, 0.0, 0.0, 0.0, 0.014194808900356293, 0.5905365943908691, 0.05320793390274048, 0.0, 0.0, 0.03490009531378746, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0010473962174728513, 0.046245377510786057, 0.020423943176865578, 0.01581401377916336, 0.02056143805384636, 0.03417075052857399, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.016184139996767044, 0.4938567280769348, 0.0, 0.00021339775412343442, 0.0, 0.0, 0.0, 0.0, 0.004526297561824322, 0.0, 0.0, 0.023938728496432304, 0.005131401587277651, 0.0, 0.5154485702514648, 0.0, 0.017865722998976707, 0.0, 0.3199508488178253, 0.0, 0.0, 0.03506378456950188, 0.1896708756685257, 0.04639645293354988, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.5041667222976685, 0.0, 0.062197960913181305, 0.0, 0.0, 0.018159128725528717, 0.003862688783556223, 0.0, 0.020258639007806778, 0.0, 0.002788449404761195, 0.0, 0.0, 0.0, 0.0, 0.0011450236197561026]]]},\n            {\n            'hltrCFG': {'tokenization_config': {\"token_prefix\": \"\\u0120\", \"partial_token_prefix\": \"##\"}\n                }\n            })\n         }, function (err) {\n            console.log(err);\n        })"},"metadata":{}}]},{"cell_type":"markdown","source":"# Visualizing Attentions in Our Finetuned Model:\n - multiple views that each offer a unique lens into the attention mechanism.[2]","metadata":{}},{"cell_type":"code","source":"from bertviz import head_view, model_view\nfrom bertviz.neuron_view import show\nfrom IPython.display import IFrame","metadata":{"execution":{"iopub.status.busy":"2023-05-31T22:26:25.761310Z","iopub.execute_input":"2023-05-31T22:26:25.762346Z","iopub.status.idle":"2023-05-31T22:26:25.767655Z","shell.execute_reply.started":"2023-05-31T22:26:25.762296Z","shell.execute_reply":"2023-05-31T22:26:25.766349Z"},"trusted":true},"execution_count":14,"outputs":[]},{"cell_type":"markdown","source":"## Model V(Finetuned DistilBERT):","metadata":{}},{"cell_type":"code","source":"sentence_a = \"It was a boring day\"\nsentence_b = \"The ocean is so peaceful\"\ninputs = tokenizer_V.encode_plus(sentence_a, sentence_b, return_tensors='pt')\noutputs = model_V(**inputs)\nattention = outputs[-1]\ntokens = tokenizer_V.convert_ids_to_tokens(inputs[\"input_ids\"][0]) \nhtml_model_view = model_view(attention, tokens, html_action='return', display_mode=\"light\")\nhtml_head_view = head_view(attention, tokens, html_action='return')\n# model_view(attention, tokens, display_mode=\"light\")\nwith open('model_V_model_view.html', 'w') as file:\n    file.write(html_model_view.data)\nwith open('model_V_head_view.html', 'w') as file:\n    file.write(html_head_view.data)\nhtml_file_path = ['model_V_model_view.html','model_V_head_view.html']","metadata":{"execution":{"iopub.status.busy":"2023-05-31T22:26:39.285256Z","iopub.execute_input":"2023-05-31T22:26:39.285962Z","iopub.status.idle":"2023-05-31T22:26:39.423094Z","shell.execute_reply.started":"2023-05-31T22:26:39.285923Z","shell.execute_reply":"2023-05-31T22:26:39.422138Z"},"trusted":true},"execution_count":15,"outputs":[]},{"cell_type":"code","source":"display(IFrame(src=html_file_path[0], width='100%', height='500px'))","metadata":{"execution":{"iopub.status.busy":"2023-05-31T22:26:40.545760Z","iopub.execute_input":"2023-05-31T22:26:40.546126Z","iopub.status.idle":"2023-05-31T22:26:40.552462Z","shell.execute_reply.started":"2023-05-31T22:26:40.546096Z","shell.execute_reply":"2023-05-31T22:26:40.551506Z"},"trusted":true},"execution_count":16,"outputs":[{"output_type":"display_data","data":{"text/plain":"<IPython.lib.display.IFrame at 0x7db00bb16170>","text/html":"\n        <iframe\n            width=\"100%\"\n            height=\"500px\"\n            src=\"model_V_model_view.html\"\n            frameborder=\"0\"\n            allowfullscreen\n            \n        ></iframe>\n        "},"metadata":{}}]},{"cell_type":"code","source":"display(IFrame(src=html_file_path[1], width='100%', height='500px'))","metadata":{"execution":{"iopub.status.busy":"2023-05-31T22:26:42.937264Z","iopub.execute_input":"2023-05-31T22:26:42.937949Z","iopub.status.idle":"2023-05-31T22:26:42.944313Z","shell.execute_reply.started":"2023-05-31T22:26:42.937914Z","shell.execute_reply":"2023-05-31T22:26:42.943260Z"},"trusted":true},"execution_count":17,"outputs":[{"output_type":"display_data","data":{"text/plain":"<IPython.lib.display.IFrame at 0x7db00bb17580>","text/html":"\n        <iframe\n            width=\"100%\"\n            height=\"500px\"\n            src=\"model_V_head_view.html\"\n            frameborder=\"0\"\n            allowfullscreen\n            \n        ></iframe>\n        "},"metadata":{}}]},{"cell_type":"markdown","source":"## Model A(Fiinetuned DistilBERT):","metadata":{}},{"cell_type":"code","source":"sentence_a = \"I hate you and disgust you\"\nsentence_b = \"I am sad to be alive\"\ninputs = tokenizer_A.encode_plus(sentence_a, sentence_b, return_tensors='pt')\noutputs = model_A(**inputs)\nattention = outputs[-1]\ntokens = tokenizer_A.convert_ids_to_tokens(inputs[\"input_ids\"][0]) \nhtml_model_view = model_view(attention, tokens, html_action='return', display_mode=\"light\")\nhtml_head_view = head_view(attention, tokens, html_action='return')\n# model_view(attention, tokens, display_mode=\"light\")\nwith open('model_A_model_view.html', 'w') as file:\n    file.write(html_model_view.data)\nwith open('model_A_head_view.html', 'w') as file:\n    file.write(html_head_view.data)\nhtml_file_path = ['model_A_model_view.html','model_A_head_view.html']","metadata":{"execution":{"iopub.status.busy":"2023-05-31T22:26:43.539282Z","iopub.execute_input":"2023-05-31T22:26:43.539973Z","iopub.status.idle":"2023-05-31T22:26:43.656557Z","shell.execute_reply.started":"2023-05-31T22:26:43.539936Z","shell.execute_reply":"2023-05-31T22:26:43.654744Z"},"trusted":true},"execution_count":18,"outputs":[]},{"cell_type":"code","source":"display(IFrame(src=html_file_path[0], width='100%', height='500px'))","metadata":{"execution":{"iopub.status.busy":"2023-05-31T22:26:43.750221Z","iopub.execute_input":"2023-05-31T22:26:43.750898Z","iopub.status.idle":"2023-05-31T22:26:43.757577Z","shell.execute_reply.started":"2023-05-31T22:26:43.750865Z","shell.execute_reply":"2023-05-31T22:26:43.756626Z"},"trusted":true},"execution_count":19,"outputs":[{"output_type":"display_data","data":{"text/plain":"<IPython.lib.display.IFrame at 0x7db00bb161a0>","text/html":"\n        <iframe\n            width=\"100%\"\n            height=\"500px\"\n            src=\"model_A_model_view.html\"\n            frameborder=\"0\"\n            allowfullscreen\n            \n        ></iframe>\n        "},"metadata":{}}]},{"cell_type":"code","source":"display(IFrame(src=html_file_path[1], width='100%', height='500px'))","metadata":{"execution":{"iopub.status.busy":"2023-05-31T22:26:43.959896Z","iopub.execute_input":"2023-05-31T22:26:43.962682Z","iopub.status.idle":"2023-05-31T22:26:43.968923Z","shell.execute_reply.started":"2023-05-31T22:26:43.962646Z","shell.execute_reply":"2023-05-31T22:26:43.967913Z"},"trusted":true},"execution_count":20,"outputs":[{"output_type":"display_data","data":{"text/plain":"<IPython.lib.display.IFrame at 0x7db00bb161a0>","text/html":"\n        <iframe\n            width=\"100%\"\n            height=\"500px\"\n            src=\"model_A_head_view.html\"\n            frameborder=\"0\"\n            allowfullscreen\n            \n        ></iframe>\n        "},"metadata":{}}]},{"cell_type":"markdown","source":"## Model D(Finetuned RoBERTa):","metadata":{}},{"cell_type":"code","source":"sentence_a = \"I hope for a better future\"\nsentence_b = \"You should obey my order\"\ninputs = tokenizer_D.encode_plus(sentence_a, sentence_b, return_tensors='pt')\noutputs = model_D(**inputs)\nattention = outputs[-1]\ntokens = tokenizer_D.convert_ids_to_tokens(inputs[\"input_ids\"][0]) \nhtml_model_view = model_view(attention, tokens, html_action='return', display_mode=\"light\")\nhtml_head_view = head_view(attention, tokens, html_action='return')\n# model_view(attention, tokens, display_mode=\"light\")\nwith open('model_D_model_view.html', 'w') as file:\n    file.write(html_model_view.data)\nwith open('model_D_head_view.html', 'w') as file:\n    file.write(html_head_view.data)\nhtml_file_path = ['model_D_model_view.html','model_A_head_view.html']","metadata":{"execution":{"iopub.status.busy":"2023-05-31T22:26:44.374288Z","iopub.execute_input":"2023-05-31T22:26:44.374981Z","iopub.status.idle":"2023-05-31T22:26:44.597436Z","shell.execute_reply.started":"2023-05-31T22:26:44.374948Z","shell.execute_reply":"2023-05-31T22:26:44.596466Z"},"trusted":true},"execution_count":21,"outputs":[]},{"cell_type":"code","source":"display(IFrame(src=html_file_path[0], width='100%', height='500px'))","metadata":{"execution":{"iopub.status.busy":"2023-05-31T22:26:44.601093Z","iopub.execute_input":"2023-05-31T22:26:44.601388Z","iopub.status.idle":"2023-05-31T22:26:44.608539Z","shell.execute_reply.started":"2023-05-31T22:26:44.601363Z","shell.execute_reply":"2023-05-31T22:26:44.607682Z"},"trusted":true},"execution_count":22,"outputs":[{"output_type":"display_data","data":{"text/plain":"<IPython.lib.display.IFrame at 0x7db00bb17700>","text/html":"\n        <iframe\n            width=\"100%\"\n            height=\"500px\"\n            src=\"model_D_model_view.html\"\n            frameborder=\"0\"\n            allowfullscreen\n            \n        ></iframe>\n        "},"metadata":{}}]},{"cell_type":"code","source":"display(IFrame(src=html_file_path[1], width='100%', height='500px'))","metadata":{"execution":{"iopub.status.busy":"2023-05-31T22:26:44.969456Z","iopub.execute_input":"2023-05-31T22:26:44.970161Z","iopub.status.idle":"2023-05-31T22:26:44.976754Z","shell.execute_reply.started":"2023-05-31T22:26:44.970125Z","shell.execute_reply":"2023-05-31T22:26:44.975714Z"},"trusted":true},"execution_count":23,"outputs":[{"output_type":"display_data","data":{"text/plain":"<IPython.lib.display.IFrame at 0x7db00bb176a0>","text/html":"\n        <iframe\n            width=\"100%\"\n            height=\"500px\"\n            src=\"model_A_head_view.html\"\n            frameborder=\"0\"\n            allowfullscreen\n            \n        ></iframe>\n        "},"metadata":{}}]},{"cell_type":"markdown","source":"# Reference:","metadata":{}},{"cell_type":"markdown","source":"1. Alammar, J. (2020). Interfaces for Explaining Transformer Language Models  \n[Blog post]. Retrieved from https://jalammar.github.io/explaining-transformers/\n\n2. Vig, Jesse. (2019). A Multiscale Visualization of Attention in the Transformer Model \nhttps://github.com/jessevig/bertviz","metadata":{}}]}