notebooks/test_mlm_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "f20ea101",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "80"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import AutoTokenizer\n",
    "from tokenizers import Tokenizer\n",
    "from transformers import PreTrainedTokenizerFast\n",
    "from tokenizers.models import WordLevel, BPE\n",
    "from tokenizers.pre_tokenizers import Whitespace,Split,ByteLevel, WhitespaceSplit\n",
    "from tokenizers.normalizers import Lowercase, NFKC\n",
    "import os\n",
    "import polars as pl\n",
    "from joblib import Parallel, delayed\n",
    "import multiprocessing\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import time\n",
    "import json\n",
    "from tokenizers.trainers import BpeTrainer, UnigramTrainer, WordLevelTrainer, WordPieceTrainer\n",
    "import gc\n",
    "from transformers import AutoConfig, AutoTokenizer, AutoModel, DataCollatorWithPadding, DataCollatorForLanguageModeling\n",
    "import mapply\n",
    "from collections import Counter\n",
    "from rdkit import Chem\n",
    "from rdkit.Chem import AllChem\n",
    "from functools import partial\n",
    "\n",
    "multiprocessing.cpu_count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24f8660e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cfe04f4e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.06128192972391844 GB\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (878_022, 1)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>molecule</th></tr><tr><td>str</td></tr></thead><tbody><tr><td>&quot;C#CCCC[C@H](Nc…</td></tr><tr><td>&quot;C#CCCC[C@H](Nc…</td></tr><tr><td>&quot;C#CCCC[C@H](Nc…</td></tr><tr><td>&quot;C#CCCC[C@H](Nc…</td></tr><tr><td>&quot;C#CCCC[C@H](Nc…</td></tr><tr><td>&hellip;</td></tr><tr><td>&quot;Cn1ncc2cc(Nc3n…</td></tr><tr><td>&quot;[N-]=[N+]=NCCC…</td></tr><tr><td>&quot;COC(=O)c1ccnc(…</td></tr><tr><td>&quot;COC1CCC(CCNc2n…</td></tr><tr><td>&quot;[N-]=[N+]=NCCC…</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (878_022, 1)\n",
       "┌───────────────────────────────────┐\n",
       "│ molecule                          │\n",
       "│ ---                               │\n",
       "│ str                               │\n",
       "╞═══════════════════════════════════╡\n",
       "│ C#CCCC[C@H](Nc1nc(Nc2ccc(C=C)cc2… │\n",
       "│ C#CCCC[C@H](Nc1nc(Nc2ccc(C=C)cc2… │\n",
       "│ C#CCCC[C@H](Nc1nc(NCC2(O)CCCC2(C… │\n",
       "│ C#CCCC[C@H](Nc1nc(Nc2ccc(C=C)cc2… │\n",
       "│ C#CCCC[C@H](Nc1nc(NCC2CCC(SC)CC2… │\n",
       "│ …                                 │\n",
       "│ Cn1ncc2cc(Nc3nc(Nc4nncs4)nc(N[C@… │\n",
       "│ [N-]=[N+]=NCCC[C@H](Nc1nc(NCC2CC… │\n",
       "│ COC(=O)c1ccnc(Nc2nc(Nc3noc4ccc(F… │\n",
       "│ COC1CCC(CCNc2nc(Nc3noc4ccc(F)cc3… │\n",
       "│ [N-]=[N+]=NCCC[C@H](Nc1nc(NCc2cc… │\n",
       "└───────────────────────────────────┘"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df = pl.scan_csv('/home/dangnh36/datasets/competitions/leash_belka/processed/test_v4.csv').select(\n",
    "        pl.col('molecule'),\n",
    "#         pl.col('bb1', 'bb2', 'bb3').cast(pl.UInt16),\n",
    "        # pl.col('BRD4', 'HSA', 'sEH').cast(pl.UInt8),\n",
    "    ).collect()\n",
    "print(test_df.estimated_size('gb'), 'GB')\n",
    "test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1edde9c5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e402878",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "95abcba1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PreTrainedTokenizerFast(name_or_path='/home/dangnh36/datasets/competitions/leash_belka/processed/tokenizer_v2/smiles_char/', vocab_size=44, model_max_length=1000000000000000019884624838656, is_fast=True, padding_side='right', truncation_side='right', special_tokens={'bos_token': '[BOS]', 'eos_token': '[EOS]', 'unk_token': '[UNK]', 'sep_token': '[SEP]', 'pad_token': '[PAD]', 'cls_token': '[CLS]', 'mask_token': '[MASK]'}, clean_up_tokenization_spaces=True),  added_tokens_decoder={\n",
       "\t0: AddedToken(\"[PAD]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "\t1: AddedToken(\"[UNK]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "\t2: AddedToken(\"[CLS]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "\t3: AddedToken(\"[MASK]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "\t4: AddedToken(\"[SEP]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "\t5: AddedToken(\"[BOS]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "\t6: AddedToken(\"[EOS]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "\t7: AddedToken(\"Br\", rstrip=False, lstrip=False, single_word=False, normalized=True, special=False),\n",
       "\t14: AddedToken(\"Cl\", rstrip=False, lstrip=False, single_word=False, normalized=True, special=False),\n",
       "\t32: AddedToken(\"@@\", rstrip=False, lstrip=False, single_word=False, normalized=True, special=False),\n",
       "\t43: AddedToken(\"[Dy]\", rstrip=False, lstrip=False, single_word=False, normalized=True, special=False),\n",
       "}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained('/home/dangnh36/datasets/competitions/leash_belka/processed/tokenizer_v2/smiles_char/',\n",
    "                                         trust_remote_code=True\n",
    "                                         )\n",
    "tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "63938c9e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'[PAD]': 0,\n",
       " '[UNK]': 1,\n",
       " '[CLS]': 2,\n",
       " '[MASK]': 3,\n",
       " '[SEP]': 4,\n",
       " '[BOS]': 5,\n",
       " '[EOS]': 6,\n",
       " 'Br': 7,\n",
       " 'C': 8,\n",
       " 'N': 9,\n",
       " 'O': 10,\n",
       " 'H': 11,\n",
       " 'S': 12,\n",
       " 'F': 13,\n",
       " 'Cl': 14,\n",
       " 'B': 15,\n",
       " 'I': 16,\n",
       " 's': 17,\n",
       " 'o': 18,\n",
       " 'c': 19,\n",
       " 'n': 20,\n",
       " 'i': 21,\n",
       " '.': 22,\n",
       " '=': 23,\n",
       " '#': 24,\n",
       " '/': 25,\n",
       " '-': 26,\n",
       " '+': 27,\n",
       " '[': 28,\n",
       " ']': 29,\n",
       " '(': 30,\n",
       " ')': 31,\n",
       " '@@': 32,\n",
       " '@': 33,\n",
       " '1': 34,\n",
       " '2': 35,\n",
       " '3': 36,\n",
       " '4': 37,\n",
       " '5': 38,\n",
       " '6': 39,\n",
       " '7': 40,\n",
       " '8': 41,\n",
       " '9': 42,\n",
       " '[Dy]': 43}"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "{k:v for k, v in sorted(tokenizer.get_vocab().items(), key = lambda x: x[1])}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a854a272",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2048"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "smiles_list = test_df[:2048, 'molecule'].to_list()\n",
    "smiles_list = [f'[CLS][BOS]{e}[EOS]' for e in smiles_list]\n",
    "len(smiles_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "4a66427e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['input_ids', 'attention_mask', 'special_tokens_mask', 'length'])"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ret = tokenizer(\n",
    "            smiles_list,\n",
    "            add_special_tokens=True,\n",
    "            padding='longest',\n",
    "            truncation=False,\n",
    "            max_length=512,\n",
    "            is_split_into_words=False,\n",
    "            pad_to_multiple_of=None,\n",
    "            return_tensors='pt',\n",
    "            return_token_type_ids=False,\n",
    "            return_attention_mask=True,\n",
    "            return_special_tokens_mask=True,\n",
    "            return_length=True,\n",
    "            verbose=True)\n",
    "# batch = {\n",
    "#     'idx': torch.tensor(idxs),\n",
    "#     'input_ids': ret['input_ids'].long(),\n",
    "#     'padding_mask': ret['attention_mask'].bool(),\n",
    "#     'length': ret['length'],\n",
    "#     'mtr_target': mtr_target\n",
    "# }\n",
    "\n",
    "ret.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "3138ea9e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[2, 5, 8,  ..., 0, 0, 0],\n",
       "        [2, 5, 8,  ..., 0, 0, 0],\n",
       "        [2, 5, 8,  ..., 0, 0, 0],\n",
       "        ...,\n",
       "        [2, 5, 8,  ..., 0, 0, 0],\n",
       "        [2, 5, 8,  ..., 0, 0, 0],\n",
       "        [2, 5, 8,  ..., 0, 0, 0]])"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ret['input_ids']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "b1a89272",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0, 0, 0,  ..., 1, 1, 1],\n",
       "        [0, 0, 0,  ..., 1, 1, 1],\n",
       "        [0, 0, 0,  ..., 1, 1, 1],\n",
       "        ...,\n",
       "        [0, 0, 0,  ..., 1, 1, 1],\n",
       "        [0, 0, 0,  ..., 1, 1, 1],\n",
       "        [0, 0, 0,  ..., 1, 1, 1]])"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ret['special_tokens_mask']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "230e14d7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "        1, 1, 1, 1, 1, 1, 1, 1])"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ret['special_tokens_mask'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "2d0b1529",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n"
     ]
    }
   ],
   "source": [
    "print(tokenizer.get_special_tokens_mask(ret['input_ids'][1], already_has_special_tokens = True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "f3749bbb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.mask_token_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "a615f8d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "\n",
    "class MLMMasker:\n",
    "    def __init__(self, tokenizer, mlm_prob = 0.15, mask_prob = 0.8, random_prob = 0.1):\n",
    "        self.tokenizer = tokenizer\n",
    "        self.mlm_prob = mlm_prob\n",
    "        self.mask_prob = mask_prob\n",
    "        self.random_prob = random_prob\n",
    "        self._random_prob = random_prob / (1. - mask_prob)\n",
    "#         print(self._random_prob)\n",
    "        self.mask_token_id = tokenizer.mask_token_id\n",
    "        print('Mask token id:', self.mask_token_id)\n",
    "\n",
    "    def __call__(self, inputs, special_tokens_mask = None):\n",
    "        \"\"\"\n",
    "        Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original.\n",
    "        \"\"\"\n",
    "        labels = inputs.clone()\n",
    "        # We sample a few tokens in each sequence for MLM training (with probability `self.mlm_probability`)\n",
    "        probability_matrix = torch.full(labels.shape, self.mlm_prob)\n",
    "        if special_tokens_mask is None:\n",
    "            special_tokens_mask = [\n",
    "                self.tokenizer.get_special_tokens_mask(val, already_has_special_tokens=True) for val in labels.tolist()\n",
    "            ]\n",
    "            special_tokens_mask = torch.tensor(special_tokens_mask, dtype=torch.bool)\n",
    "        else:\n",
    "            special_tokens_mask = special_tokens_mask.bool()\n",
    "#         print('special tokens mask:\\n', special_tokens_mask[0])\n",
    "\n",
    "        probability_matrix.masked_fill_(special_tokens_mask, value=0.0)\n",
    "        masked_indices = torch.bernoulli(probability_matrix).bool()\n",
    "        labels[~masked_indices] = -1  # We only compute loss on masked tokens\n",
    "\n",
    "        # 80% of the time, we replace masked input tokens with tokenizer.mask_token ([MASK])\n",
    "        indices_replaced = torch.bernoulli(torch.full(labels.shape, self.mask_prob)).bool() & masked_indices\n",
    "        inputs[indices_replaced] = self.mask_token_id\n",
    "\n",
    "        # 10% of the time, we replace masked input tokens with random word\n",
    "        indices_random = torch.bernoulli(torch.full(labels.shape, self._random_prob)).bool() & masked_indices & ~indices_replaced\n",
    "        random_words = torch.randint(len(self.tokenizer), labels.shape, dtype=torch.long)\n",
    "        inputs[indices_random] = random_words[indices_random]\n",
    "\n",
    "        # The rest of the time (10% of the time) we keep the masked input tokens unchanged\n",
    "        return inputs, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "41099bf8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mask token id: 3\n"
     ]
    }
   ],
   "source": [
    "masker = MLMMasker(tokenizer, 0.15, 0.8, 0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "56924743",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[2, 5, 3,  ..., 0, 0, 0],\n",
       "         [2, 5, 3,  ..., 0, 0, 0],\n",
       "         [2, 5, 3,  ..., 0, 0, 0],\n",
       "         ...,\n",
       "         [2, 5, 3,  ..., 0, 0, 0],\n",
       "         [2, 5, 8,  ..., 0, 0, 0],\n",
       "         [2, 5, 3,  ..., 0, 0, 0]]),\n",
       " tensor([[-1, -1, -1,  ..., -1, -1, -1],\n",
       "         [-1, -1, -1,  ..., -1, -1, -1],\n",
       "         [-1, -1, -1,  ..., -1, -1, -1],\n",
       "         ...,\n",
       "         [-1, -1, -1,  ..., -1, -1, -1],\n",
       "         [-1, -1, -1,  ..., -1, -1, -1],\n",
       "         [-1, -1, -1,  ..., -1, -1, -1]]))"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ret2 = masker(ret['input_ids'], special_tokens_mask = None)\n",
    "ret2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "9e180c22",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0)"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(ret2[1][:, :2] != -1).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "f24bd28b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[MASK] [MASK] [MASK] [MASK] [MASK] [MASK] + [MASK]\n",
      "# [ ( n c N c )\n",
      "\n",
      "\n",
      "[MASK] [MASK] c 3\n",
      "C [ c 3\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] N [MASK] [MASK]\n",
      "O 2 C ( N 1 )\n",
      "\n",
      "\n",
      "[MASK] S [MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK]\n",
      "C c c ) c ( c 1 C )\n",
      "\n",
      "\n",
      "[MASK] [ C C [MASK] [MASK]\n",
      "N [ C C c (\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] [MASK] [MASK] (\n",
      "C N c ( [Dy] C (\n",
      "\n",
      "\n",
      "+ [MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK] n [MASK] c [MASK] [MASK]\n",
      "C H 1 c N c c ( c ) n N c 2 1\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] [MASK] [MASK]\n",
      "C c C N n N\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] [MASK] [MASK]\n",
      "C H 2 O n )\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK]\n",
      "H 2 c (\n",
      "\n",
      "\n",
      "[MASK] [MASK] ( c [MASK] 8 [MASK]\n",
      "H c ( c 3 ( =\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK]\n",
      "N N n [Dy]\n",
      "\n",
      "\n",
      "[MASK] [MASK]\n",
      "1 C\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK]\n",
      "C @ O N\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [UNK]\n",
      "# C C ( n c 2 ) O\n",
      "\n",
      "\n",
      "[MASK] [MASK]\n",
      "n c\n",
      "\n",
      "\n",
      "8 [MASK] 7 [MASK]\n",
      "C 1 O [Dy]\n",
      "\n",
      "\n",
      "C [MASK] [MASK] [MASK] c [MASK] [MASK] [MASK] [MASK]\n",
      "C c c B c C ) c n\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] n [MASK] (\n",
      "C n c = n n (\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK]\n",
      "C 6 C 2 [ H C\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] C [MASK] [MASK] [MASK] [MASK] [MASK]\n",
      "n 2 c C c ) ( N 2\n",
      "\n",
      "\n",
      "[MASK] [MASK] c [MASK] [MASK] [MASK] [MASK] [MASK]\n",
      "# N c n ( n c =\n",
      "\n",
      "\n",
      "@ [MASK] [MASK] [MASK] [CLS] [MASK]\n",
      "@ N O C ) n\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [Dy]\n",
      "C C ( c c C [Dy]\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK] O\n",
      "C [ 6 n = ( c C 1\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] C [MASK] [MASK] [MASK] B [MASK] [MASK] [MASK]\n",
      "1 n C C c n c c ( ) [Dy]\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] ) [MASK] [MASK] [MASK] ) [MASK]\n",
      "H c c ) c n c n [Dy]\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK]\n",
      "H C O o C C n C O )\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] [MASK] [MASK]\n",
      "@ n O c ) )\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK]\n",
      "# ( C C\n",
      "\n",
      "\n",
      "[MASK] c s 2 [MASK]\n",
      "( c s 2 )\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] F [MASK] [MASK]\n",
      "C = = O ) ( F\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] [MASK]\n",
      "o ( c 2 )\n",
      "\n",
      "\n",
      "1 c [MASK] [MASK] [MASK] i [MASK] 6 [MASK]\n",
      "1 c ( ) O 3 ( N [Dy]\n",
      "\n",
      "\n",
      "Br [MASK]\n",
      "n c\n",
      "\n",
      "\n",
      "1 [MASK] [MASK] [MASK] [MASK] [UNK]\n",
      "1 ) ( 3 C O\n",
      "\n",
      "\n",
      "[MASK] S [MASK] [MASK]\n",
      "@ C ) )\n",
      "\n",
      "\n",
      "# [ [MASK] [MASK] [MASK]\n",
      "# [ @ c )\n",
      "\n",
      "\n",
      "[MASK] ] [MASK] [MASK] [MASK] [MASK]\n",
      "C ] C C 2 (\n",
      "\n",
      "\n",
      "[MASK] 4 [MASK] [MASK] [MASK] [MASK] [MASK] [MASK]\n",
      "# C ] ( C C C O\n",
      "\n",
      "\n",
      "c / [MASK] [MASK]\n",
      "c 2 ) N\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK]\n",
      "c c )\n",
      "\n",
      "\n",
      "[MASK] F [MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK] 8\n",
      "# [ ( c ( c C S C C C\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] [MASK] [MASK] N [MASK]\n",
      "C H N O 2 C 1 )\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] [MASK] [MASK] )\n",
      "5 1 3 2 C O )\n",
      "\n",
      "\n",
      "[MASK] @ [MASK] [MASK] [MASK] F [MASK] B [MASK] [MASK] c [MASK]\n",
      "C N [ n H F @ H 3 ) C N\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] )\n",
      "H C c ( )\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] [MASK] [MASK]\n",
      "( N C Cl 1 N\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] [MASK] [MASK] o [MASK]\n",
      "C ( n c C C n =\n",
      "\n",
      "\n",
      "[MASK] [MASK] C [MASK] [MASK] [MASK]\n",
      "# C C ) C (\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK]\n",
      "1 c C c c ) C\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK] ) [MASK]\n",
      "C N n O C C F ) [Dy]\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK] )\n",
      "[ c 1 n C 2 O 2 C )\n",
      "\n",
      "\n",
      "[MASK] c [MASK] 3\n",
      ") c n C\n",
      "\n",
      "\n",
      "C [MASK] [MASK] [MASK] [MASK] [MASK] [SEP] [MASK] [MASK] [MASK]\n",
      "C C c C C n ( c n 1\n",
      "\n",
      "\n",
      "5 [MASK] [MASK] [MASK] [MASK] [MASK] B\n",
      "# [ C ] N C n\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK]\n",
      "C C @ ) 3 1 )\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] 2 [MASK] [MASK] [MASK] [MASK]\n",
      "c 1 n 2 c ) 1 )\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK]\n",
      "C C [ 1 n 2 ( C (\n",
      "\n",
      "\n",
      "[MASK] @ [MASK] [MASK] [MASK] [MASK] [MASK] [MASK] -\n",
      "[ @ 2 2 @@ C ) c S\n",
      "\n",
      "\n",
      "[MASK] F [MASK] [MASK] [MASK] [MASK] [MASK] [MASK] 9 2\n",
      "C [ @ ) c ( O 2 O )\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK]\n",
      "# C C [ c C 2 (\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] n [MASK]\n",
      "c O ) n O\n",
      "\n",
      "\n",
      "[MASK] [MASK] C [MASK] [MASK] [MASK]\n",
      "# 5 C C ( )\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] [MASK] [PAD] [MASK]\n",
      "c n N c ( O N\n",
      "\n",
      "\n",
      "C [MASK] [MASK] [MASK] [MASK] 9 n [MASK] [MASK]\n",
      "C C c O 3 c n c C\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK]\n",
      "C ( C N ( O c (\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK] 6\n",
      "C ( N n C C [ c )\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK]\n",
      "c 3 2 )\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK]\n",
      "9 C ( N c c (\n",
      "\n",
      "\n",
      "[MASK] [MASK] 7 [MASK] [MASK] [MASK] [MASK]\n",
      "C [ ] C c ) N\n",
      "\n",
      "\n",
      "c Cl [MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK]\n",
      "c Cl ) c c 2 ) ( N\n",
      "\n",
      "\n",
      "[MASK] H [MASK]\n",
      "C c c\n",
      "\n",
      "\n",
      "[MASK] [MASK] = [MASK] [MASK] [MASK] c [MASK] [MASK]\n",
      "C H N c 2 c c c =\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] [MASK]\n",
      "# @ c N c\n",
      "\n",
      "\n",
      "[MASK] [MASK] c [MASK] [MASK] [MASK] [MASK] [MASK] [MASK]\n",
      "# [ c 1 c n c ) )\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] [MASK] [MASK] Cl [MASK] [MASK]\n",
      "# 7 ] c c 2 N 3 =\n",
      "\n",
      "\n",
      "[MASK] [MASK] ( [MASK] [MASK] [MASK]\n",
      "c c ( N 2 O\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK]\n",
      "( c N\n",
      "\n",
      "\n",
      "[MASK] S [MASK] [MASK]\n",
      "C # c O\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] [MASK] 7 C\n",
      "C N n c n ( )\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] n [MASK] [MASK] [UNK] [MASK] [MASK]\n",
      "C c 1 n n N O - 2 1\n",
      "\n",
      "\n",
      "[MASK] [MASK] c [MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK] [MASK]\n",
      "C ] c c ) ) c @@ C O ) -\n",
      "\n",
      "\n",
      "[ [MASK] [MASK] [MASK] @ [MASK]\n",
      "[ H O C + I\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] - [MASK] 4\n",
      "C C ( c c N C\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] ] [MASK] [MASK]\n",
      "C ) c ] ) N\n",
      "\n",
      "\n",
      "[MASK] [MASK] c 8 O ] [\n",
      "N ( c N O = 9\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] N + [MASK] [MASK] [MASK] [MASK] [MASK] [MASK] - [MASK] [MASK] [MASK]\n",
      "C c ( ( N + c O c c ( O - ] ) [\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] [MASK] C [MASK]\n",
      "C Br H ) c C c\n",
      "\n",
      "\n",
      "[MASK] [MASK] 2 [MASK] [MASK]\n",
      "1 n 2 O O\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] c [MASK] [MASK]\n",
      "+ N 2 2 c ] [\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] [MASK]\n",
      "1 c ) - N\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] 2 [MASK] [MASK]\n",
      "n ( n 2 O C\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] F [MASK] c [MASK] [MASK] [MASK] [MASK] [MASK]\n",
      "C c c c 2 c 2 ] - O )\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] [MASK]\n",
      "1 c N = N\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] [MASK] [MASK] ] [MASK]\n",
      "C [ 1 n n 3 ] [Dy]\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] B [MASK] [MASK] [MASK] N [MASK]\n",
      "# C C c ( c [ N -\n",
      "\n",
      "\n",
      "C [MASK] c [MASK] [MASK] N [MASK] [MASK] [MASK] [MASK]\n",
      "C 2 c c 3 N ) ( ] )\n",
      "\n",
      "\n",
      "[MASK] ] [MASK] @ [MASK] n [MASK] C 2\n",
      "c ( H @ c n c = 2\n",
      "\n",
      "\n",
      "[MASK] [MASK] [MASK] [MASK] [MASK] [MASK]\n",
      "c I c N O c\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in range(100):\n",
    "    input_ids = ret2[0][i]\n",
    "    labels = ret2[1][i]\n",
    "    mask = labels != -1\n",
    "    print(tokenizer.decode(input_ids[mask]))\n",
    "    print(tokenizer.decode(labels[mask]))\n",
    "    print('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b0eb6d2f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2230cf88",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d4a70878",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6dbc12a0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3b6237d2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5652a3d5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d5ef27cc",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}