scripts/run_lm_experiments.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "yoA9cfPe-owX",
    "outputId": "bfd0b41f-ef8f-43bd-ad67-71ed1e27afc2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting git+https://github.com/huggingface/transformers\n",
      "  Cloning https://github.com/huggingface/transformers to /tmp/pip-req-build-8j3rlphh\n",
      "  Running command git clone -q https://github.com/huggingface/transformers /tmp/pip-req-build-8j3rlphh\n",
      "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
      "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
      "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers==4.2.0.dev0) (20.8)\n",
      "Collecting tokenizers==0.9.4\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/0f/1c/e789a8b12e28be5bc1ce2156cf87cb522b379be9cadc7ad8091a4cc107c4/tokenizers-0.9.4-cp36-cp36m-manylinux2010_x86_64.whl (2.9MB)\n",
      "\u001b[K     |████████████████████████████████| 2.9MB 4.2MB/s \n",
      "\u001b[?25hRequirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers==4.2.0.dev0) (0.8)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers==4.2.0.dev0) (2.23.0)\n",
      "Collecting sacremoses\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
      "\u001b[K     |████████████████████████████████| 890kB 29.9MB/s \n",
      "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers==4.2.0.dev0) (1.19.4)\n",
      "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers==4.2.0.dev0) (3.0.12)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers==4.2.0.dev0) (2019.12.20)\n",
      "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers==4.2.0.dev0) (4.41.1)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers==4.2.0.dev0) (2.4.7)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==4.2.0.dev0) (2.10)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==4.2.0.dev0) (1.24.3)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==4.2.0.dev0) (2020.12.5)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==4.2.0.dev0) (3.0.4)\n",
      "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==4.2.0.dev0) (1.15.0)\n",
      "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==4.2.0.dev0) (7.1.2)\n",
      "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==4.2.0.dev0) (1.0.0)\n",
      "Building wheels for collected packages: transformers\n",
      "  Building wheel for transformers (PEP 517) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for transformers: filename=transformers-4.2.0.dev0-cp36-none-any.whl size=1527265 sha256=77247ea9e70ee326823612e606c6542062f070627708f60ebbcdaea8b99f0cb9\n",
      "  Stored in directory: /tmp/pip-ephem-wheel-cache-fago6skx/wheels/70/d3/52/b3fa4f8b8ef04167ac62e5bb2accb62ae764db2a378247490e\n",
      "Successfully built transformers\n",
      "Building wheels for collected packages: sacremoses\n",
      "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893261 sha256=bdee8487f7db16ca3c5bce685f0970af64a29d38f4efe0ed612c274e86def013\n",
      "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
      "Successfully built sacremoses\n",
      "Installing collected packages: tokenizers, sacremoses, transformers\n",
      "Successfully installed sacremoses-0.0.43 tokenizers-0.9.4 transformers-4.2.0.dev0\n",
      "Collecting textacy\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/65/99/054efc5dea92c84a850639c490541de6cba29bc148debc3c73848c5e64c2/textacy-0.10.1-py3-none-any.whl (183kB)\n",
      "\u001b[K     |████████████████████████████████| 184kB 4.4MB/s \n",
      "\u001b[?25hRequirement already satisfied: tqdm>=4.19.6 in /usr/local/lib/python3.6/dist-packages (from textacy) (4.41.1)\n",
      "Collecting cytoolz>=0.8.0\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/58/67/1c60da8ba831bfefedb64c78b9f6820bdf58972797c95644ee3191daf27a/cytoolz-0.11.0.tar.gz (477kB)\n",
      "\u001b[K     |████████████████████████████████| 481kB 7.6MB/s \n",
      "\u001b[?25hRequirement already satisfied: joblib>=0.13.0 in /usr/local/lib/python3.6/dist-packages (from textacy) (1.0.0)\n",
      "Requirement already satisfied: networkx>=2.0 in /usr/local/lib/python3.6/dist-packages (from textacy) (2.5)\n",
      "Requirement already satisfied: requests>=2.10.0 in /usr/local/lib/python3.6/dist-packages (from textacy) (2.23.0)\n",
      "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.6/dist-packages (from textacy) (1.4.1)\n",
      "Requirement already satisfied: numpy>=1.17.0 in /usr/local/lib/python3.6/dist-packages (from textacy) (1.19.4)\n",
      "Requirement already satisfied: srsly>=0.0.5 in /usr/local/lib/python3.6/dist-packages (from textacy) (1.0.5)\n",
      "Requirement already satisfied: spacy<3.0.0,>=2.2.0 in /usr/local/lib/python3.6/dist-packages (from textacy) (2.2.4)\n",
      "Requirement already satisfied: scikit-learn<0.24.0,>=0.19.0 in /usr/local/lib/python3.6/dist-packages (from textacy) (0.22.2.post1)\n",
      "Collecting jellyfish>=0.7.0\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/6c/09/927ae35fc5a9f70abb6cc2c27ee88fc48549f7bc4786c1d4b177c22e997d/jellyfish-0.8.2-cp36-cp36m-manylinux2014_x86_64.whl (93kB)\n",
      "\u001b[K     |████████████████████████████████| 102kB 7.1MB/s \n",
      "\u001b[?25hRequirement already satisfied: cachetools>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from textacy) (4.2.0)\n",
      "Collecting pyphen>=0.9.4\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7c/5a/5bc036e01389bc6a6667a932bac3e388de6e7fa5777a6ff50e652f60ec79/Pyphen-0.10.0-py3-none-any.whl (1.9MB)\n",
      "\u001b[K     |████████████████████████████████| 1.9MB 8.9MB/s \n",
      "\u001b[?25hRequirement already satisfied: pyemd>=0.5.0 in /usr/local/lib/python3.6/dist-packages (from textacy) (0.5.1)\n",
      "Requirement already satisfied: toolz>=0.8.0 in /usr/local/lib/python3.6/dist-packages (from cytoolz>=0.8.0->textacy) (0.11.1)\n",
      "Requirement already satisfied: decorator>=4.3.0 in /usr/local/lib/python3.6/dist-packages (from networkx>=2.0->textacy) (4.4.2)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests>=2.10.0->textacy) (2.10)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests>=2.10.0->textacy) (2020.12.5)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests>=2.10.0->textacy) (1.24.3)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests>=2.10.0->textacy) (3.0.4)\n",
      "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy<3.0.0,>=2.2.0->textacy) (2.0.5)\n",
      "Requirement already satisfied: thinc==7.4.0 in /usr/local/lib/python3.6/dist-packages (from spacy<3.0.0,>=2.2.0->textacy) (7.4.0)\n",
      "Requirement already satisfied: wasabi<1.1.0,>=0.4.0 in /usr/local/lib/python3.6/dist-packages (from spacy<3.0.0,>=2.2.0->textacy) (0.8.0)\n",
      "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy<3.0.0,>=2.2.0->textacy) (3.0.5)\n",
      "Requirement already satisfied: blis<0.5.0,>=0.4.0 in /usr/local/lib/python3.6/dist-packages (from spacy<3.0.0,>=2.2.0->textacy) (0.4.1)\n",
      "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.6/dist-packages (from spacy<3.0.0,>=2.2.0->textacy) (1.0.5)\n",
      "Requirement already satisfied: catalogue<1.1.0,>=0.0.7 in /usr/local/lib/python3.6/dist-packages (from spacy<3.0.0,>=2.2.0->textacy) (1.0.0)\n",
      "Requirement already satisfied: plac<1.2.0,>=0.9.6 in /usr/local/lib/python3.6/dist-packages (from spacy<3.0.0,>=2.2.0->textacy) (1.1.3)\n",
      "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from spacy<3.0.0,>=2.2.0->textacy) (51.0.0)\n",
      "Requirement already satisfied: importlib-metadata>=0.20; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from catalogue<1.1.0,>=0.0.7->spacy<3.0.0,>=2.2.0->textacy) (3.3.0)\n",
      "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata>=0.20; python_version < \"3.8\"->catalogue<1.1.0,>=0.0.7->spacy<3.0.0,>=2.2.0->textacy) (3.4.0)\n",
      "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from importlib-metadata>=0.20; python_version < \"3.8\"->catalogue<1.1.0,>=0.0.7->spacy<3.0.0,>=2.2.0->textacy) (3.7.4.3)\n",
      "Building wheels for collected packages: cytoolz\n",
      "  Building wheel for cytoolz (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for cytoolz: filename=cytoolz-0.11.0-cp36-cp36m-linux_x86_64.whl size=1225552 sha256=c84f393d0242bbe4d622003e8f56f1e61215e832d36dba520c6132d55e03b649\n",
      "  Stored in directory: /root/.cache/pip/wheels/a1/32/3c/9c9926b510647cacdde744b2c7acdf1ccd5896fbb7f8d5df0c\n",
      "Successfully built cytoolz\n",
      "Installing collected packages: cytoolz, jellyfish, pyphen, textacy\n",
      "Successfully installed cytoolz-0.11.0 jellyfish-0.8.2 pyphen-0.10.0 textacy-0.10.1\n"
     ]
    }
   ],
   "source": [
    "!pip install git+https://github.com/huggingface/transformers\n",
    "!pip install textacy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SpaCy Experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "9FpAhiEXAKpR",
    "outputId": "98861758-b6c4-4da2-a05d-7aeb5c553bb9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/content/drive/MyDrive\n",
      "/content/drive/MyDrive/lm_experiments\n",
      "2020-12-28 04:26:45.717024: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.10.1\n",
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForMaskedLM: ['cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
      "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Done!\n",
      "2020-12-28 04:34:49.487834: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.10.1\n",
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForMaskedLM: ['cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
      "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Done!\n",
      "2020-12-28 05:25:21.694441: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.10.1\n",
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForMaskedLM: ['cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
      "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Done!\n",
      "2020-12-28 06:39:02.048074: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.10.1\n",
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForMaskedLM: ['cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
      "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Done!\n"
     ]
    }
   ],
   "source": [
    "%cd /content/drive/MyDrive\n",
    "# !mkdir lm_experiments\n",
    "%cd lm_experiments\n",
    "!python ../run_knowledge_graph_experiments.py squad Bert spacy\n",
    "!python ../run_knowledge_graph_experiments.py re-date-birth Bert spacy\n",
    "!python ../run_knowledge_graph_experiments.py re-place-birth Bert spacy\n",
    "!python ../run_knowledge_graph_experiments.py re-place-death Bert spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "3SwW0Yr3-7S-",
    "outputId": "f27462f0-42b7-4aa8-bcb1-22965906433d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-12-28 15:37:30.892504: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.10.1\n",
      "Downloading: 100% 899k/899k [00:00<00:00, 12.9MB/s]\n",
      "Downloading: 100% 456k/456k [00:00<00:00, 6.99MB/s]\n",
      "Downloading: 100% 481/481 [00:00<00:00, 284kB/s]\n",
      "Downloading: 100% 501M/501M [00:08<00:00, 62.0MB/s]\n",
      "Downloading: 100% 1.36M/1.36M [00:00<00:00, 16.7MB/s]\n",
      "Some weights of RobertaForMaskedLM were not initialized from the model checkpoint at roberta-base and are newly initialized: ['lm_head.decoder.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "Done!\n",
      "2020-12-28 15:47:10.461390: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.10.1\n",
      "Some weights of RobertaForMaskedLM were not initialized from the model checkpoint at roberta-base and are newly initialized: ['lm_head.decoder.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "Done!\n",
      "2020-12-28 16:36:35.019791: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.10.1\n",
      "Some weights of RobertaForMaskedLM were not initialized from the model checkpoint at roberta-base and are newly initialized: ['lm_head.decoder.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "Done!\n",
      "2020-12-28 17:52:29.901692: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.10.1\n",
      "Some weights of RobertaForMaskedLM were not initialized from the model checkpoint at roberta-base and are newly initialized: ['lm_head.decoder.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "Done!\n"
     ]
    }
   ],
   "source": [
    "!python ../run_knowledge_graph_experiments.py squad Roberta spacy\n",
    "!python ../run_knowledge_graph_experiments.py re-date-birth Roberta spacy\n",
    "!python ../run_knowledge_graph_experiments.py re-place-birth Roberta spacy\n",
    "!python ../run_knowledge_graph_experiments.py re-place-death Roberta spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "cVhGzqH7_A17",
    "outputId": "f7b4ebce-4474-4342-bdf9-b06b00f5489c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-12-28 07:02:09.369784: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.10.1\n",
      "Downloading: 100% 442/442 [00:00<00:00, 280kB/s]\n",
      "Downloading: 100% 268M/268M [00:04<00:00, 66.0MB/s]\n",
      "Done!\n",
      "2020-12-28 07:10:08.320335: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.10.1\n",
      "Done!\n",
      "2020-12-28 07:58:00.569079: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.10.1\n",
      "Done!\n",
      "2020-12-28 09:10:45.219335: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.10.1\n",
      "Done!\n"
     ]
    }
   ],
   "source": [
    "!python ../run_knowledge_graph_experiments.py squad DistilBert spacy\n",
    "!python ../run_knowledge_graph_experiments.py re-date-birth DistilBert spacy\n",
    "!python ../run_knowledge_graph_experiments.py re-place-birth DistilBert spacy\n",
    "!python ../run_knowledge_graph_experiments.py re-place-death DistilBert spacy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "D5qxS9sJ_J1z"
   },
   "source": [
    "# Textacy experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "K8jbU44fBqlM",
    "outputId": "5517c2dd-d7cc-4420-8992-b27ec53f7757"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/content/drive/My Drive/lm_experiments/textacy_experiments\n"
     ]
    }
   ],
   "source": [
    "# !mkdir textacy_experiments\n",
    "%cd textacy_experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "xx93FT2s_ITS",
    "outputId": "9bfce2b6-ca92-4e6d-eb6c-598f8ff62eb3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-12-28 18:14:06.358560: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.10.1\n",
      "Downloading: 100% 232k/232k [00:00<00:00, 4.96MB/s]\n",
      "Downloading: 100% 433/433 [00:00<00:00, 276kB/s]\n",
      "Downloading: 100% 440M/440M [00:07<00:00, 55.9MB/s]\n",
      "Downloading: 100% 466k/466k [00:00<00:00, 7.20MB/s]\n",
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForMaskedLM: ['cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
      "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Done!\n",
      "2020-12-28 18:22:29.074468: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.10.1\n",
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForMaskedLM: ['cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
      "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Done!\n",
      "2020-12-28 19:10:54.548679: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.10.1\n",
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForMaskedLM: ['cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
      "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Done!\n",
      "2020-12-28 20:20:38.494362: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.10.1\n",
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForMaskedLM: ['cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
      "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Done!\n",
      "2020-12-28 20:41:43.472627: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.10.1\n",
      "Some weights of RobertaForMaskedLM were not initialized from the model checkpoint at roberta-base and are newly initialized: ['lm_head.decoder.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "Done!\n",
      "2020-12-28 20:50:04.701737: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.10.1\n",
      "Some weights of RobertaForMaskedLM were not initialized from the model checkpoint at roberta-base and are newly initialized: ['lm_head.decoder.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "Done!\n",
      "2020-12-28 21:39:03.079734: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.10.1\n",
      "Some weights of RobertaForMaskedLM were not initialized from the model checkpoint at roberta-base and are newly initialized: ['lm_head.decoder.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "Done!\n",
      "2020-12-28 22:50:43.004007: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.10.1\n",
      "Some weights of RobertaForMaskedLM were not initialized from the model checkpoint at roberta-base and are newly initialized: ['lm_head.decoder.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "Done!\n",
      "2020-12-28 23:11:11.079586: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.10.1\n",
      "Downloading: 100% 442/442 [00:00<00:00, 310kB/s]\n",
      "Downloading: 100% 268M/268M [00:05<00:00, 47.2MB/s]\n",
      "Done!\n",
      "2020-12-28 23:18:56.122657: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.10.1\n",
      "Done!\n",
      "2020-12-29 00:02:50.306891: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.10.1\n",
      "Done!\n",
      "2020-12-29 01:08:06.053073: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.10.1\n",
      "Done!\n"
     ]
    }
   ],
   "source": [
    "!python ../../run_knowledge_graph_experiments.py squad Bert\n",
    "!python ../../run_knowledge_graph_experiments.py re-date-birth Bert\n",
    "!python ../../run_knowledge_graph_experiments.py re-place-birth Bert\n",
    "!python ../../run_knowledge_graph_experiments.py re-place-death Bert\n",
    "\n",
    "!python ../../run_knowledge_graph_experiments.py squad Roberta\n",
    "!python ../../run_knowledge_graph_experiments.py re-date-birth Roberta\n",
    "!python ../../run_knowledge_graph_experiments.py re-place-birth Roberta\n",
    "!python ../../run_knowledge_graph_experiments.py re-place-death Roberta\n",
    "\n",
    "!python ../../run_knowledge_graph_experiments.py squad DistilBert\n",
    "!python ../../run_knowledge_graph_experiments.py re-date-birth DistilBert\n",
    "!python ../../run_knowledge_graph_experiments.py re-place-birth DistilBert\n",
    "!python ../../run_knowledge_graph_experiments.py re-place-death DistilBert"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "run_lm_experiments.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}