BERT.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "1_Japanese_BERT_on_Google_Colaboratory.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kYC6J-E6cjQV",
        "colab_type": "text"
      },
      "source": [
        "## BERT \n",
        "\n",
        "URL：\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nOAjVollcR4z",
        "colab_type": "text"
      },
      "source": [
        "##：MeCab"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n9TejYvKcVBK",
        "colab_type": "code",
        "outputId": "dbec6c74-1b98-4b58-c226-dd1cbced02b4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 474
        }
      },
      "source": [
        "!apt install aptitude swig\n",
        "!apt install mecab libmecab-dev mecab-ipadic-utf8 git make curl xz-utils file -y\n"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "aptitude is already the newest version (0.8.10-6ubuntu1).\n",
            "swig is already the newest version (3.0.12-1).\n",
            "mecab is already installed at the requested version (0.996-5)\n",
            "libmecab-dev is already installed at the requested version (0.996-5)\n",
            "mecab-ipadic-utf8 is already installed at the requested version (2.7.0-20070801+main-1)\n",
            "git is already installed at the requested version (1:2.17.1-1ubuntu0.7)\n",
            "make is already installed at the requested version (4.1-9.1ubuntu1)\n",
            "curl is already installed at the requested version (7.58.0-2ubuntu3.8)\n",
            "xz-utils is already installed at the requested version (5.2.2-1.3)\n",
            "file is already installed at the requested version (1:5.32-2ubuntu0.3)\n",
            "mecab is already installed at the requested version (0.996-5)\n",
            "libmecab-dev is already installed at the requested version (0.996-5)\n",
            "mecab-ipadic-utf8 is already installed at the requested version (2.7.0-20070801+main-1)\n",
            "git is already installed at the requested version (1:2.17.1-1ubuntu0.7)\n",
            "make is already installed at the requested version (4.1-9.1ubuntu1)\n",
            "curl is already installed at the requested version (7.58.0-2ubuntu3.8)\n",
            "xz-utils is already installed at the requested version (5.2.2-1.3)\n",
            "file is already installed at the requested version (1:5.32-2ubuntu0.3)\n",
            "No packages will be installed, upgraded, or removed.\n",
            "0 packages upgraded, 0 newly installed, 0 to remove and 29 not upgraded.\n",
            "Need to get 0 B of archives. After unpacking 0 B will be used.\n",
            "                            \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5anbSNgUcXPw",
        "colab_type": "code",
        "outputId": "879f8baf-6e95-4360-c1bf-498da483787b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "!pip install mecab-python3"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: mecab-python3 in /usr/local/lib/python3.6/dist-packages (0.996.5)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_pQ5MY_Jcpu0",
        "colab_type": "code",
        "outputId": "922d3716-c8c2-498a-f2df-a884f8906442",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "!git clone --depth 1 https://github.com/neologd/mecab-ipadic-neologd.git\n",
        "!echo yes | mecab-ipadic-neologd/bin/install-mecab-ipadic-neologd -n -a"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "fatal: destination path 'mecab-ipadic-neologd' already exists and is not an empty directory.\n",
            "[install-mecab-ipadic-NEologd] : Start..\n",
            "[install-mecab-ipadic-NEologd] : Check the existance of libraries\n",
            "[install-mecab-ipadic-NEologd] :     find => ok\n",
            "[install-mecab-ipadic-NEologd] :     sort => ok\n",
            "[install-mecab-ipadic-NEologd] :     head => ok\n",
            "[install-mecab-ipadic-NEologd] :     cut => ok\n",
            "[install-mecab-ipadic-NEologd] :     egrep => ok\n",
            "[install-mecab-ipadic-NEologd] :     mecab => ok\n",
            "[install-mecab-ipadic-NEologd] :     mecab-config => ok\n",
            "[install-mecab-ipadic-NEologd] :     make => ok\n",
            "[install-mecab-ipadic-NEologd] :     curl => ok\n",
            "[install-mecab-ipadic-NEologd] :     sed => ok\n",
            "[install-mecab-ipadic-NEologd] :     cat => ok\n",
            "[install-mecab-ipadic-NEologd] :     diff => ok\n",
            "[install-mecab-ipadic-NEologd] :     tar => ok\n",
            "[install-mecab-ipadic-NEologd] :     unxz => ok\n",
            "[install-mecab-ipadic-NEologd] :     xargs => ok\n",
            "[install-mecab-ipadic-NEologd] :     grep => ok\n",
            "[install-mecab-ipadic-NEologd] :     iconv => ok\n",
            "[install-mecab-ipadic-NEologd] :     patch => ok\n",
            "[install-mecab-ipadic-NEologd] :     which => ok\n",
            "[install-mecab-ipadic-NEologd] :     file => ok\n",
            "[install-mecab-ipadic-NEologd] :     openssl => ok\n",
            "[install-mecab-ipadic-NEologd] :     awk => ok\n",
            "\n",
            "[install-mecab-ipadic-NEologd] : mecab-ipadic-NEologd is already up-to-date\n",
            "\n",
            "[install-mecab-ipadic-NEologd] : mecab-ipadic-NEologd will be install to /usr/lib/x86_64-linux-gnu/mecab/dic/mecab-ipadic-neologd\n",
            "\n",
            "[install-mecab-ipadic-NEologd] : Make mecab-ipadic-NEologd\n",
            "[make-mecab-ipadic-NEologd] : Start..\n",
            "[make-mecab-ipadic-NEologd] : Check local seed directory\n",
            "[make-mecab-ipadic-NEologd] : Check local seed file\n",
            "[make-mecab-ipadic-NEologd] : Check local build directory\n",
            "[make-mecab-ipadic-NEologd] : Download original mecab-ipadic file\n",
            "[make-mecab-ipadic-NEologd] : Original mecab-ipadic file is already there.\n",
            "[make-mecab-ipadic-NEologd] : Decompress original mecab-ipadic file\n",
            "[make-mecab-ipadic-NEologd] : Delete old mecab-ipadic-2.7.0-20070801-neologd-20200430 directory\n",
            "mecab-ipadic-2.7.0-20070801/\n",
            "mecab-ipadic-2.7.0-20070801/README\n",
            "mecab-ipadic-2.7.0-20070801/AUTHORS\n",
            "mecab-ipadic-2.7.0-20070801/COPYING\n",
            "mecab-ipadic-2.7.0-20070801/ChangeLog\n",
            "mecab-ipadic-2.7.0-20070801/INSTALL\n",
            "mecab-ipadic-2.7.0-20070801/Makefile.am\n",
            "mecab-ipadic-2.7.0-20070801/Makefile.in\n",
            "mecab-ipadic-2.7.0-20070801/NEWS\n",
            "mecab-ipadic-2.7.0-20070801/aclocal.m4\n",
            "mecab-ipadic-2.7.0-20070801/config.guess\n",
            "mecab-ipadic-2.7.0-20070801/config.sub\n",
            "mecab-ipadic-2.7.0-20070801/configure\n",
            "mecab-ipadic-2.7.0-20070801/configure.in\n",
            "mecab-ipadic-2.7.0-20070801/install-sh\n",
            "mecab-ipadic-2.7.0-20070801/missing\n",
            "mecab-ipadic-2.7.0-20070801/mkinstalldirs\n",
            "mecab-ipadic-2.7.0-20070801/Adj.csv\n",
            "mecab-ipadic-2.7.0-20070801/Adnominal.csv\n",
            "mecab-ipadic-2.7.0-20070801/Adverb.csv\n",
            "mecab-ipadic-2.7.0-20070801/Auxil.csv\n",
            "mecab-ipadic-2.7.0-20070801/Conjunction.csv\n",
            "mecab-ipadic-2.7.0-20070801/Filler.csv\n",
            "mecab-ipadic-2.7.0-20070801/Interjection.csv\n",
            "mecab-ipadic-2.7.0-20070801/Noun.adjv.csv\n",
            "mecab-ipadic-2.7.0-20070801/Noun.adverbal.csv\n",
            "mecab-ipadic-2.7.0-20070801/Noun.csv\n",
            "mecab-ipadic-2.7.0-20070801/Noun.demonst.csv\n",
            "mecab-ipadic-2.7.0-20070801/Noun.nai.csv\n",
            "mecab-ipadic-2.7.0-20070801/Noun.name.csv\n",
            "mecab-ipadic-2.7.0-20070801/Noun.number.csv\n",
            "mecab-ipadic-2.7.0-20070801/Noun.org.csv\n",
            "mecab-ipadic-2.7.0-20070801/Noun.others.csv\n",
            "mecab-ipadic-2.7.0-20070801/Noun.place.csv\n",
            "mecab-ipadic-2.7.0-20070801/Noun.proper.csv\n",
            "mecab-ipadic-2.7.0-20070801/Noun.verbal.csv\n",
            "mecab-ipadic-2.7.0-20070801/Others.csv\n",
            "mecab-ipadic-2.7.0-20070801/Postp-col.csv\n",
            "mecab-ipadic-2.7.0-20070801/Postp.csv\n",
            "mecab-ipadic-2.7.0-20070801/Prefix.csv\n",
            "mecab-ipadic-2.7.0-20070801/Suffix.csv\n",
            "mecab-ipadic-2.7.0-20070801/Symbol.csv\n",
            "mecab-ipadic-2.7.0-20070801/Verb.csv\n",
            "mecab-ipadic-2.7.0-20070801/char.def\n",
            "mecab-ipadic-2.7.0-20070801/feature.def\n",
            "mecab-ipadic-2.7.0-20070801/left-id.def\n",
            "mecab-ipadic-2.7.0-20070801/matrix.def\n",
            "mecab-ipadic-2.7.0-20070801/pos-id.def\n",
            "mecab-ipadic-2.7.0-20070801/rewrite.def\n",
            "mecab-ipadic-2.7.0-20070801/right-id.def\n",
            "mecab-ipadic-2.7.0-20070801/unk.def\n",
            "mecab-ipadic-2.7.0-20070801/dicrc\n",
            "mecab-ipadic-2.7.0-20070801/RESULT\n",
            "[make-mecab-ipadic-NEologd] : Configure custom system dictionary on /content/mecab-ipadic-neologd/libexec/../build/mecab-ipadic-2.7.0-20070801-neologd-20200430\n",
            "checking for a BSD-compatible install... /usr/bin/install -c\n",
            "checking whether build environment is sane... yes\n",
            "checking whether make sets $(MAKE)... yes\n",
            "checking for working aclocal-1.4... missing\n",
            "checking for working autoconf... missing\n",
            "checking for working automake-1.4... missing\n",
            "checking for working autoheader... missing\n",
            "checking for working makeinfo... missing\n",
            "checking for a BSD-compatible install... /usr/bin/install -c\n",
            "checking for mecab-config... /usr/bin/mecab-config\n",
            "configure: creating ./config.status\n",
            "config.status: creating Makefile\n",
            "[make-mecab-ipadic-NEologd] : Encode the character encoding of system dictionary resources from EUC_JP to UTF-8\n",
            "./../../libexec/iconv_euc_to_utf8.sh ./Adverb.csv \n",
            "./../../libexec/iconv_euc_to_utf8.sh ./Interjection.csv \n",
            "./../../libexec/iconv_euc_to_utf8.sh ./Noun.org.csv \n",
            "./../../libexec/iconv_euc_to_utf8.sh ./Noun.place.csv \n",
            "./../../libexec/iconv_euc_to_utf8.sh ./Auxil.csv \n",
            "./../../libexec/iconv_euc_to_utf8.sh ./Symbol.csv \n",
            "./../../libexec/iconv_euc_to_utf8.sh ./Prefix.csv \n",
            "./../../libexec/iconv_euc_to_utf8.sh ./Verb.csv \n",
            "./../../libexec/iconv_euc_to_utf8.sh ./Noun.proper.csv \n",
            "./../../libexec/iconv_euc_to_utf8.sh ./Noun.name.csv \n",
            "./../../libexec/iconv_euc_to_utf8.sh ./Conjunction.csv \n",
            "./../../libexec/iconv_euc_to_utf8.sh ./Others.csv \n",
            "./../../libexec/iconv_euc_to_utf8.sh ./Filler.csv \n",
            "./../../libexec/iconv_euc_to_utf8.sh ./Noun.csv \n",
            "./../../libexec/iconv_euc_to_utf8.sh ./Noun.nai.csv \n",
            "./../../libexec/iconv_euc_to_utf8.sh ./Suffix.csv \n",
            "./../../libexec/iconv_euc_to_utf8.sh ./Noun.verbal.csv \n",
            "./../../libexec/iconv_euc_to_utf8.sh ./Postp.csv \n",
            "./../../libexec/iconv_euc_to_utf8.sh ./Noun.demonst.csv \n",
            "./../../libexec/iconv_euc_to_utf8.sh ./Adnominal.csv \n",
            "./../../libexec/iconv_euc_to_utf8.sh ./Postp-col.csv \n",
            "./../../libexec/iconv_euc_to_utf8.sh ./Adj.csv \n",
            "./../../libexec/iconv_euc_to_utf8.sh ./Noun.number.csv \n",
            "./../../libexec/iconv_euc_to_utf8.sh ./Noun.adjv.csv \n",
            "./../../libexec/iconv_euc_to_utf8.sh ./Noun.others.csv \n",
            "./../../libexec/iconv_euc_to_utf8.sh ./Noun.adverbal.csv \n",
            "rm ./Adverb.csv \n",
            "rm ./Interjection.csv \n",
            "rm ./Noun.org.csv \n",
            "rm ./Noun.place.csv \n",
            "rm ./Auxil.csv \n",
            "rm ./Symbol.csv \n",
            "rm ./Prefix.csv \n",
            "rm ./Verb.csv \n",
            "rm ./Noun.proper.csv \n",
            "rm ./Noun.name.csv \n",
            "rm ./Conjunction.csv \n",
            "rm ./Others.csv \n",
            "rm ./Filler.csv \n",
            "rm ./Noun.csv \n",
            "rm ./Noun.nai.csv \n",
            "rm ./Suffix.csv \n",
            "rm ./Noun.verbal.csv \n",
            "rm ./Postp.csv \n",
            "rm ./Noun.demonst.csv \n",
            "rm ./Adnominal.csv \n",
            "rm ./Postp-col.csv \n",
            "rm ./Adj.csv \n",
            "rm ./Noun.number.csv \n",
            "rm ./Noun.adjv.csv \n",
            "rm ./Noun.others.csv \n",
            "rm ./Noun.adverbal.csv \n",
            "./../../libexec/iconv_euc_to_utf8.sh ./pos-id.def \n",
            "./../../libexec/iconv_euc_to_utf8.sh ./feature.def \n",
            "./../../libexec/iconv_euc_to_utf8.sh ./right-id.def \n",
            "./../../libexec/iconv_euc_to_utf8.sh ./char.def \n",
            "./../../libexec/iconv_euc_to_utf8.sh ./left-id.def \n",
            "./../../libexec/iconv_euc_to_utf8.sh ./matrix.def \n",
            "./../../libexec/iconv_euc_to_utf8.sh ./rewrite.def \n",
            "./../../libexec/iconv_euc_to_utf8.sh ./unk.def \n",
            "rm ./pos-id.def \n",
            "rm ./feature.def \n",
            "rm ./right-id.def \n",
            "rm ./char.def \n",
            "rm ./left-id.def \n",
            "rm ./matrix.def \n",
            "rm ./rewrite.def \n",
            "rm ./unk.def \n",
            "mv ./left-id.def.utf8 ./left-id.def \n",
            "mv ./Others.csv.utf8 ./Others.csv \n",
            "mv ./Postp.csv.utf8 ./Postp.csv \n",
            "mv ./char.def.utf8 ./char.def \n",
            "mv ./Filler.csv.utf8 ./Filler.csv \n",
            "mv ./Noun.csv.utf8 ./Noun.csv \n",
            "mv ./Symbol.csv.utf8 ./Symbol.csv \n",
            "mv ./Prefix.csv.utf8 ./Prefix.csv \n",
            "mv ./Adj.csv.utf8 ./Adj.csv \n",
            "mv ./Noun.name.csv.utf8 ./Noun.name.csv \n",
            "mv ./Interjection.csv.utf8 ./Interjection.csv \n",
            "mv ./matrix.def.utf8 ./matrix.def \n",
            "mv ./pos-id.def.utf8 ./pos-id.def \n",
            "mv ./Noun.proper.csv.utf8 ./Noun.proper.csv \n",
            "mv ./Noun.org.csv.utf8 ./Noun.org.csv \n",
            "mv ./Auxil.csv.utf8 ./Auxil.csv \n",
            "mv ./Noun.adjv.csv.utf8 ./Noun.adjv.csv \n",
            "mv ./Noun.place.csv.utf8 ./Noun.place.csv \n",
            "mv ./Noun.nai.csv.utf8 ./Noun.nai.csv \n",
            "mv ./Adverb.csv.utf8 ./Adverb.csv \n",
            "mv ./Adnominal.csv.utf8 ./Adnominal.csv \n",
            "mv ./Verb.csv.utf8 ./Verb.csv \n",
            "mv ./Noun.demonst.csv.utf8 ./Noun.demonst.csv \n",
            "mv ./Noun.verbal.csv.utf8 ./Noun.verbal.csv \n",
            "mv ./Noun.number.csv.utf8 ./Noun.number.csv \n",
            "mv ./Postp-col.csv.utf8 ./Postp-col.csv \n",
            "mv ./rewrite.def.utf8 ./rewrite.def \n",
            "mv ./Noun.others.csv.utf8 ./Noun.others.csv \n",
            "mv ./right-id.def.utf8 ./right-id.def \n",
            "mv ./unk.def.utf8 ./unk.def \n",
            "mv ./Conjunction.csv.utf8 ./Conjunction.csv \n",
            "mv ./Noun.adverbal.csv.utf8 ./Noun.adverbal.csv \n",
            "mv ./feature.def.utf8 ./feature.def \n",
            "mv ./Suffix.csv.utf8 ./Suffix.csv \n",
            "[make-mecab-ipadic-NEologd] : Fix yomigana field of IPA dictionary\n",
            "patching file Noun.csv\n",
            "patching file Noun.place.csv\n",
            "patching file Verb.csv\n",
            "patching file Noun.verbal.csv\n",
            "patching file Noun.name.csv\n",
            "patching file Noun.adverbal.csv\n",
            "patching file Noun.csv\n",
            "patching file Noun.name.csv\n",
            "patching file Noun.org.csv\n",
            "patching file Noun.others.csv\n",
            "patching file Noun.place.csv\n",
            "patching file Noun.proper.csv\n",
            "patching file Noun.verbal.csv\n",
            "patching file Prefix.csv\n",
            "patching file Suffix.csv\n",
            "patching file Noun.proper.csv\n",
            "patching file Noun.csv\n",
            "patching file Noun.name.csv\n",
            "patching file Noun.org.csv\n",
            "patching file Noun.place.csv\n",
            "patching file Noun.proper.csv\n",
            "patching file Noun.verbal.csv\n",
            "patching file Noun.name.csv\n",
            "patching file Noun.org.csv\n",
            "patching file Noun.place.csv\n",
            "patching file Noun.proper.csv\n",
            "patching file Suffix.csv\n",
            "patching file Noun.demonst.csv\n",
            "patching file Noun.csv\n",
            "patching file Noun.name.csv\n",
            "[make-mecab-ipadic-NEologd] : Copy user dictionary resource\n",
            "[make-mecab-ipadic-NEologd] : Install adverb entries using /content/mecab-ipadic-neologd/libexec/../seed/neologd-adverb-dict-seed.20150623.csv.xz\n",
            "[make-mecab-ipadic-NEologd] : Install interjection entries using /content/mecab-ipadic-neologd/libexec/../seed/neologd-interjection-dict-seed.20170216.csv.xz\n",
            "[make-mecab-ipadic-NEologd] : Install noun orthographic variant entries using /content/mecab-ipadic-neologd/libexec/../seed/neologd-common-noun-ortho-variant-dict-seed.20170228.csv.xz\n",
            "[make-mecab-ipadic-NEologd] : Install noun orthographic variant entries using /content/mecab-ipadic-neologd/libexec/../seed/neologd-proper-noun-ortho-variant-dict-seed.20161110.csv.xz\n",
            "[make-mecab-ipadic-NEologd] : Install entries of orthographic variant of a noun used as verb form using /content/mecab-ipadic-neologd/libexec/../seed/neologd-noun-sahen-conn-ortho-variant-dict-seed.20160323.csv.xz\n",
            "[make-mecab-ipadic-NEologd] : Install frequent adjective orthographic variant entries using /content/mecab-ipadic-neologd/libexec/../seed/neologd-adjective-std-dict-seed.20151126.csv.xz\n",
            "[make-mecab-ipadic-NEologd] : Install infrequent adjective orthographic variant entries using /content/mecab-ipadic-neologd/libexec/../seed/neologd-adjective-exp-dict-seed.20151126.csv.xz\n",
            "[make-mecab-ipadic-NEologd] : Install adjective verb orthographic variant entries using /content/mecab-ipadic-neologd/libexec/../seed/neologd-adjective-verb-dict-seed.20160324.csv.xz\n",
            "[make-mecab-ipadic-NEologd] : Install infrequent datetime representation entries using /content/mecab-ipadic-neologd/libexec/../seed/neologd-date-time-infreq-dict-seed.20190415.csv.xz\n",
            "[make-mecab-ipadic-NEologd] : Install infrequent quantity representation entries using /content/mecab-ipadic-neologd/libexec/../seed/neologd-quantity-infreq-dict-seed.20190415.csv.xz\n",
            "[make-mecab-ipadic-NEologd] : Install entries of ill formed words using /content/mecab-ipadic-neologd/libexec/../seed/neologd-ill-formed-words-dict-seed.20170127.csv.xz\n",
            "[make-mecab-ipadic-NEologd] : Re-Index system dictionary\n",
            "reading ./unk.def ... 40\n",
            "emitting double-array: 100% |###########################################| \n",
            "./model.def is not found. skipped.\n",
            "reading ./neologd-common-noun-ortho-variant-dict-seed.20170228.csv ... 152869\n",
            "reading ./Adverb.csv ... 3032\n",
            "reading ./Interjection.csv ... 252\n",
            "reading ./Noun.org.csv ... 17149\n",
            "reading ./neologd-adjective-std-dict-seed.20151126.csv ... 507812\n",
            "reading ./Noun.place.csv ... 73194\n",
            "reading ./Auxil.csv ... 199\n",
            "reading ./Symbol.csv ... 208\n",
            "reading ./neologd-proper-noun-ortho-variant-dict-seed.20161110.csv ... 138379\n",
            "reading ./neologd-quantity-infreq-dict-seed.20190415.csv ... 229216\n",
            "reading ./Prefix.csv ... 224\n",
            "reading ./neologd-date-time-infreq-dict-seed.20190415.csv ... 16866\n",
            "reading ./Verb.csv ... 130750\n",
            "reading ./Noun.proper.csv ... 27493\n",
            "reading ./Noun.name.csv ... 34215\n",
            "reading ./Conjunction.csv ... 171\n",
            "reading ./neologd-adverb-dict-seed.20150623.csv ... 139792\n",
            "reading ./Others.csv ... 2\n",
            "reading ./Filler.csv ... 19\n",
            "reading ./Noun.csv ... 60734\n",
            "reading ./neologd-ill-formed-words-dict-seed.20170127.csv ... 60616\n",
            "reading ./Noun.nai.csv ... 42\n",
            "reading ./Suffix.csv ... 1448\n",
            "reading ./neologd-adjective-verb-dict-seed.20160324.csv ... 20268\n",
            "reading ./Noun.verbal.csv ... 12150\n",
            "reading ./neologd-interjection-dict-seed.20170216.csv ... 4701\n",
            "reading ./Postp.csv ... 146\n",
            "reading ./Noun.demonst.csv ... 120\n",
            "reading ./Adnominal.csv ... 135\n",
            "reading ./Postp-col.csv ... 91\n",
            "reading ./Adj.csv ... 27210\n",
            "reading ./mecab-user-dict-seed.20200430.csv ... 3190186\n",
            "reading ./neologd-adjective-exp-dict-seed.20151126.csv ... 1051146\n",
            "reading ./Noun.number.csv ... 42\n",
            "reading ./Noun.adjv.csv ... 3328\n",
            "reading ./Noun.others.csv ... 153\n",
            "reading ./Noun.adverbal.csv ... 808\n",
            "reading ./neologd-noun-sahen-conn-ortho-variant-dict-seed.20160323.csv ... 26058\n",
            "emitting double-array: 100% |###########################################| \n",
            "reading ./matrix.def ... 1316x1316\n",
            "emitting matrix      : 100% |###########################################| \n",
            "\n",
            "done!\n",
            "[make-mecab-ipadic-NEologd] : Make custom system dictionary on /content/mecab-ipadic-neologd/libexec/../build/mecab-ipadic-2.7.0-20070801-neologd-20200430\n",
            "make: Nothing to be done for 'all'.\n",
            "[make-mecab-ipadic-NEologd] : Finish..\n",
            "[install-mecab-ipadic-NEologd] : Get results of tokenize test\n",
            "[test-mecab-ipadic-NEologd] : Start..\n",
            "[test-mecab-ipadic-NEologd] : Replace timestamp from 'git clone' date to 'git commit' date\n",
            "[test-mecab-ipadic-NEologd] : Get buzz phrases\n",
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "100  1573  100  1573    0     0   1237      0  0:00:01  0:00:01 --:--:--  1237\n",
            "[test-mecab-ipadic-NEologd] : Get difference between default system dictionary and mecab-ipadic-NEologd\n",
            "[test-mecab-ipadic-NEologd] : Tokenize phrase using default system dictionary\n",
            "[test-mecab-ipadic-NEologd] : Tokenize phrase using mecab-ipadic-NEologd\n",
            "[test-mecab-ipadic-NEologd] : Get result of diff\n",
            "[test-mecab-ipadic-NEologd] : Please check difference between default system dictionary and mecab-ipadic-NEologd\n",
            "\n",
            "default system dictionary\t  |\tmecab-ipadic-NEologd\n",
            "\n",
            "[test-mecab-ipadic-NEologd] : Finish..\n",
            "\n",
            "[install-mecab-ipadic-NEologd] : Please check the list of differences in the upper part.\n",
            "\n",
            "[install-mecab-ipadic-NEologd] : Do you want to install mecab-ipadic-NEologd? Type yes or no.\n",
            "[install-mecab-ipadic-NEologd] : OK. Let's install mecab-ipadic-NEologd.\n",
            "[install-mecab-ipadic-NEologd] : Start..\n",
            "[install-mecab-ipadic-NEologd] : /usr/lib/x86_64-linux-gnu/mecab/dic is current user's directory\n",
            "[install-mecab-ipadic-NEologd] : Make install to /usr/lib/x86_64-linux-gnu/mecab/dic/mecab-ipadic-neologd\n",
            "make[1]: Entering directory '/content/mecab-ipadic-neologd/build/mecab-ipadic-2.7.0-20070801-neologd-20200430'\n",
            "make[1]: Nothing to be done for 'install-exec-am'.\n",
            "/bin/bash ./mkinstalldirs /usr/lib/x86_64-linux-gnu/mecab/dic/mecab-ipadic-neologd\n",
            " /usr/bin/install -c -m 644 ./matrix.bin /usr/lib/x86_64-linux-gnu/mecab/dic/mecab-ipadic-neologd/matrix.bin\n",
            " /usr/bin/install -c -m 644 ./char.bin /usr/lib/x86_64-linux-gnu/mecab/dic/mecab-ipadic-neologd/char.bin\n",
            " /usr/bin/install -c -m 644 ./sys.dic /usr/lib/x86_64-linux-gnu/mecab/dic/mecab-ipadic-neologd/sys.dic\n",
            " /usr/bin/install -c -m 644 ./unk.dic /usr/lib/x86_64-linux-gnu/mecab/dic/mecab-ipadic-neologd/unk.dic\n",
            " /usr/bin/install -c -m 644 ./left-id.def /usr/lib/x86_64-linux-gnu/mecab/dic/mecab-ipadic-neologd/left-id.def\n",
            " /usr/bin/install -c -m 644 ./right-id.def /usr/lib/x86_64-linux-gnu/mecab/dic/mecab-ipadic-neologd/right-id.def\n",
            " /usr/bin/install -c -m 644 ./rewrite.def /usr/lib/x86_64-linux-gnu/mecab/dic/mecab-ipadic-neologd/rewrite.def\n",
            " /usr/bin/install -c -m 644 ./pos-id.def /usr/lib/x86_64-linux-gnu/mecab/dic/mecab-ipadic-neologd/pos-id.def\n",
            " /usr/bin/install -c -m 644 ./dicrc /usr/lib/x86_64-linux-gnu/mecab/dic/mecab-ipadic-neologd/dicrc\n",
            "make[1]: Leaving directory '/content/mecab-ipadic-neologd/build/mecab-ipadic-2.7.0-20070801-neologd-20200430'\n",
            "\n",
            "[install-mecab-ipadic-NEologd] : Install completed.\n",
            "[install-mecab-ipadic-NEologd] : When you use MeCab, you can set '/usr/lib/x86_64-linux-gnu/mecab/dic/mecab-ipadic-neologd' as a value of '-d' option of MeCab.\n",
            "[install-mecab-ipadic-NEologd] : Usage of mecab-ipadic-NEologd is here.\n",
            "Usage:\n",
            "    $ mecab -d /usr/lib/x86_64-linux-gnu/mecab/dic/mecab-ipadic-neologd ...\n",
            "\n",
            "[install-mecab-ipadic-NEologd] : Finish..\n",
            "[install-mecab-ipadic-NEologd] : Finish..\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "80SqXmdfc3hO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import subprocess\n",
        "\n",
        "cmd='echo `mecab-config --dicdir`\"/mecab-ipadic-neologd\"'\n",
        "path_neologd = (subprocess.Popen(cmd, stdout=subprocess.PIPE,\n",
        "                           shell=True).communicate()[0]).decode('utf-8')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JOTmm1u_c7qE",
        "colab_type": "text"
      },
      "source": [
        "## 2：MeCab"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xo5kFZUhegX9",
        "colab_type": "code",
        "outputId": "f1f4f459-5026-455b-d4ea-3a338aa85479",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 193
        }
      },
      "source": [
        "import MeCab\n",
        "\n",
        "m=MeCab.Tagger(\"-Ochasen\")\n",
        "\n",
        "text = \"예제。\"\n",
        "\n",
        "text_segmented = m.parse(text)\n",
        "print(text_segmented)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "EOS\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "46pmM04neg5F",
        "colab_type": "code",
        "outputId": "860ddfe1-973c-4253-9952-09635051cfaa",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "m=MeCab.Tagger(\"-Owakati\")\n",
        "text_segmented = m.parse(text)\n",
        "print(text_segmented)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " \n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pqrfZ5w2fEIb",
        "colab_type": "code",
        "outputId": "c9de2978-40f3-41a2-864a-441d755ce61e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "m=MeCab.Tagger(\"-Oyomi\")\n",
        "text_segmented = m.parse(text)\n",
        "print(text_segmented)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y6pbeCbRgI8L",
        "colab_type": "text"
      },
      "source": [
        "### "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pHuCvyP1gFOp",
        "colab_type": "code",
        "outputId": "64f82674-9ab4-44ff-9d21-c57c9e75adb8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 175
        }
      },
      "source": [
        "m=MeCab.Tagger(\"-Ochasen -d \"+str(path_neologd))  # NEologd\n",
        "\n",
        "text = \"\"\n",
        "\n",
        "text_segmented = m.parse(text)\n",
        "print(text_segmented)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [

            "EOS\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F04QpCE2gLy-",
        "colab_type": "code",
        "outputId": "4e6f9792-cd7c-4489-cd4a-48084202b2d3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "m=MeCab.Tagger(\"-Owakati -d \"+str(path_neologd))  # NEologdへのパスを追加\n",
        "text_segmented = m.parse(text)\n",
        "print(text_segmented)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QMQ46P0mlyjC",
        "colab_type": "text"
      },
      "source": [
        "## 3：BERT"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P5_fTFX9gO6D",
        "colab_type": "code",
        "outputId": "605601b9-be0d-470e-9661-df80b70d593c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 316
        }
      },
      "source": [
        "!pip install transformers==2.9.0"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: transformers==2.9.0 in /usr/local/lib/python3.6/dist-packages (2.9.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers==2.9.0) (2.23.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers==2.9.0) (4.41.1)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.6/dist-packages (from transformers==2.9.0) (0.0.43)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.6/dist-packages (from transformers==2.9.0) (0.1.86)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers==2.9.0) (3.0.12)\n",
            "Requirement already satisfied: tokenizers==0.7.0 in /usr/local/lib/python3.6/dist-packages (from transformers==2.9.0) (0.7.0)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers==2.9.0) (0.7)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers==2.9.0) (2019.12.20)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers==2.9.0) (1.18.4)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.9.0) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.9.0) (2020.4.5.1)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.9.0) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.9.0) (2.9)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==2.9.0) (0.14.1)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==2.9.0) (7.1.2)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==2.9.0) (1.12.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0PHUrr-DmIdX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "from transformers.modeling_bert import BertModel\n",
        "from transformers.tokenization_bert_japanese import BertJapaneseTokenizer\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XuwcEk47n3U0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 分かち書きをするtokenizerです\n",
        "tokenizer = BertJapaneseTokenizer.from_pretrained('bert-base-japanese-whole-word-masking')\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y-Djqpnjn-g6",
        "colab_type": "code",
        "outputId": "698e8218-641b-4660-a036-a36af0d5ee02",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "# BERT\n",
        "model = BertModel.from_pretrained('bert-base-japanese-whole-word-masking')\n",
        "print(model)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "BertModel(\n",
            "  (embeddings): BertEmbeddings(\n",
            "    (word_embeddings): Embedding(32000, 768, padding_idx=0)\n",
            "    (position_embeddings): Embedding(512, 768)\n",
            "    (token_type_embeddings): Embedding(2, 768)\n",
            "    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "    (dropout): Dropout(p=0.1, inplace=False)\n",
            "  )\n",
            "  (encoder): BertEncoder(\n",
            "    (layer): ModuleList(\n",
            "      (0): BertLayer(\n",
            "        (attention): BertAttention(\n",
            "          (self): BertSelfAttention(\n",
            "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (output): BertSelfOutput(\n",
            "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (intermediate): BertIntermediate(\n",
            "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "        )\n",
            "        (output): BertOutput(\n",
            "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "      (1): BertLayer(\n",
            "        (attention): BertAttention(\n",
            "          (self): BertSelfAttention(\n",
            "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (output): BertSelfOutput(\n",
            "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (intermediate): BertIntermediate(\n",
            "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "        )\n",
            "        (output): BertOutput(\n",
            "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "      (2): BertLayer(\n",
            "        (attention): BertAttention(\n",
            "          (self): BertSelfAttention(\n",
            "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (output): BertSelfOutput(\n",
            "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (intermediate): BertIntermediate(\n",
            "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "        )\n",
            "        (output): BertOutput(\n",
            "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "      (3): BertLayer(\n",
            "        (attention): BertAttention(\n",
            "          (self): BertSelfAttention(\n",
            "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (output): BertSelfOutput(\n",
            "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (intermediate): BertIntermediate(\n",
            "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "        )\n",
            "        (output): BertOutput(\n",
            "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "      (4): BertLayer(\n",
            "        (attention): BertAttention(\n",
            "          (self): BertSelfAttention(\n",
            "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (output): BertSelfOutput(\n",
            "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (intermediate): BertIntermediate(\n",
            "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "        )\n",
            "        (output): BertOutput(\n",
            "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "      (5): BertLayer(\n",
            "        (attention): BertAttention(\n",
            "          (self): BertSelfAttention(\n",
            "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (output): BertSelfOutput(\n",
            "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (intermediate): BertIntermediate(\n",
            "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "        )\n",
            "        (output): BertOutput(\n",
            "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "      (6): BertLayer(\n",
            "        (attention): BertAttention(\n",
            "          (self): BertSelfAttention(\n",
            "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (output): BertSelfOutput(\n",
            "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (intermediate): BertIntermediate(\n",
            "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "        )\n",
            "        (output): BertOutput(\n",
            "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "      (7): BertLayer(\n",
            "        (attention): BertAttention(\n",
            "          (self): BertSelfAttention(\n",
            "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (output): BertSelfOutput(\n",
            "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (intermediate): BertIntermediate(\n",
            "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "        )\n",
            "        (output): BertOutput(\n",
            "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "      (8): BertLayer(\n",
            "        (attention): BertAttention(\n",
            "          (self): BertSelfAttention(\n",
            "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (output): BertSelfOutput(\n",
            "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (intermediate): BertIntermediate(\n",
            "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "        )\n",
            "        (output): BertOutput(\n",
            "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "      (9): BertLayer(\n",
            "        (attention): BertAttention(\n",
            "          (self): BertSelfAttention(\n",
            "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (output): BertSelfOutput(\n",
            "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (intermediate): BertIntermediate(\n",
            "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "        )\n",
            "        (output): BertOutput(\n",
            "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "      (10): BertLayer(\n",
            "        (attention): BertAttention(\n",
            "          (self): BertSelfAttention(\n",
            "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (output): BertSelfOutput(\n",
            "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (intermediate): BertIntermediate(\n",
            "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "        )\n",
            "        (output): BertOutput(\n",
            "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "      (11): BertLayer(\n",
            "        (attention): BertAttention(\n",
            "          (self): BertSelfAttention(\n",
            "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (output): BertSelfOutput(\n",
            "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (intermediate): BertIntermediate(\n",
            "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "        )\n",
            "        (output): BertOutput(\n",
            "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "  )\n",
            "  (pooler): BertPooler(\n",
            "    (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "    (activation): Tanh()\n",
            "  )\n",
            ")\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9COwchQZoF0y",
        "colab_type": "code",
        "outputId": "4ce7d7a8-1d92-4e8b-b2c9-772aa6471a75",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 369
        }
      },
      "source": [
        "from transformers import BertConfig\n",
        "\n",
        "# \n",
        "config_japanese = BertConfig.from_pretrained('bert-base-japanese-whole-word-masking')\n",
        "print(config_japanese)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "BertConfig {\n",
            "  \"architectures\": [\n",
            "    \"BertForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"layer_norm_eps\": 1e-12,\n",
            "  \"max_position_embeddings\": 512,\n",
            "  \"model_type\": \"bert\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"type_vocab_size\": 2,\n",
            "  \"vocab_size\": 32000\n",
            "}\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3MrPVPzTr_AM",
        "colab_type": "text"
      },
      "source": [
        "。"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YjGRSQwEzj8K",
        "colab_type": "text"
      },
      "source": [
        "## BERT"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LxEbvY5jzk4R",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "text1 = \"\"\n",
        "text2 = \"\"\n",
        "text3 = \"\"\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QJLVJIJZ0t-1",
        "colab_type": "code",
        "outputId": "22625cfc-5bc5-4177-b026-86aed971b37a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "# \n",
        "input_ids1 = tokenizer.encode(text1, return_tensors='pt')  # ptはPyTorchの略\n",
        "\n",
        "print(tokenizer.convert_ids_to_tokens(input_ids1[0].tolist()))  # 文章\n",
        "print(input_ids1)  # id\n"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['[CLS]', ]\n",
            "tensor([[    2,   811,    11, 13700,     7,    58,    10,     8,     3]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RkxX9n4k1yjk",
        "colab_type": "code",
        "outputId": "c8e663bf-cc8f-4707-dd5e-c5cab265a174",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        }
      },
      "source": [
        "# \n",
        "input_ids2 = tokenizer.encode(text2, return_tensors='pt')  # ptはPyTorch\n",
        "\n",
        "print(tokenizer.convert_ids_to_tokens(input_ids2[0].tolist()))  # \n",
        "print(input_ids2)  # id\n"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['[CLS]', '[SEP]']\n",
            "tensor([[    2,  5521,  3118,  4027,    12, 13700,    14,  4897, 28457,     8,\n",
            "             3]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2voPx2yE03nj",
        "colab_type": "code",
        "outputId": "1e24284e-76c6-4ece-d417-926f9b5e6f8b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "# id\n",
        "input_ids3 = tokenizer.encode(text3, return_tensors='pt')  # ptはPyTorchの略\n",
        "\n",
        "print(tokenizer.convert_ids_to_tokens(input_ids3[0].tolist()))  # 文章\n",
        "print(input_ids3)  # id\n"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['[CLS]', '[SEP]']\n",
            "tensor([[   2,  811,   11, 7279,   26,   20,   10,    8,    3]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zvj4O1b516u7",
        "colab_type": "code",
        "outputId": "78e9a4ba-d808-467f-c6ad-18cb988d59c6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "# BERT\n",
        "result1 = model(input_ids1)\n",
        "\n",
        "print(result1[0].shape)\n",
        "print(result1[1].shape)\n",
        "\n",
        "# reult sequence_output, pooled_output, (hidden_states), (attentions)。\n",
        "# 。\n"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([1, 9, 768])\n",
            "torch.Size([1, 768])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bTuqqREy3JJg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# ERT\n",
        "result2 = model(input_ids2)\n",
        "result3 = model(input_ids3)\n",
        "\n",
        "word_vec1 = result1[0][0][3][:]  # 1\n",
        "word_vec2 = result2[0][0][5][:]  # 2\n",
        "word_vec3 = result3[0][0][3][:]  # 3\ㅅ"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mFNkFPwX5Rw7",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "ea958b5a-5ae0-4527-f8b2-a39fddbbcd43"
      },
      "source": [
        "# \n",
        "cos = torch.nn.CosineSimilarity(dim=0)\n",
        "cos_sim_12 = cos(word_vec1, word_vec2)\n",
        "cos_sim_13 = cos(word_vec1, word_vec3)\n",
        "\n",
        "print(cos_sim_12)\n",
        "print(cos_sim_13)"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor(0.6647, grad_fn=<DivBackward0>)\n",
            "tensor(0.7841, grad_fn=<DivBackward0>)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tRGDDBVyQQRD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}