imdb.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "imdb.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.7"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/tianjianjiang/nlp_data_aug/blob/%231-control_random_factors/imdb.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o9LbB0krq6z2",
        "colab_type": "text"
      },
      "source": [
        "# Prepare"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jMnqAd1HJ1Br",
        "colab_type": "text"
      },
      "source": [
        "An example as baseline: [ULMFit](https://nbviewer.jupyter.org/github/fastai/fastai/blob/master/examples/ULMFit.ipynb) tutorial.\n",
        "\n",
        "> Fine-tuning a forward and backward langauge model to get to 95.4% accuracy on the IMDB movie reviews dataset. This tutorial is done with fastai v1.0.53.\n",
        "\n",
        "> The example was run on a Titan RTX (24 GB of RAM) so you will probably need to adjust the batch size accordinly. If you divide it by 2, don't forget to divide the learning rate by 2 as well in the following cells. You can also reduce a little bit the bptt to gain a bit of memory."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QF718toQqs83",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Ensure GPU spec; T4 is for colab and one can change it for another env.\n",
        "gpu_list = !nvidia-smi -L\n",
        "if gpu_list[0].startswith('NVIDIA-SMI has failed'):\n",
        "  print('Runtime type should be GPU.')\n",
        "elif not gpu_list[0].startswith('GPU 0: Tesla T4'):\n",
        "  display(gpu_list)\n",
        "  print('Please reset all runtimes. We need a Tesla T4 to reproduce the experiments!')\n",
        "else:\n",
        "  display(gpu_list)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0OcNDqMgrCca",
        "colab_type": "text"
      },
      "source": [
        "## Dependency"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jrjoIzboW09N",
        "colab_type": "text"
      },
      "source": [
        "### Install"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2rwtTuykrEWy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Ensure no surprises from conflict packages.\n",
        "!pip check"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bymTjo9wrz5f",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%capture pip_logs\n",
        "!pip install -U fastai==1.0.55 ipyexperiments jupyter-console==5.2.0 coveralls coverage datascience albumentations"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ojbk71i1xaFi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip check"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7z1_46NArGi_",
        "colab_type": "text"
      },
      "source": [
        "### Import"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LMm8qgQqD2eC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import gc\n",
        "import math\n",
        "from pathlib import Path\n",
        "import random\n",
        "\n",
        "import numpy as np\n",
        "import torch\n",
        "from google.colab import drive\n",
        "\n",
        "from fastai import basic_data, basic_train, core\n",
        "from fastai import *\n",
        "from fastai.callbacks import CSVLogger\n",
        "from fastai.core import plt\n",
        "from fastai.text import *\n",
        "from fastprogress import fastprogress\n",
        "\n",
        "from ipyexperiments import *"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DTVchfsRnLgI",
        "colab_type": "text"
      },
      "source": [
        "### Init\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QnxWkqafNxvA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Not set earlier because pip may require a restart.\n",
        "SESSN_START_T, = !date +%Y%m%dT%H%M"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iTX1BGNQN6VB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%load_ext autoreload\n",
        "%autoreload 2"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "db_RafAMN2mK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# A special treatment for colab to decrease network traffic.\n",
        "fastprogress.NO_BAR = True\n",
        "master_bar, progress_bar = fastprogress.force_console_behavior()\n",
        "basic_train.master_bar, basic_train.progress_bar = master_bar, progress_bar\n",
        "basic_data.master_bar, basic_data.progress_bar = master_bar, progress_bar\n",
        "dataclass.master_bar, dataclass.progress_bar = master_bar, progress_bar\n",
        "text.master_bar, text.progress_bar = master_bar, progress_bar\n",
        "text.data.master_bar, text.data.progress_bar = master_bar, progress_bar\n",
        "core.master_bar, core.progress_bar = master_bar, progress_bar"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iDMmtPemp8H3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "COLAB_CONTENT_DIR_P = Path('/content')\n",
        "GD_DIR_P = COLAB_CONTENT_DIR_P / 'gdrive'\n",
        "drive.mount(str(GD_DIR_P), force_remount=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uDRg1_MUpXjy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "BASE_DIR_P = GD_DIR_P / 'My Drive/imdb'\n",
        "BASE_DIR_P.mkdir(parents=True, exist_ok=True)\n",
        "DATA_DIR_P = BASE_DIR_P / 'data'\n",
        "DATA_DIR_P.mkdir(parents=True, exist_ok=True)\n",
        "MDLS_DIR_P = BASE_DIR_P / 'models'\n",
        "MDLS_DIR_P.mkdir(parents=True, exist_ok=True)\n",
        "LOGS_DIR_P = BASE_DIR_P / 'logs'\n",
        "LOGS_DIR_P.mkdir(parents=True, exist_ok=True)\n",
        "\n",
        "FASTAI_DATA_DIR_P = Path('/root/.fastai/data')\n",
        "FASTAI_DATA_DIR_P.mkdir(parents=True, exist_ok=True)\n",
        "\n",
        "COLAB_DATA_DIR_P = COLAB_CONTENT_DIR_P / 'data'\n",
        "if not COLAB_DATA_DIR_P.is_symlink():\n",
        "  COLAB_DATA_DIR_P.symlink_to(FASTAI_DATA_DIR_P)\n",
        "if (COLAB_CONTENT_DIR_P / 'sample_data').exists():\n",
        "  !set -x; rm -rf /content/sample_data/"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O2RKejF7YdPR",
        "colab_type": "text"
      },
      "source": [
        "# Assign"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sjmk-4Y_OWc-",
        "colab_type": "text"
      },
      "source": [
        "## Shared Hyperparams"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y18xZgbnXVMo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "lm_bs = 128\n",
        "cf_bs = round(lm_bs / 2)\n",
        "print(f'Our lm_bs: {lm_bs}; cf_bs: {cf_bs}')\n",
        "bptt = 80  # From the example, but fastai defaults to 70.\n",
        "moms = (0.8, 0.7)\n",
        "\n",
        "FW_LM_DBNCH_FILE_S = f'fw_lm_dbnch-b{lm_bs}.pkl'\n",
        "BW_LM_DBNCH_FILE_S = f'bw_lm_dbnch-b{lm_bs}.pkl'\n",
        "FW_CF_DBNCH_FILE_S = f'fw_cf_dbnch-b{cf_bs}.pkl'\n",
        "BW_CF_DBNCH_FILE_S = f'bw_cf_dbnch-b{cf_bs}.pkl'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B_qzqZBBnWM2",
        "colab_type": "text"
      },
      "source": [
        "## LM-specific Hyperparams"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aw3aDcB_nT20",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Decrease the lr from the example's 2e-2 proportionally to the orig lm bs 256.\n",
        "ORIG_LM_BS = 256\n",
        "ORIG_LM_LR = 2e-2\n",
        "# lm_lr = ORIG_LM_LR\n",
        "# lm_lr = lm_bs / ORIG_LM_BS * ORIG_LM_LR\n",
        "# lm_lr = round(lm_lr, 7)\n",
        "# print(f'In proportion to our lm_bs, our lm_lr : {lm_lr}')\n",
        "\n",
        "lm_drop_mult = 1.0\n",
        "lm_wd = 0.1  # From the example, except forward classifier uses fastai default 1e-2.\n",
        "\n",
        "# FW_ENC_NAME = f'fw_enc-b{lm_bs}-lr{lm_lr}'\n",
        "# BW_ENC_NAME = f'bw_enc-b{lm_bs}-lr{lm_lr}'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jwAPR52UFvUW",
        "colab_type": "text"
      },
      "source": [
        "## CF-specific Hyperparams"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NR72_ylXFu7l",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ORIG_CF_BS = round(ORIG_LM_BS / 2)\n",
        "ORIG_CF_LR = 1e-1\n",
        "# cf_lr = ORIG_CF_LR\n",
        "# cf_lr = cf_bs / ORIG_CF_BS * ORIG_CF_LR * 1.2\n",
        "# cf_lr = round(cf_lr, 7)\n",
        "# print(f'In proportion to our cf_bs, our cf_lr: {cf_lr}')\n",
        "\n",
        "cf_drop_mult = lm_drop_mult / 2\n",
        "cf_wd = 0.1\n",
        "\n",
        "# FW_CF_NAME = f'fw_cf-b{cf_bs}-lr{cf_lr}'\n",
        "# BW_CF_NAME = f'bw_cf-b{cf_bs}-lr{cf_lr}'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DNg8nWEyOj1F",
        "colab_type": "text"
      },
      "source": [
        "## Args"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P381SP4eYxv8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Set num_workers to main process since the training set will be shuffled.\n",
        "n_dbnch_wrkrs = 0"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ruGmJGQX9zh-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "plt.style.use(['dark_background','seaborn-poster','seaborn-deep'])\n",
        "plt.rcParams['axes.grid'] = True\n",
        "plt.rcParams['axes.grid.axis'] = 'x'\n",
        "plt.rcParams['axes.grid.which'] = 'both'\n",
        "plt.rcParams['grid.alpha'] = 0.5\n",
        "plt.rcParams['grid.color'] = 'xkcd:lime green'\n",
        "plt.rcParams['grid.linestyle'] = ':'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CEfPdqf-I-bZ",
        "colab_type": "text"
      },
      "source": [
        "# Define"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tAhmWnGMZh7q",
        "colab_type": "text"
      },
      "source": [
        "## Random State Fixer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H6ETnToMbEuG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Set a constant seed for every random number generator.\n",
        "SEED = 42\n",
        "\n",
        "def reset_all_nondeterministic_states(seed=SEED):\n",
        "  random.seed(SEED)\n",
        "  np.random.seed(SEED)\n",
        "  torch.manual_seed(SEED)\n",
        "  if torch.cuda.is_available(): torch.cuda.manual_seed_all(SEED)\n",
        "  torch.backends.cudnn.deterministic = True  # About 15% slower but...\n",
        "  torch.backends.cudnn.benchmark = False"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dJwkXFIyJlkB",
        "colab_type": "text"
      },
      "source": [
        "## LM-specific Helpers"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hjm3439wUJUE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def build_lm_databunch(data_dir_p, n_workers, bs, bptt):\n",
        "  reset_all_nondeterministic_states()\n",
        "  return (TextList.from_folder(data_dir_p)\n",
        "          #Inputs: all the text files in path\n",
        "          .filter_by_folder(include=['train', 'test', 'unsup'])\n",
        "          #We may have other temp folders that contain text files so we only keep what's in train and test\n",
        "          .split_by_rand_pct(\n",
        "              0.1,\n",
        "              seed=SEED  # Set the seed again since in theory one can call np.random before this.\n",
        "          )\n",
        "          #We randomly split and keep 10% (10,000 reviews) for validation\n",
        "          .label_for_lm()\n",
        "          #We want to do a language model so we label accordingly\n",
        "          .databunch(bs=bs, bptt=bptt, num_workers=n_workers))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NJnkgQiJqCyZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def init_lm_learner_with_ulmfit(dbnch, drop_mult, base_path=BASE_DIR_P):\n",
        "  reset_all_nondeterministic_states()\n",
        "  lm_learn = language_model_learner(dbnch, AWD_LSTM, drop_mult=drop_mult, path=base_path)\n",
        "  lm_learn = lm_learn.to_fp16(clip=0.1)  # 2x faster\n",
        "  return lm_learn"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MMflNyYdm-S7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def init_lm_cycles(learner, lr, moms, wd, clbks=[], n_cycles=1):\n",
        "  print(f'init lm lr: {lr}')\n",
        "  reset_all_nondeterministic_states()\n",
        "  learner.fit_one_cycle(n_cycles, lr, moms=moms, wd=wd, callbacks=clbks)\n",
        "  return learner"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BngsYt2inEak",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def tune_lm_cycles(learner, lr, moms, wd, clbks=[], n_cycles=10):\n",
        "  print(f'tune lm lr: {lr}')\n",
        "  reset_all_nondeterministic_states()\n",
        "  learner.unfreeze()\n",
        "  learner.fit_one_cycle(n_cycles, lr, moms=moms, wd=wd, callbacks=clbks)\n",
        "  return learner"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "k7ygpuHbJo0t",
        "colab_type": "text"
      },
      "source": [
        "## CF-specific Helpers"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sPkphOtiXTHH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def build_cf_databunch(data_dir_p, n_workers, bs, vocab):\n",
        "  reset_all_nondeterministic_states()\n",
        "  return (TextList.from_folder(data_dir_p, vocab=vocab)\n",
        "          #grab all the text files in path\n",
        "          .split_by_folder(valid='test')\n",
        "          #split by train and valid folder (that only keeps 'train' and 'test' so no need to filter)\n",
        "          .label_from_folder(classes=['neg', 'pos'])\n",
        "          #label them all with their folders\n",
        "          .databunch(bs=bs, num_workers=n_workers))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1_754QK6Ju61",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def init_cf_learner_with_encoder(dbnch, drop_mult, enc_name, base_path=BASE_DIR_P):\n",
        "  reset_all_nondeterministic_states()\n",
        "  cf_learn = text_classifier_learner(dbnch, AWD_LSTM, drop_mult=drop_mult, path=base_path, pretrained=False)\n",
        "  cf_learn.load_encoder(enc_name)\n",
        "  return cf_learn"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "chGUzivkKYy-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def init_cf_cycles(learner, lr, moms, wd, clbks, n_cycles=1):\n",
        "  print(f'init cf lr: {lr}')\n",
        "  reset_all_nondeterministic_states()\n",
        "  learner.fit_one_cycle(n_cycles, lr, moms=moms, wd=wd, callbacks=clbks)\n",
        "  return learner"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gx3iwNXOLJQG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def tune_cf_cycles(\n",
        "    learner,\n",
        "    lr,\n",
        "    moms,\n",
        "    wd,\n",
        "    clbks_tuple,\n",
        "    n_cycles_tuple=(1,1,2),\n",
        "    freeze_steps=(-2,-3,None),\n",
        "    lr_decays=(2,2,5)\n",
        "):\n",
        "  reset_all_nondeterministic_states()\n",
        "  for n_cycles, freeze_step, lr_decay, clbks in zip(\n",
        "      n_cycles_tuple, freeze_steps, lr_decays, clbks_tuple):\n",
        "    if freeze_step is not None:\n",
        "      learner.freeze_to(freeze_step)\n",
        "    else:\n",
        "      learner.unfreeze()\n",
        "    lr /= lr_decay\n",
        "    print(f'tune cf lr: {lr}')\n",
        "    learner.fit_one_cycle(n_cycles, slice(lr/(2.6**4),lr), moms=moms, wd=wd, callbacks=clbks)\n",
        "  return learner"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4puv1GrLOp5R",
        "colab_type": "text"
      },
      "source": [
        "# Fit"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g3uudRrTXGq3",
        "colab_type": "text"
      },
      "source": [
        "## Forward LM"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wopKpHoLPRj5",
        "colab_type": "text"
      },
      "source": [
        "### Process Data Once"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PIOwmjovkNjR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "IMDB_DATA_IN_COLAB_DIR_P = COLAB_DATA_DIR_P / 'imdb'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aUayc5jQ-ugt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Untar into colab disk so no latency to GDrive.\n",
        "downloaded_imdb_data_dir_p = untar_data(URLs.IMDB, dest=FASTAI_DATA_DIR_P)\n",
        "assert IMDB_DATA_IN_COLAB_DIR_P.resolve() == downloaded_imdb_data_dir_p"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a0NkAvKG-ug3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "fw_lm_dbnch = build_lm_databunch(IMDB_DATA_IN_COLAB_DIR_P, n_dbnch_wrkrs, lm_bs, bptt)\n",
        "# fw_lm_dbnch.show_batch()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zt0DL_dOjokO",
        "colab_type": "text"
      },
      "source": [
        "### Use Persistent Path"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cm_heHy78zKl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Save the databunch to a non-voatile path (e.g.: GDrive).\n",
        "fw_lm_dbnch.save(DATA_DIR_P / FW_LM_DBNCH_FILE_S)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AbszfuaWPxbu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "reset_all_nondeterministic_states()\n",
        "fw_lm_dbnch = load_data(DATA_DIR_P, FW_LM_DBNCH_FILE_S, bs=lm_bs, bptt=bptt, num_workers=n_dbnch_wrkrs)\n",
        "# fw_lm_dbnch.path.ls()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FX1Rpmq5-iL4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# The batch should look the same if the above efforts keep the reproducibility.\n",
        "# fw_lm_dbnch.show_batch()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M_IIOjBt8a0O",
        "colab_type": "text"
      },
      "source": [
        "### Find Learning Rate"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2dflRYKkYtv2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "assert fw_lm_dbnch.train_dl.batch_size == lm_bs\n",
        "lm_epoch_sz = math.ceil(len(fw_lm_dbnch.train_ds) / lm_bs)\n",
        "lm_epoch_sz"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ilR1t6qwMJ7L",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "lr_find_scope = IPyExperimentsPytorch(cl_enable=False)\n",
        "fw_lm_learn = init_lm_learner_with_ulmfit(fw_lm_dbnch, lm_drop_mult)\n",
        "fw_lm_learn.lr_find(end_lr=1, num_it=math.ceil(lm_epoch_sz/9), wd=lm_wd)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UqTHVxFymu4G",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%capture lr_find_log\n",
        "fw_lm_learn.recorder.plot(suggestion=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wRKl8uPjblRN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "(found_lr_name,\n",
        " found_lr_val_str), _ = [line.split(': ')\n",
        "                         for line in lr_find_log.stdout.split('\\n') if line]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PowQQIevQIX7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "display(lr_find_log.outputs[0])\n",
        "print(found_lr_name, found_lr_val_str)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wvzb13qas2rT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "lr_find_scope.keep_var_names('found_lr_val_str')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "InSvZOYOQckx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "del lr_find_scope; gc.collect()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BKqmxK2NHvbY",
        "colab_type": "text"
      },
      "source": [
        "### Init-fit"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CJHXFu6XqUlT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# lm_lr = 0.0251\n",
        "lm_lr = float(found_lr_val_str)\n",
        "fw_lm_learn = init_lm_learner_with_ulmfit(fw_lm_dbnch, lm_drop_mult)\n",
        "init_fw_lm_log_p = LOGS_DIR_P / f'{SESSN_START_T}_history-init_fw_lm-b{lm_bs}-lr{lm_lr}'  # w/o .csv\n",
        "init_fw_lm_clbks = [CSVLogger(fw_lm_learn, init_fw_lm_log_p, append=True)]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5DelajIzQCx0",
        "colab_type": "code",
        "outputId": "48a6b6ba-ce99-42d1-af95-38845875200d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84
        }
      },
      "source": [
        "fw_lm_learn = init_lm_cycles(fw_lm_learn, lm_lr, moms, lm_wd, init_fw_lm_clbks)\n",
        "# fw_lm_learn.csv_logger.read_logged_file()\n",
        "fw_lm_learn.save(f'init_fw_lm-b{lm_bs}-lr{lm_lr}')\n",
        "# (fw_lm_learn.path/fw_lm_learn.model_dir).ls()"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "init lm lr: 0.0251\n",
            "epoch     train_loss  valid_loss  accuracy  time    \n",
            "0         4.344767    4.036683    0.291658  18:57     \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NhGjAxk1IHCQ",
        "colab_type": "text"
      },
      "source": [
        "### Fine-tune"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b0EXNg0NQLpz",
        "colab_type": "code",
        "outputId": "2ca1fb4f-3413-460f-ea03-ac33a696405c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 282
        }
      },
      "source": [
        "# reset_all_nondeterministic_states()\n",
        "# fw_lm_learn = init_lm_learner_with_ulmfit(fw_lm_dbnch, lm_drop_mult)\n",
        "# fw_lm_learn = fw_lm_learn.load(f'init_fw_lm-b{lm_bs}-lr{lm_lr}')\n",
        "tune_lm_lr = round(lm_lr/10, 5)\n",
        "tune_fw_lm_log_p = LOGS_DIR_P / f'{SESSN_START_T}_history-tune_fw_lm-b{lm_bs}-lr{tune_lm_lr}'\n",
        "tune_fw_lm_clbks = [CSVLogger(fw_lm_learn, tune_fw_lm_log_p, append=True)]\n",
        "fw_lm_learn = tune_lm_cycles(fw_lm_learn, tune_lm_lr, moms, lm_wd, tune_fw_lm_clbks)"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tune lm lr: 0.00251\n",
            "epoch     train_loss  valid_loss  accuracy  time    \n",
            "0         4.054212    3.867507    0.311515  20:14     \n",
            "1         4.004817    3.832057    0.315235  20:21     \n",
            "2         3.987055    3.814739    0.317900  20:20     \n",
            "3         3.951325    3.784540    0.321025  20:22     \n",
            "4         3.930473    3.760004    0.323519  20:21     \n",
            "5         3.869126    3.720796    0.327975  20:22     \n",
            "6         3.844987    3.685802    0.332085  20:19     \n",
            "7         3.768153    3.649244    0.336223  20:20     \n",
            "8         3.716011    3.626792    0.338881  20:20     \n",
            "9         3.689258    3.621286    0.339571  20:20     \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SvF0RhTu-uhH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "fw_lm_learn.save(f'tuned_fw_lm-b{lm_bs}-lr{lm_lr}')\n",
        "FW_ENC_NAME = f'fw_enc-b{lm_bs}-lr{lm_lr}'\n",
        "fw_lm_learn.save_encoder(FW_ENC_NAME)\n",
        "# (fw_lm_learn.path/fw_learn_lm.model_dir).ls()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V8L_H4uC91mn",
        "colab_type": "text"
      },
      "source": [
        "## Forward CF"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KN8hrD2U-uhS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# reset_all_nondeterministic_states()\n",
        "# fw_lm_dbnch = load_data(DATA_DIR_P, FW_LM_DBNCH_FILE_S, bs=lm_bs, bptt=bptt, num_workers=n_dbnch_wrkrs)\n",
        "\n",
        "fw_cf_dbnch = build_cf_databunch(IMDB_DATA_IN_COLAB_DIR_P, n_dbnch_wrkrs, cf_bs, fw_lm_dbnch.vocab)\n",
        "fw_cf_dbnch.save(DATA_DIR_P / FW_CF_DBNCH_FILE_S)\n",
        "# fw_cf_dbnch.show_batch()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e8k1uFIeY2ah",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "assert fw_cf_dbnch.train_dl.batch_size == cf_bs\n",
        "cf_epoch_sz = math.ceil(len(fw_cf_dbnch.train_ds) / fw_cf_dbnch.train_dl.batch_size)\n",
        "cf_epoch_sz"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "apd3mZaJZC4p",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "lr_find_scope = IPyExperimentsPytorch(cl_enable=False)\n",
        "fw_cf_learn = init_cf_learner_with_encoder(fw_cf_dbnch, cf_drop_mult, FW_ENC_NAME)\n",
        "fw_cf_learn.lr_find(end_lr=10, num_it=math.ceil(cf_epoch_sz/8), wd=cf_wd)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gtZnIS_TZOj3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%capture lr_find_log\n",
        "fw_cf_learn.recorder.plot(suggestion=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_DrDJC6QZOTC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "list(map(partial(str.split, sep=': '), filter(None, lr_find_log.stdout.split('\\n'))))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aUpWGtGWZClJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "display(lr_find_log.outputs[0])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KFFVaPXVZZdE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "del lr_find_scope; gc.collect()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "87rleD4t-uhV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# reset_all_nondeterministic_states()\n",
        "# fw_cf_dbnch = load_data(DATA_DIR_P, FW_CF_DBNCH_FILE_S, bs=cf_bs, num_workers=n_dbnch_wrkrs)\n",
        "\n",
        "fw_cf_learn = init_cf_learner_with_encoder(fw_cf_dbnch, cf_drop_mult, FW_ENC_NAME)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "954RvBOqqzz7",
        "colab_type": "code",
        "outputId": "1805eb27-8452-4b61-8a96-02713afaaddc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84
        }
      },
      "source": [
        "cf_lr = 5.18e-2\n",
        "init_fw_cf_log_p = LOGS_DIR_P / f'{SESSN_START_T}_history-init_fw_cf-b{cf_bs}-lr{cf_lr}'\n",
        "init_fw_cf_clbks = [CSVLogger(fw_cf_learn, init_fw_cf_log_p, append=True)]\n",
        "fw_cf_learn = init_cf_cycles(fw_cf_learn, cf_lr, moms, cf_wd, init_fw_cf_clbks)"
      ],
      "execution_count": 66,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "init cf lr: 0.0518\n",
            "epoch     train_loss  valid_loss  accuracy  time    \n",
            "0         0.222152    0.176532    0.934160  04:57     \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZoxxCHVhgJ0S",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 238
        },
        "outputId": "99c89b51-bb49-4442-e57c-1bc2ffc52879"
      },
      "source": [
        "tune_fw_cf_clbks_tuple = (\n",
        "    [CSVLogger(\n",
        "        fw_cf_learn,\n",
        "        LOGS_DIR_P / f'{SESSN_START_T}_history-tune_fw_cf-b{cf_bs}-p{period}',\n",
        "        append=True)]\n",
        "    for period in range(1,4)\n",
        ")\n",
        "fw_cf_learn = tune_cf_cycles(fw_cf_learn, cf_lr, moms, cf_wd, tune_fw_cf_clbks_tuple)\n",
        "fw_cf_learn.save(FW_CF_NAME)\n",
        "# (fw_cf_learn.path/fw_cf_learn.model_dir).ls()"
      ],
      "execution_count": 67,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tune cf lr: 0.0259\n",
            "epoch     train_loss  valid_loss  accuracy  time    \n",
            "0         0.190170    0.153062    0.944160  05:36     \n",
            "tune cf lr: 0.01295\n",
            "epoch     train_loss  valid_loss  accuracy  time    \n",
            "0         0.180019    0.138620    0.949240  08:08     \n",
            "tune cf lr: 0.00259\n",
            "epoch     train_loss  valid_loss  accuracy  time    \n",
            "0         0.130078    0.143215    0.945520  09:48     \n",
            "1         0.083496    0.142682    0.949720  09:03     \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zIoqUI5sB_1q",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "fw_cf_learn.export(MDLS_DIR_P / f'export-fw_cf-b{cf_bs}-lr{cf_lr}')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FhV8Q5Y6pWp3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "fw_cf_learn.destroy(); del fw_cf_learn; gc.collect()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wHXVFNpK-ioI",
        "colab_type": "text"
      },
      "source": [
        "## Backward LM"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QhP0h1Xd-mbr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "reset_all_nondeterministic_states()\n",
        "bw_lm_dbnch = load_data(DATA_DIR_P, FW_LM_DBNCH_FILE_S, bs=lm_bs, bptt=bptt, num_workers=n_dbnch_wrkrs, backwards=True)\n",
        "# bw_lm_dbnch.show_batch()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "raVmaLmN_V1H",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "bw_lm_learn = init_lm_learner_with_ulmfit(bw_lm_dbnch, lm_drop_mult)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jjc0wEQfeUv1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "init_bw_lm_log_p = LOGS_DIR_P / f'{SESSN_START_T}_history-init_bw_lm-b{lm_bs}'\n",
        "init_bw_lm_clbks = [CSVLogger(bw_lm_learn, init_bw_lm_log_p, append=True)]\n",
        "bw_lm_learn = init_lm_cycles(bw_lm_learn, lm_lr, moms, lm_wd, init_bw_lm_clbks)\n",
        "bw_lm_learn.save(f'init_bw_lm-b{lm_bs}')\n",
        "# (bw_lm_learn.path/bw_lm_learn.model_dir).ls()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AseLa6Z6eWZv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# reset_all_nondeterministic_states()\n",
        "# bw_lm_learn = bw_lm_learn.load(f'init_bw_lm-b{lm_bs}')\n",
        "\n",
        "tune_bw_lm_log_p = LOGS_DIR_P / f'{SESSN_START_T}_history-tune_bw_lm-b{lm_bs}'\n",
        "tune_bw_lm_clbks = [CSVLogger(bw_lm_learn, tune_bw_lm_log_p, append=True)]\n",
        "bw_lm_learn = tune_lm_cycles(bw_lm_learn, lm_lr/10, moms, lm_wd, tune_bw_lm_clbks)\n",
        "bw_lm_learn.save(f'tuned_bw_lm-b{lm_bs}')\n",
        "bw_lm_learn.save_encoder(BW_ENC_NAME)\n",
        "# (bw_lm_learn.path/bw_lm_learn.model_dir).ls()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4-hoCyr3-qBS",
        "colab_type": "text"
      },
      "source": [
        "## Backward CF"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zrP_-kGk-sUx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "reset_all_nondeterministic_states()\n",
        "bw_cf_dbnch = load_data(DATA_DIR_P, FW_CF_DBNCH_FILE_S, bs=cf_bs, num_workers=n_dbnch_wrkrs, backwards=True)\n",
        "# bw_cf_dbnch.show_batch()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4uskEorwDQjd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "bw_cf_learn = init_cf_learner_with_encoder(bw_cf_dbnch, cf_drop_mult, BW_ENC_NAME)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XE2n0Ht4eYLH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "init_bw_cf_log_p = LOGS_DIR_P / f'{SESSN_START_T}_history-init_bw_cf-b{cf_bs}'\n",
        "init_bw_cf_clbks = [CSVLogger(bw_cf_learn, init_bw_cf_log_p, append=True)]\n",
        "bw_cf_learn = init_cf_cycles(bw_cf_learn, cf_lr, moms, cf_wd, init_bw_cf_clbks)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WGxByUJHeZKp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tune_bw_cf_clbks_tuple = (\n",
        "    [CSVLogger(\n",
        "        bw_cf_learn,\n",
        "        LOGS_DIR_P / f'{SESSN_START_T}_history-tune_bw_cf-b{cf_bs}-p{period}',\n",
        "        append=True)]\n",
        "    for period in range(1,4)\n",
        ")\n",
        "bw_cf_learn = tune_cf_cycles(bw_cf_learn, cf_lr, moms, cf_wd, tune_bw_cf_clbks_tuple)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M6EkkUVvPaeG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "bw_cf_learn.save(BW_CF_NAME)\n",
        "# (bw_cf_learn.path/bw_cf_learn.model_dir).ls()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-5jXi1C4y5BT",
        "colab_type": "text"
      },
      "source": [
        "# Ensemble"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DG7qKU4Xy4kI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pred_fw, lbl_fw = fw_cf_learn.get_preds(ordered=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5eqG0vYj1tCh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pred_bw, lbl_bw = bw_cf_learn.get_preds(ordered=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GeQ8zEtC1vIz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "avg_pred = (pred_fw + pred_bw) / 2"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EbRCEDvglmbF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "accuracy(avg_pred, lbl_fw)"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}