FirstModel.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# WHEN CREATING DATA, NEED TO SHIFT INPUT TOKEN INDEXES BY ONE!!!\n",
    "# 1. Try with real data\n",
    "# 2. Figure out if this is working the way we expect it to work or not\n",
    "# 3. Check out all TODO :\n",
    "# WEIGHT SHARING!!!!!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Not implemented in Autograd\n",
    "# x .+= something -> does not work with Autograd\n",
    "# x = x .+ something -> works\n",
    "\n",
    "# Not implemented in Knet\n",
    "# getindex for 3d KnetArray\n",
    "# Also there is a bug with std function. Inside it uses _var, which uses iterate, which fails for KnetArrays"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The things I've solved\n",
    "# 1. bmm with 4 dims\n",
    "# 2. segment_ids paddings are 0, we can't index with them\n",
    "# 3. ignoring indexes with labels -1 while calculating loss\n",
    "# 4. 2d matrix, 3d tensor multiplication"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "iterate (generic function with 207 methods)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import Base: length, iterate\n",
    "using Random\n",
    "\n",
    "mutable struct PreTrainingData\n",
    "    input_ids\n",
    "    input_mask\n",
    "    segment_ids\n",
    "    mlm_labels\n",
    "    nsp_labels\n",
    "    batchsize\n",
    "    ninstances\n",
    "    shuffled\n",
    "end\n",
    "\n",
    "\n",
    "function PreTrainingData(input_file; batchsize=16, shuffled=true, seq_len=256)\n",
    "    input_ids = []\n",
    "    input_mask = []\n",
    "    segment_ids = []\n",
    "    mlm_labels = []\n",
    "    nsp_labels = []\n",
    "    f = open(input_file)\n",
    "    tmp = split.(readlines(f), \"\\t\")\n",
    "    for i in 1:length(tmp)\n",
    "        instance = eval.(Meta.parse.(tmp[i]))\n",
    "#         push!(input_ids, instance[1] .+ 1)\n",
    "#         push!(input_mask, instance[2])\n",
    "#         push!(segment_ids, instance[3] .+ 1)\n",
    "#         push!(mlm_labels, map(x -> x != -1 ? x + 1 : x, instance[4]))\n",
    "#         push!(nsp_labels, instance[5] + 1)\n",
    "        push!(input_ids, (instance[1] .+ 1)[1:seq_len])\n",
    "        push!(input_mask, instance[2][1:seq_len])\n",
    "        push!(segment_ids, (instance[3] .+ 1)[1:seq_len])\n",
    "        push!(mlm_labels, map(x -> x != -1 ? x + 1 : x, instance[4])[1:seq_len])\n",
    "        push!(nsp_labels, (instance[5] + 1))\n",
    "    end\n",
    "    ninstances = length(input_ids)\n",
    "    return PreTrainingData(input_ids, input_mask, segment_ids, mlm_labels, nsp_labels, batchsize, ninstances, shuffled)\n",
    "end\n",
    "\n",
    "\n",
    "function length(d::PreTrainingData)\n",
    "    d, r = divrem(d.ninstances, d.batchsize)\n",
    "    return r == 0 ? d : d+1\n",
    "end\n",
    "\n",
    "function iterate(d::PreTrainingData, state=ifelse(d.shuffled, randperm(d.ninstances), 1:d.ninstances))\n",
    "\n",
    "    state === nothing && return nothing\n",
    "\n",
    "    if length(state) > d.batchsize\n",
    "        new_state = state[d.batchsize+1:end]\n",
    "        input_ids = hcat(d.input_ids[state[1:d.batchsize]]...)\n",
    "        input_mask = hcat(d.input_mask[state[1:d.batchsize]]...)\n",
    "        segment_ids = hcat(d.segment_ids[state[1:d.batchsize]]...)\n",
    "        mlm_labels = hcat(d.mlm_labels[state[1:d.batchsize]]...)\n",
    "        nsp_labels = hcat(d.nsp_labels[state[1:d.batchsize]]...)\n",
    "    else\n",
    "        new_state = nothing\n",
    "        input_ids = hcat(d.input_ids[state]...)\n",
    "        input_mask = hcat(d.input_mask[state]...)\n",
    "        segment_ids = hcat(d.segment_ids[state]...)\n",
    "        mlm_labels = hcat(d.mlm_labels[state]...)\n",
    "        nsp_labels = hcat(d.nsp_labels[state]...)\n",
    "    end\n",
    "    \n",
    "    return ((input_ids, input_mask, segment_ids, mlm_labels, nsp_labels), new_state)\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "load_from_torch_classification (generic function with 1 method)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "include(\"model.jl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Config(768, 30522, 3072, 512, 256, 2, 12, 12, 2, KnetArray{Float32,N} where N, 0.1, 0.1, gelu)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mutable struct Config\n",
    "    embed_size::Int\n",
    "    vocab_size::Int\n",
    "    ff_hidden_size::Int\n",
    "    max_seq_len::Int\n",
    "    seq_len::Int\n",
    "    num_segment::Int\n",
    "    num_heads::Int\n",
    "    num_encoder::Int\n",
    "    batchsize::Int\n",
    "    atype\n",
    "    pdrop\n",
    "    attention_pdrop\n",
    "    func\n",
    "end\n",
    "\n",
    "config = Config(768, 30522, 3072, 512, 256, 2, 12, 12, 2, KnetArray{Float32}, 0.1, 0.1, gelu)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PreTrainingData(Any[[102, 1001, 7571, 1013, 104, 1013, 10232, 1025, 10769, 2100  …  5556, 1014, 3571, 1014, 6164, 17789, 17466, 104, 18140, 103], [102, 1001, 2097, 2117, 4842, 104, 2043, 24666, 17687, 2006  …  2617, 1011, 2044, 104, 3311, 2001, 6987, 2092, 6257, 103], [102, 1997, 104, 25043, 8820, 2863, 1013, 2152, 2232, 3037  …  2009, 1013, 2046, 2098, 2023, 1038, 3790, 2007, 2024, 103], [102, 1525, 12839, 2212, 1013, 1524, 2044, 2058, 2032, 4481  …  5470, 3442, 2273, 2040, 2036, 1997, 2052, 2044, 2018, 103], [102, 4120, 1997, 24530, 2051, 1011, 3392, 2044, 2112, 5255  …  5049, 1013, 2022, 1997, 3022, 1522, 1056, 2523, 1012, 103], [102, 1001, 2688, 1012, 5642, 1012, 2261, 104, 1025, 4003  …  1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [102, 1056, 1013, 4002, 2680, 104, 2000, 1038, 4358, 2009  …  1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [102, 2917, 2007, 1997, 2796, 1011, 2022, 1046, 2229, 2010  …  1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [102, 104, 1011, 2000, 27943, 15371, 1997, 10644, 2041, 2019  …  1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [102, 2148, 3086, 7422, 3640, 2292, 2516, 2026, 4840, 2652  …  1, 1, 1, 1, 1, 1, 1, 1, 1, 1]  …  [102, 2007, 104, 6079, 2006, 104, 11769, 1013, 2007, 9433  …  1997, 10341, 2021, 3833, 2001, 1521, 4653, 1522, 11327, 103], [102, 1998, 1997, 2089, 1522, 1056, 3589, 6352, 5167, 1013  …  5226, 1038, 5014, 4317, 2007, 1038, 2271, 2127, 1011, 103], [102, 11521, 5158, 2014, 2048, 3122, 1999, 3942, 2065, 2023  …  3392, 2044, 2112, 5255, 2038, 4344, 7451, 6544, 2917, 103], [102, 20982, 3225, 104, 1011, 6283, 3868, 1998, 2711, 1522  …  1009, 8680, 1038, 4947, 7282, 2031, 2152, 2061, 2829, 103], [102, 2598, 1998, 3774, 2050, 3924, 104, 11769, 6690, 2097  …  2023, 5906, 1998, 1521, 15597, 3427, 104, 2066, 2028, 103], [102, 2169, 2559, 1013, 2097, 2123, 12879, 2032, 2026, 17545  …  2028, 2025, 2184, 2001, 2378, 1038, 2503, 104, 2000, 103], [102, 1001, 2007, 2038, 7973, 104, 3066, 2652, 9466, 10273  …  2076, 2056, 1038, 104, 3999, 104, 3478, 7246, 2014, 103], [102, 2101, 4303, 2001, 2132, 2015, 3790, 2007, 2024, 9872  …  4437, 2340, 1997, 2176, 2362, 2004, 2184, 2001, 3614, 103], [102, 2001, 5999, 2001, 9959, 1038, 3485, 104, 2932, 104  …  1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [102, 1001, 1032, 1057, 2362, 1012, 2679, 24472, 2141, 1028  …  2007, 2024, 1999, 104, 104, 1050, 3493, 9658, 2058, 103]], Any[[1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  0, 0, 0, 0, 0, 0, 0, 0, 0, 0]  …  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], Any[[1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  1, 1, 1, 1, 1, 1, 1, 1, 1, 1]  …  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1  …  2, 2, 2, 2, 2, 2, 2, 2, 2, 2]], Any[[-1, -1, -1, -1, 3390, -1, -1, -1, -1, -1  …  -1, -1, -1, -1, -1, -1, -1, 2621, -1, -1], [-1, -1, -1, -1, -1, 2032, -1, -1, -1, -1  …  -1, -1, -1, 2010, -1, -1, -1, -1, -1, -1], [-1, -1, 4425, -1, -1, -1, -1, -1, -1, -1  …  -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1  …  -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1  …  -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, 5841, -1, -1  …  -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, 1013, -1, -1, -1, -1  …  -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1  …  -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, 2724, -1, -1, -1, -1, -1, -1, -1, -1  …  -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1  …  -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]  …  [-1, -1, 2191, -1, -1, 8162, -1, -1, -1, -1  …  -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1  …  -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1  …  -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, 26207, -1, -1, -1, -1, -1, -1  …  -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, 6352, -1, -1, -1  …  -1, -1, -1, -1, 15597, -1, 1522, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1  …  -1, -1, -1, -1, -1, -1, -1, 2113, -1, -1], [-1, -1, -1, 2038, -1, 2558, -1, -1, -1, -1  …  -1, -1, -1, 1003, -1, 2455, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1  …  -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, 1998, -1, 7207  …  -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1  …  -1, -1, -1, 1046, 1522, -1, -1, -1, -1, -1]], Any[1, 2, 2, 1, 2, 2, 1, 1, 2, 2  …  2, 2, 2, 2, 2, 1, 1, 1, 1, 2], 2, 100, true)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_file = \"/scratch/users/omutlu/hyperpartisan/data_20181122_big/pre_test_tiny.tsv\" #pre_test_tiny Each line has 5 columns: input ids, input mask, segment ids, masked lm labels, next sentence label\n",
    "dtrn = PreTrainingData(input_file, batchsize=config.batchsize, seq_len=config.seq_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertPreTraining(Bert(EmbedLayer(Embedding(P(KnetArray{Float32,2}(768,30522))), Embedding(P(KnetArray{Float32,2}(768,512))), Embedding(P(KnetArray{Float32,2}(768,2))), LayerNormalization(P(KnetArray{Float32,1}(768)), P(KnetArray{Float32,1}(768)), 1.0e-12), 256, 0.1), Encoder[Encoder(SelfAttention(Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), 12, 256, 768, 64, 8, 0.1, 0.1), LayerNormalization(P(KnetArray{Float32,1}(768)), P(KnetArray{Float32,1}(768)), 1.0e-12), FeedForward(Dense(Linear3D(P(KnetArray{Float32,2}(3072,768)), P(KnetArray{Float32,1}(3072))), 0.0, gelu), Linear3D(P(KnetArray{Float32,2}(768,3072)), P(KnetArray{Float32,1}(768))), 0.1), LayerNormalization(P(KnetArray{Float32,1}(768)), P(KnetArray{Float32,1}(768)), 1.0e-12)), Encoder(SelfAttention(Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), 12, 256, 768, 64, 8, 0.1, 0.1), LayerNormalization(P(KnetArray{Float32,1}(768)), P(KnetArray{Float32,1}(768)), 1.0e-12), FeedForward(Dense(Linear3D(P(KnetArray{Float32,2}(3072,768)), P(KnetArray{Float32,1}(3072))), 0.0, gelu), Linear3D(P(KnetArray{Float32,2}(768,3072)), P(KnetArray{Float32,1}(768))), 0.1), LayerNormalization(P(KnetArray{Float32,1}(768)), P(KnetArray{Float32,1}(768)), 1.0e-12)), Encoder(SelfAttention(Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), 12, 256, 768, 64, 8, 0.1, 0.1), LayerNormalization(P(KnetArray{Float32,1}(768)), P(KnetArray{Float32,1}(768)), 1.0e-12), FeedForward(Dense(Linear3D(P(KnetArray{Float32,2}(3072,768)), P(KnetArray{Float32,1}(3072))), 0.0, gelu), Linear3D(P(KnetArray{Float32,2}(768,3072)), P(KnetArray{Float32,1}(768))), 0.1), LayerNormalization(P(KnetArray{Float32,1}(768)), P(KnetArray{Float32,1}(768)), 1.0e-12)), Encoder(SelfAttention(Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), 12, 256, 768, 64, 8, 0.1, 0.1), LayerNormalization(P(KnetArray{Float32,1}(768)), P(KnetArray{Float32,1}(768)), 1.0e-12), FeedForward(Dense(Linear3D(P(KnetArray{Float32,2}(3072,768)), P(KnetArray{Float32,1}(3072))), 0.0, gelu), Linear3D(P(KnetArray{Float32,2}(768,3072)), P(KnetArray{Float32,1}(768))), 0.1), LayerNormalization(P(KnetArray{Float32,1}(768)), P(KnetArray{Float32,1}(768)), 1.0e-12)), Encoder(SelfAttention(Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), 12, 256, 768, 64, 8, 0.1, 0.1), LayerNormalization(P(KnetArray{Float32,1}(768)), P(KnetArray{Float32,1}(768)), 1.0e-12), FeedForward(Dense(Linear3D(P(KnetArray{Float32,2}(3072,768)), P(KnetArray{Float32,1}(3072))), 0.0, gelu), Linear3D(P(KnetArray{Float32,2}(768,3072)), P(KnetArray{Float32,1}(768))), 0.1), LayerNormalization(P(KnetArray{Float32,1}(768)), P(KnetArray{Float32,1}(768)), 1.0e-12)), Encoder(SelfAttention(Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), 12, 256, 768, 64, 8, 0.1, 0.1), LayerNormalization(P(KnetArray{Float32,1}(768)), P(KnetArray{Float32,1}(768)), 1.0e-12), FeedForward(Dense(Linear3D(P(KnetArray{Float32,2}(3072,768)), P(KnetArray{Float32,1}(3072))), 0.0, gelu), Linear3D(P(KnetArray{Float32,2}(768,3072)), P(KnetArray{Float32,1}(768))), 0.1), LayerNormalization(P(KnetArray{Float32,1}(768)), P(KnetArray{Float32,1}(768)), 1.0e-12)), Encoder(SelfAttention(Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), 12, 256, 768, 64, 8, 0.1, 0.1), LayerNormalization(P(KnetArray{Float32,1}(768)), P(KnetArray{Float32,1}(768)), 1.0e-12), FeedForward(Dense(Linear3D(P(KnetArray{Float32,2}(3072,768)), P(KnetArray{Float32,1}(3072))), 0.0, gelu), Linear3D(P(KnetArray{Float32,2}(768,3072)), P(KnetArray{Float32,1}(768))), 0.1), LayerNormalization(P(KnetArray{Float32,1}(768)), P(KnetArray{Float32,1}(768)), 1.0e-12)), Encoder(SelfAttention(Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), 12, 256, 768, 64, 8, 0.1, 0.1), LayerNormalization(P(KnetArray{Float32,1}(768)), P(KnetArray{Float32,1}(768)), 1.0e-12), FeedForward(Dense(Linear3D(P(KnetArray{Float32,2}(3072,768)), P(KnetArray{Float32,1}(3072))), 0.0, gelu), Linear3D(P(KnetArray{Float32,2}(768,3072)), P(KnetArray{Float32,1}(768))), 0.1), LayerNormalization(P(KnetArray{Float32,1}(768)), P(KnetArray{Float32,1}(768)), 1.0e-12)), Encoder(SelfAttention(Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), 12, 256, 768, 64, 8, 0.1, 0.1), LayerNormalization(P(KnetArray{Float32,1}(768)), P(KnetArray{Float32,1}(768)), 1.0e-12), FeedForward(Dense(Linear3D(P(KnetArray{Float32,2}(3072,768)), P(KnetArray{Float32,1}(3072))), 0.0, gelu), Linear3D(P(KnetArray{Float32,2}(768,3072)), P(KnetArray{Float32,1}(768))), 0.1), LayerNormalization(P(KnetArray{Float32,1}(768)), P(KnetArray{Float32,1}(768)), 1.0e-12)), Encoder(SelfAttention(Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), 12, 256, 768, 64, 8, 0.1, 0.1), LayerNormalization(P(KnetArray{Float32,1}(768)), P(KnetArray{Float32,1}(768)), 1.0e-12), FeedForward(Dense(Linear3D(P(KnetArray{Float32,2}(3072,768)), P(KnetArray{Float32,1}(3072))), 0.0, gelu), Linear3D(P(KnetArray{Float32,2}(768,3072)), P(KnetArray{Float32,1}(768))), 0.1), LayerNormalization(P(KnetArray{Float32,1}(768)), P(KnetArray{Float32,1}(768)), 1.0e-12)), Encoder(SelfAttention(Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), 12, 256, 768, 64, 8, 0.1, 0.1), LayerNormalization(P(KnetArray{Float32,1}(768)), P(KnetArray{Float32,1}(768)), 1.0e-12), FeedForward(Dense(Linear3D(P(KnetArray{Float32,2}(3072,768)), P(KnetArray{Float32,1}(3072))), 0.0, gelu), Linear3D(P(KnetArray{Float32,2}(768,3072)), P(KnetArray{Float32,1}(768))), 0.1), LayerNormalization(P(KnetArray{Float32,1}(768)), P(KnetArray{Float32,1}(768)), 1.0e-12)), Encoder(SelfAttention(Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), 12, 256, 768, 64, 8, 0.1, 0.1), LayerNormalization(P(KnetArray{Float32,1}(768)), P(KnetArray{Float32,1}(768)), 1.0e-12), FeedForward(Dense(Linear3D(P(KnetArray{Float32,2}(3072,768)), P(KnetArray{Float32,1}(3072))), 0.0, gelu), Linear3D(P(KnetArray{Float32,2}(768,3072)), P(KnetArray{Float32,1}(768))), 0.1), LayerNormalization(P(KnetArray{Float32,1}(768)), P(KnetArray{Float32,1}(768)), 1.0e-12))], KnetArray{Float32,N} where N), Pooler(Linear(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768)))), NSPHead(Linear(P(KnetArray{Float32,2}(2,768)), P(KnetArray{Float32,1}(2)))), MLMHead(Dense(Linear3D(P(KnetArray{Float32,2}(768,768)), P(KnetArray{Float32,1}(768))), 0.0, gelu), LayerNormalization(P(KnetArray{Float32,1}(768)), P(KnetArray{Float32,1}(768)), 1.0e-12), Linear3D(P(KnetArray{Float32,2}(30522,768)), P(KnetArray{Float32,1}(30522)))))"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = BertPreTraining(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11.008953f0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#model = load_from_torch(model, config.num_encoder, config.atype, torch_model)\n",
    "model(dtrn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "T(11.1834545)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(x, attention_mask, segment_ids, mlm_labels, nsp_labels) = first(dtrn)\n",
    "lss = @diff model(x, segment_ids, mlm_labels, nsp_labels, attention_mask=attention_mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "768×30522 KnetArray{Float32,2}:\n",
       "  0.00461547    0.00151799    0.000901209  …   0.0012588     0.000919951\n",
       " -0.00586825    0.00566539    0.00193223      -0.00394992   -0.00716168 \n",
       "  0.00650557    0.00487576    0.00760175      -0.000711747  -0.0021168  \n",
       " -0.00627392    0.00571866    0.0047814       -0.00681153    0.00375465 \n",
       "  0.00213985   -0.00441026   -0.00562068      -0.00748848    0.00399292 \n",
       "  0.000395741   0.00470316   -0.002642     …   0.00740453   -0.00722955 \n",
       " -0.00434567   -0.00466482   -0.00266708       0.00478306    0.00655464 \n",
       " -0.00444133    0.00517291   -0.00109169      -0.00737192   -0.00713584 \n",
       "  0.00524661    9.41981e-5    0.00139285       0.00638318    0.00528462 \n",
       "  0.00118155   -0.00225733   -0.00128265      -0.00705595    0.00703708 \n",
       " -0.00341957   -0.00397716   -0.000586801  …  -0.00131311    0.00582492 \n",
       "  0.00422543    0.00169392    0.00486032       0.00200846   -0.00626015 \n",
       "  0.00269984    0.00254733    0.00113057       0.00577424    0.000879644\n",
       "  ⋮                                        ⋱   ⋮                        \n",
       "  0.0062623    -0.000881262  -0.00210801      -0.005191      0.00186184 \n",
       " -0.000320184   0.00225435   -0.000768536      0.00545059   -0.00598594 \n",
       "  0.00334694    0.00607022   -0.00171581      -0.00709285   -0.00318438 \n",
       " -0.00274955    0.00735439   -0.0013757        0.00382156    0.00162034 \n",
       "  0.00457037   -0.00134745    0.00305793   …  -0.000963628  -0.00763661 \n",
       " -0.00574135    0.00471641   -9.30518e-5      -0.00477306    0.00271846 \n",
       "  0.00273393    0.00404361   -0.00395694      -0.00357274    0.00509825 \n",
       "  0.00679028    0.00338155   -0.00207599       0.00325515    6.6819e-5  \n",
       " -0.00402736   -0.00143467   -0.000790801     -0.00760835   -0.00650389 \n",
       "  0.00339637   -0.00512148    0.000320873  …   0.00737339   -0.00490292 \n",
       "  0.0078746     0.00303856   -0.00477798       0.00664281   -0.00240869 \n",
       " -0.0079354    -0.00146264   -0.0017859        0.0072548    -0.0035765  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "asd = value(model.bert.embed_layer.wordpiece.w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30522×768 KnetArray{Float32,2}:\n",
       " -0.00544862    0.00320057    0.000257203  …   0.00435162   -0.00623963 \n",
       " -0.00570309    0.00346152   -0.00437561      -0.00653618    0.00719334 \n",
       "  0.00607219    0.00114515   -0.00459044      -0.00510588    0.000917494\n",
       " -0.00492966    0.000598111  -0.000402751     -0.00134538    0.00232236 \n",
       "  0.00697437   -0.00417384    0.00664084       0.00608568   -0.00536711 \n",
       " -0.000193653   0.00794645   -0.0052714    …   0.00284943   -0.00711361 \n",
       "  0.00724944   -0.00108576    0.0018597       -0.00493013    0.00289999 \n",
       "  0.000102509  -0.00658922    0.00309253      -0.00783502   -0.00174047 \n",
       " -0.00100046    0.000146513   0.00687421       0.00205391    0.000753212\n",
       " -0.00550281    2.90573e-5    0.000389272      0.000507431   0.00590259 \n",
       " -0.000906374   0.0054985    -0.000317218  …  -0.0042433    -0.00263875 \n",
       "  0.00247484    0.000509544   0.00199289       0.000101284  -0.007735   \n",
       "  0.00701112   -0.00150363   -0.00439735      -9.49757e-5    0.00328095 \n",
       "  ⋮                                        ⋱                            \n",
       " -0.00345283    0.00729535    0.00423608   …   0.00628901    0.00338837 \n",
       " -0.00761385    0.00672965    0.00661537       0.00193035    0.00300304 \n",
       " -0.00424437   -0.00490936    0.00710599      -0.000531558  -0.00721822 \n",
       "  0.00415142    0.00787003   -0.00529891       0.00358786   -0.00134085 \n",
       "  0.00618495   -0.00342518    0.00165314      -0.00184947    0.00261964 \n",
       "  0.00527565    0.00669467    0.000979714  …  -0.00147035   -0.00462631 \n",
       "  0.000456068  -0.000657395   0.00696813      -0.000283759  -0.00455562 \n",
       " -0.00444152   -0.00183299    0.00247932      -0.00671845    0.00643057 \n",
       "  0.00711912   -0.00404958    0.00327513       0.00563357    0.00792492 \n",
       " -0.000765622  -0.00443248    0.00616504      -0.00777905   -0.00789324 \n",
       " -0.000375419  -0.0052468     0.0023332    …  -0.00100687    0.0035002  \n",
       " -0.00557896    9.99891e-5   -0.00163686      -0.00310932   -0.00561089 "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dsa = value(model.mlm.linear.w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4.2660485f-8"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Knet.mean(asd .== permutedims(dsa, (2,1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2×768 KnetArray{Float32,2}:\n",
       " 0.0193534   -0.0143639   0.00623309  …  0.0450153  0.045012   -0.05006   \n",
       " 0.00504928   0.00269391  0.0201054      0.0260925  0.0387428  -0.00371928"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qwe = value(model.nsp.linear.w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "function initopt!(model::BertPreTraining, optimizer=\"Adam()\")\n",
    "    for par in params(model)\n",
    "        par.opt = eval(Meta.parse(optimizer))\n",
    "    end\n",
    "end\n",
    "\n",
    "function mytrain!(model::BertPreTraining, x, attention_mask, segment_ids, mlm_labels, nsp_labels)\n",
    "    values = []\n",
    "    J = @diff model(x, segment_ids, mlm_labels, nsp_labels, attention_mask=attention_mask)\n",
    "    for par in params(model)\n",
    "        g = grad(J, par)\n",
    "        update!(value(par), g, par.opt)\n",
    "    end\n",
    "    return value(J)\n",
    "end\n",
    "\n",
    "initopt!(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14.013472f0"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "losses = []\n",
    "for (k, (x, attention_mask, segment_ids, mlm_labels, nsp_labels)) in enumerate(dtrn)\n",
    "    lss = mytrain!(model, x, attention_mask, segment_ids, mlm_labels, nsp_labels)\n",
    "    push!(losses, lss)\n",
    "end\n",
    "lss = Knet.mean(losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2×768 KnetArray{Float32,2}:\n",
       " 0.0173686   -0.0158124   0.0110227  …  0.0451003  0.0345928  -0.0409246\n",
       " 0.00703405   0.00414239  0.0153158     0.0260075  0.049162   -0.0128547"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "value(model.nsp.linear.w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0f0"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Knet.mean(value(model.bert.embed_layer.wordpiece.w) .== asd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0f0"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Knet.mean(value(model.mlm.linear.w) .== dsa)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dict{Any,Any} with 207 entries:\n",
       "  \"bert.encoder.layer.2.at… => PyObject tensor([[ 0.0641,  0.0921,  0.0645,  ..…\n",
       "  \"bert.encoder.layer.6.ou… => PyObject tensor([0.8757, 0.8678, 0.8159, 0.8294,…\n",
       "  \"bert.encoder.layer.3.ou… => PyObject tensor([0.8370, 0.8401, 0.8245, 0.8292,…\n",
       "  \"bert.encoder.layer.5.at… => PyObject tensor([[ 0.0022,  0.0240, -0.0346,  ..…\n",
       "  \"bert.encoder.layer.0.at… => PyObject tensor([[ 0.0209, -0.0050, -0.0087,  ..…\n",
       "  \"bert.encoder.layer.0.at… => PyObject tensor([[-0.0125,  0.0259, -0.0247,  ..…\n",
       "  \"bert.encoder.layer.4.in… => PyObject tensor([-0.0937, -0.1178, -0.1182,  ...…\n",
       "  \"bert.encoder.layer.4.at… => PyObject tensor([-5.8893e-03, -2.3500e-03,  2.81…\n",
       "  \"bert.encoder.layer.3.at… => PyObject tensor([0.9213, 0.8878, 0.8387, 0.8576,…\n",
       "  \"bert.encoder.layer.9.at… => PyObject tensor([0.8576, 0.8295, 0.7983, 0.7731,…\n",
       "  \"bert.encoder.layer.11.a… => PyObject tensor([[-0.0619, -0.0012, -0.0201,  ..…\n",
       "  \"bert.encoder.layer.10.a… => PyObject tensor([ 0.0225,  0.0150,  0.0542,  0.0…\n",
       "  \"bert.encoder.layer.10.o… => PyObject tensor([[ 0.0229, -0.0331,  0.0035,  ..…\n",
       "  \"bert.pooler.dense.weigh… => PyObject tensor([[ 0.0076, -0.0378, -0.0148,  ..…\n",
       "  \"bert.encoder.layer.4.ou… => PyObject tensor([0.8796, 0.8641, 0.8371, 0.8502,…\n",
       "  \"bert.encoder.layer.8.at… => PyObject tensor([ 4.8455e-05,  3.4553e-03, -1.75…\n",
       "  \"cls.predictions.bias\"    => PyObject tensor([-0.9045, -0.9146, -0.9305,  ...…\n",
       "  \"bert.encoder.layer.6.at… => PyObject tensor([[-0.0112,  0.0050, -0.0501,  ..…\n",
       "  \"bert.embeddings.word_em… => PyObject tensor([[ 0.0062, -0.0568, -0.0329,  ..…\n",
       "  \"bert.encoder.layer.10.a… => PyObject tensor([-0.0400, -0.0275, -0.0459, -0.0…\n",
       "  \"bert.encoder.layer.3.at… => PyObject tensor([[ 0.0669, -0.0302, -0.0036,  ..…\n",
       "  \"bert.encoder.layer.10.a… => PyObject tensor([[-0.0085,  0.0331,  0.0011,  ..…\n",
       "  \"bert.encoder.layer.8.at… => PyObject tensor([ 0.0242,  0.0072, -0.0357,  0.0…\n",
       "  \"bert.encoder.layer.2.ou… => PyObject tensor([-0.0655,  0.0687, -0.0304,  0.0…\n",
       "  \"bert.encoder.layer.6.at… => PyObject tensor([[-0.0321,  0.0631, -0.0146,  ..…\n",
       "  ⋮                         => ⋮"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# FIRST WAY\n",
    "# 1:100 row vector in python -> try in python Bert and check the result\n",
    "# 1:100 column vector in julia -> try in Knet and check the result\n",
    "using PyCall\n",
    "@pyimport torch\n",
    "@pyimport numpy\n",
    "torch_model = torch.load(\"/scratch/users/omutlu/hyperpartisan/data_20181122_big/pytorch_model.bin\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PyObject tensor([[ 2.8889e+02, -4.4592e+02,  1.6673e+02, -1.2045e+03,  2.1487e+02,\n",
       "         -3.1990e+02, -2.1644e+02,  6.6723e+01,  4.5352e+02, -4.5058e+01,\n",
       "         -1.8131e+02, -1.0027e+03, -5.6847e+02, -2.4317e+02,  3.0161e+02,\n",
       "          3.1995e+02, -6.8638e+02, -5.4568e+02, -1.1507e+02, -1.0731e+01,\n",
       "          1.0854e+02,  1.5143e+01,  4.5979e+01,  3.6129e+02,  6.8772e+01,\n",
       "          2.9516e+01,  2.6505e+02,  8.3867e+02, -6.1576e+02,  1.5108e+02,\n",
       "         -1.3967e+02,  4.2495e+01, -2.2941e+02,  1.0954e+02, -1.9463e+02,\n",
       "          1.2408e+02, -3.8016e+02,  3.6273e+02,  5.1584e+01,  5.6365e+02,\n",
       "         -6.4561e+02, -1.1862e+02,  1.7012e+02,  2.3527e+02, -2.4004e+02,\n",
       "          3.8593e+01, -2.8113e+02, -5.6259e+02,  6.8088e+01,  1.8635e+02,\n",
       "          5.5222e+02, -4.1687e+02, -3.2281e+01,  4.2259e+01,  2.2871e+02,\n",
       "          4.6529e+02, -4.0624e+02,  3.7479e+01, -2.7199e+02, -4.6029e+01,\n",
       "          3.9952e+02, -1.2753e+02,  5.3966e+02, -7.3936e+02,  1.0183e+03,\n",
       "         -1.7781e+02, -2.5694e+02, -5.1835e+02,  4.5996e+02,  2.8952e+02,\n",
       "          7.6185e+02,  3.2674e+02,  2.7501e+00, -1.7113e+02, -2.8254e+02,\n",
       "          8.4588e+02,  4.4586e+02, -1.0200e+02,  1.1881e+02, -1.1117e+03,\n",
       "         -7.2236e+02, -2.8382e+02,  1.9032e+02,  6.4614e+01,  6.4542e+02,\n",
       "          1.8131e+02, -4.5325e+02,  3.9027e+02, -3.7911e+02,  2.1373e+02,\n",
       "          6.4525e+02,  4.4348e+02,  1.4096e+02, -8.5868e+01,  3.6245e+02,\n",
       "         -9.2591e+02, -9.4705e+02, -3.0721e+01, -2.9471e+01,  3.7334e+02,\n",
       "          4.3277e+02,  5.6642e+02,  1.8949e+02,  5.3530e+02, -4.4705e+02,\n",
       "         -1.8848e+02, -4.0371e+01,  1.0340e+02, -8.9041e+02,  1.0089e+03,\n",
       "          7.2512e+01, -5.7359e+02, -6.4760e+02, -6.1681e+02,  2.4683e+02,\n",
       "         -2.7931e+01,  7.9403e+00, -1.6813e+02,  3.1232e+02, -2.8644e+02,\n",
       "         -2.3220e+02,  2.2288e+02, -2.5193e+01, -1.0868e+02,  8.4636e+02,\n",
       "         -2.1410e+02,  7.9401e+01,  6.1883e+02,  1.0921e+01,  4.2775e+02,\n",
       "          2.1378e+02,  2.4879e+02,  4.2275e+02,  9.1633e+01, -2.7466e+02,\n",
       "         -8.6849e+01, -2.5152e+01,  6.8277e+00,  1.9079e+02,  6.4827e+02,\n",
       "          5.6020e+02,  7.6179e+02,  1.2144e+02,  2.2841e+02,  4.9618e+02,\n",
       "          9.9012e+02, -9.4866e+01, -4.9718e+02, -2.6493e+02,  4.7401e+02,\n",
       "          3.7272e+02,  2.5260e+02,  3.2290e+02, -3.9636e+02,  4.6491e+02,\n",
       "          3.5145e+02,  3.8698e+01, -1.4286e+02, -3.9618e+02, -2.8888e+02,\n",
       "         -1.2007e+02, -3.1942e+01,  7.4168e+02, -3.3496e+02,  5.6450e+02,\n",
       "         -8.8553e+01,  1.2853e+02, -5.3987e+01, -8.9509e+02, -2.0835e+02,\n",
       "         -3.3953e+02, -9.5937e+02,  2.4696e+02,  4.9659e+02,  2.6173e+02,\n",
       "         -3.9484e+02,  4.2417e+02,  5.3226e+02, -2.3800e+01,  8.1823e+01,\n",
       "         -2.3630e+02, -1.2086e+02,  3.3349e+02,  1.1127e+02,  4.4777e+02,\n",
       "         -6.8319e+01,  3.9583e+02,  3.8084e+02, -6.7163e+01,  7.0760e+02,\n",
       "         -7.1263e+02, -5.0673e+02, -8.2307e+02, -2.6889e+02,  8.7130e+01,\n",
       "         -6.1052e+02,  4.0876e+02, -5.8390e+01, -5.5864e+02,  8.0275e+02,\n",
       "          2.7059e+02,  2.9408e+02, -2.1689e+01,  1.9940e+02,  2.8036e+02,\n",
       "         -1.2269e+02,  2.5444e+02, -8.1269e+02, -3.6329e+02, -5.7122e+00,\n",
       "         -2.5161e+02, -2.8266e+02, -5.4793e+02, -3.9149e+02, -5.2888e+02,\n",
       "         -4.4420e+02,  1.9323e+02, -2.6214e+02, -2.5632e+02, -3.1269e+02,\n",
       "          3.0264e+02, -5.7596e+02,  2.3999e+02, -6.7265e+02, -7.0677e+01,\n",
       "         -4.3100e+02, -2.8788e+02,  6.3241e+01, -1.9680e+02,  2.1473e+02,\n",
       "         -1.0010e+01, -2.0910e+02,  1.1789e+02,  5.8399e+02,  3.7427e+02,\n",
       "          6.4242e+02, -6.7696e+01, -3.7666e+02, -6.3895e+01,  3.9403e+02,\n",
       "          6.8968e+02, -3.2589e+01, -5.3759e+02,  7.5769e+02,  4.0577e+02,\n",
       "         -7.8142e+02, -1.4190e+02,  1.2426e+02,  5.9235e+01, -3.5595e+02,\n",
       "          1.9696e+02, -3.8903e+02, -2.9189e+02,  7.9980e+02, -7.8923e+01,\n",
       "          4.4674e+01, -6.5490e+02,  1.2289e+03, -6.4508e+02,  1.2580e+02,\n",
       "          4.3259e+02,  4.4681e+02,  1.2575e+03, -5.0644e+02,  5.5302e+01,\n",
       "          2.1076e+02, -2.8087e+02, -4.6958e+02,  4.8737e+01, -2.3914e+02,\n",
       "         -2.3397e+02,  2.7901e+02,  6.6770e+02, -7.4006e+02, -3.6965e+02,\n",
       "          1.1750e+02, -8.9718e+01,  4.3361e+02, -1.4188e+02, -4.5397e+01,\n",
       "          1.0154e+03,  6.3773e-01, -2.4343e+02,  2.0946e+01, -5.3812e+02,\n",
       "          1.6559e+02, -3.1403e+02,  5.2523e+02,  1.7229e+02,  5.6366e+02,\n",
       "          2.5338e+02,  4.6527e+02,  1.7387e+02, -3.4148e+02,  8.8030e-02,\n",
       "          1.4380e+02,  5.0740e+02,  3.7307e+02, -3.4816e+02, -3.6230e+01,\n",
       "         -1.9807e+02, -3.2038e+01, -6.8450e+02,  2.7361e+02,  5.9083e+01,\n",
       "         -5.0169e+02, -4.5446e+01, -2.6564e+02,  3.2312e+02,  3.7512e+02,\n",
       "         -3.9090e+02, -4.2449e+02,  2.1926e+02, -1.4234e+02,  2.5136e+02,\n",
       "         -1.4026e+01,  3.6964e+02,  2.3618e+01, -3.2436e+02, -1.2837e+03,\n",
       "         -1.4325e+01, -1.1536e+02, -2.0041e+02,  2.3834e+02,  4.2544e+02,\n",
       "          1.2565e+02, -3.6622e+02,  3.8977e+01,  2.6816e+02,  5.4912e+02,\n",
       "          4.6887e+02,  7.1023e+01, -1.6232e+02, -1.9089e+02, -2.3405e+02,\n",
       "         -8.8079e+02, -7.4223e+02, -5.5267e+01,  2.6141e+02, -1.0315e+03,\n",
       "          2.7396e+02, -9.5474e+01,  2.2451e+02,  1.0806e+01,  2.6620e+02,\n",
       "          9.2237e+02,  2.6714e+02,  7.9035e+02, -5.7377e+02, -2.7979e+02,\n",
       "         -2.7053e+02, -6.8766e+01, -3.8627e+02,  9.2206e+02, -1.2164e+02,\n",
       "          1.0327e+02, -6.4954e+02, -1.5016e+02,  1.2788e+02,  8.6339e+01,\n",
       "         -3.7921e+01,  5.9518e+01, -1.5438e+02,  3.9526e+02,  4.4069e+02,\n",
       "         -2.3947e+02, -1.8050e+02, -3.7497e+02,  6.2825e+02,  7.0113e+01,\n",
       "          1.9795e+02,  2.6687e+02,  4.1485e+02,  2.1475e+02, -6.4455e+02,\n",
       "          2.5935e+02, -2.7185e+02,  4.6344e+02,  1.9831e+02,  2.2600e+01,\n",
       "          4.0598e+01, -1.2212e+02, -1.4566e+02, -3.3568e+02,  1.8356e+02,\n",
       "         -2.9237e+02,  4.4414e+02,  6.9503e+02, -2.5916e+02, -2.4259e+02,\n",
       "          9.1105e+02,  9.7090e+01,  2.5494e+01,  7.5610e+02, -3.8378e+02,\n",
       "         -1.3976e+02, -1.7858e+02, -3.9706e+02,  1.1848e+02, -4.5847e+02,\n",
       "          2.0841e+02, -4.5975e+02, -1.2324e+02,  5.2296e+02, -5.4155e+02,\n",
       "          4.4972e+02, -6.5430e+02, -1.4461e+02,  2.8564e+02,  2.1572e+01,\n",
       "          1.5872e+02,  2.1626e+02, -8.2845e+01,  4.2202e+02,  4.9205e+02,\n",
       "          1.7639e+02, -1.6502e+02,  3.0306e+02, -1.3823e+02, -6.8398e+02,\n",
       "          5.1979e+02,  3.5334e+02, -4.1855e+02, -1.6025e+02, -5.2987e+02,\n",
       "         -8.2455e+02, -5.0728e+02,  2.6435e+02,  3.5323e+01,  2.5815e+02,\n",
       "         -6.9529e+02,  3.4908e+02,  3.3547e+02,  4.4027e+02, -5.0779e+02,\n",
       "         -4.3817e+02, -4.0281e+02, -9.1469e+02, -9.7607e+01,  6.6678e+02,\n",
       "          4.2140e+02,  3.7513e+01, -5.8438e+02, -2.9594e+02, -2.3321e+02,\n",
       "         -8.8473e+01, -1.8154e+02,  1.8004e+02, -1.7939e+02, -1.0392e+03,\n",
       "         -6.9586e+01, -1.6352e+02, -8.5006e+01,  1.6965e+02, -1.2759e+02,\n",
       "         -1.4567e+02, -2.5177e+01, -5.0272e+02,  4.7460e+02,  7.1992e+01,\n",
       "         -7.3838e+01, -3.6651e+01,  4.5545e+02,  4.5822e+02, -9.2096e+01,\n",
       "         -4.2830e+02,  3.7701e+02, -4.1535e+02,  2.2388e+02,  7.9902e+01,\n",
       "         -5.2257e+01, -1.6417e+02,  8.2422e+02, -9.7924e+01,  4.7071e+02,\n",
       "         -3.0944e+02,  7.2611e+02,  4.9859e+02, -3.0669e+01, -3.6124e+02,\n",
       "         -1.3953e+02, -4.2914e+02,  4.9659e+02,  3.9316e+02, -3.1666e+02,\n",
       "         -4.1125e+02, -1.6739e+02,  4.4393e+02,  8.0483e+00,  4.0819e+02,\n",
       "         -1.0586e+02, -4.0234e+02, -6.2819e+02,  7.2631e+02,  1.6988e+01,\n",
       "         -1.2279e+02,  7.6849e+01,  2.7899e+02, -4.6537e+02,  4.2655e+02,\n",
       "         -9.1980e+01,  2.7571e+02, -2.0620e+02, -3.9309e+02,  1.5684e+02,\n",
       "         -3.1067e+02, -3.2361e+02,  3.5487e+02, -9.1997e+01, -1.1147e+03,\n",
       "          6.4584e+02, -4.6808e+02,  1.5129e+02, -1.1218e+02, -2.4221e+02,\n",
       "          2.4173e+02,  8.8710e+01,  5.4427e+02,  1.4123e+02,  4.6462e+02,\n",
       "         -3.7672e+02,  7.2551e+02, -4.9485e+02, -2.2441e+02, -6.9252e+01,\n",
       "         -8.6532e-01, -4.4664e+02, -2.0984e+02,  2.4905e+01,  9.7146e+01,\n",
       "         -7.6141e+02,  8.8477e+01, -2.5879e+02, -4.6600e+02, -4.3579e+01,\n",
       "          5.3707e+02, -3.7185e+02,  3.3976e+02,  3.1303e+01, -2.3049e+02,\n",
       "          3.9758e+02, -1.7519e+02, -3.7043e+01, -3.9943e+02,  3.3743e+02,\n",
       "          2.2782e+02,  2.4159e+02,  1.8368e+02, -4.4273e+02,  4.8062e+02,\n",
       "          1.3584e+02,  2.9416e+02,  1.9423e+02, -1.0387e+02,  9.9483e+01,\n",
       "          3.6273e+02,  9.1483e+01,  3.8770e+02,  3.9885e+02, -1.0412e+01,\n",
       "         -4.7717e+02,  5.3602e+02,  2.4788e+02, -3.4196e+02, -3.4465e+02,\n",
       "          1.7804e+02,  4.6991e+02, -6.9310e+01, -7.4023e+02, -2.9639e+00,\n",
       "          1.4175e+02, -3.5693e+02,  6.5954e+02,  2.5452e+02,  3.9122e+02,\n",
       "         -2.7705e+02,  7.3666e+01, -1.4659e+03,  4.2509e+02,  5.3138e+01,\n",
       "         -5.3042e+00, -1.4487e+02, -2.2405e+02, -1.1034e+02, -4.0961e+02,\n",
       "          1.3572e+02,  5.4668e+01, -3.9047e+02,  8.7675e+02, -5.5364e+02,\n",
       "          4.9668e+02, -1.7020e+02, -3.5747e+02,  8.3222e+02, -5.9133e+02,\n",
       "         -4.6542e+02, -1.2796e+02,  1.3208e+02,  1.3661e+03, -2.6482e+01,\n",
       "         -1.0182e+02, -4.8932e+01, -3.2859e+02,  4.6727e+01, -3.7367e+02,\n",
       "          4.1094e+02, -1.5277e+02,  1.4192e+02, -4.7675e+02, -5.0763e+02,\n",
       "         -9.2668e+02,  6.1095e+02,  3.9502e+02, -3.4466e+02,  1.7194e+02,\n",
       "         -3.7077e+02, -2.2110e+02,  6.8594e+02, -2.0560e+02,  2.7167e+02,\n",
       "         -6.8055e+02, -1.9057e+02, -1.1861e+02, -4.6808e+02,  3.0654e+02,\n",
       "          2.7840e+02,  2.7441e+00,  5.9541e+02,  6.9475e+02,  4.9954e+02,\n",
       "         -6.3878e+02, -2.5928e+02, -1.3275e+02,  1.5788e+02,  4.6444e+02,\n",
       "         -5.2964e+02, -8.0482e+02,  3.3680e+02, -9.3501e+02,  4.0495e+02,\n",
       "         -1.1400e+02, -5.8796e+02, -1.4785e+02,  8.6085e+02, -1.0405e+02,\n",
       "          5.4163e+02,  2.3230e+02,  5.9447e+02,  9.3727e+02, -2.6732e+02,\n",
       "         -6.2658e+02,  7.5803e+00, -2.9326e+02, -1.1451e+02,  1.5272e+02,\n",
       "         -2.2072e+01,  3.6864e+02, -6.1525e+01,  1.2362e+02, -7.1540e+01,\n",
       "          3.6985e+01, -4.1313e+02, -2.2188e+02,  3.9026e+02, -3.8367e+02,\n",
       "          6.2831e+01,  3.9141e+02,  2.4991e+02, -1.1777e+02, -7.6084e+01,\n",
       "          3.2167e+02, -3.5445e+02, -3.7104e+02,  9.2464e+02, -2.9764e+02,\n",
       "          7.3742e+01, -4.8488e+02, -4.9975e+02, -5.7916e+02, -1.9291e+02,\n",
       "         -3.6183e+02, -2.0930e+02, -4.4956e+02,  1.3652e+02, -1.4960e+02,\n",
       "         -2.5542e+02,  4.3504e+02, -1.9320e+02, -5.4805e+02, -3.3432e+02,\n",
       "          4.0912e+01,  6.1552e+02, -2.2757e+02, -1.3956e+02, -2.6146e+01,\n",
       "          2.8843e+02,  1.5241e+02,  1.7518e+02,  5.6542e+01, -3.2075e+02,\n",
       "          1.9810e+02,  9.6542e-02, -5.5208e+02, -1.6427e+02, -8.2022e+02,\n",
       "         -7.4343e+02, -9.6633e+02, -6.4457e+01, -1.7622e+02, -4.0802e+02,\n",
       "          6.2523e+01, -2.0747e+02,  2.0634e+02,  4.4508e+02, -8.9856e+02,\n",
       "          2.0088e+02, -5.9346e+02,  1.1087e+03, -3.4555e+02, -6.5246e+02,\n",
       "          3.6947e+02,  3.5772e+02, -4.4921e+02, -9.0990e+00,  6.4548e+01,\n",
       "          3.9060e+02,  7.6371e+02,  2.2217e+01, -3.8402e+02,  3.0521e+02,\n",
       "          1.4130e+02, -6.7182e+02,  5.4663e+02,  2.1563e+02, -8.4616e+02,\n",
       "          8.1430e+02,  3.7077e+02,  1.5854e+01, -1.4623e+02, -1.8585e+02,\n",
       "         -2.2223e+02, -5.4151e+01,  1.8132e+02,  3.0353e+01,  2.1823e+02,\n",
       "          7.3801e+01, -4.6897e+01, -1.4134e+02,  6.6142e+02,  7.2181e+01,\n",
       "          2.8441e+02, -8.0700e+01, -2.0282e+01, -7.7386e+01,  2.5071e+02,\n",
       "          1.3102e+02, -7.1981e+01,  4.6047e+02, -3.1557e+02, -1.4651e+02,\n",
       "          1.7395e+02, -5.1684e+02,  4.6460e+02, -1.7805e+02, -2.6076e+02,\n",
       "         -4.4614e+02,  5.5470e+02,  3.4788e+02]])"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.matmul(torch_tensor[:unsqueeze](0), torch_model[\"bert.encoder.layer.5.attention.self.value.weight\"][:cpu]())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2×768 Array{Float32,2}:\n",
       " -0.0210782  -0.0298697  -0.0114933   0.0265236  …  -0.0291932   0.00926572\n",
       "  0.0132354   0.0372161   0.0114068  -0.0150693      0.035094   -0.0266885 "
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch_model[\"cls.seq_relationship.weight\"][:cpu]()[:numpy]()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "768×1 Array{Float32,2}:\n",
       "   288.88898 \n",
       "  -445.91956 \n",
       "   166.7278  \n",
       " -1204.4648  \n",
       "   214.86862 \n",
       "  -319.90027 \n",
       "  -216.44069 \n",
       "    66.7233  \n",
       "   453.52155 \n",
       "   -45.058266\n",
       "  -181.30641 \n",
       " -1002.7122  \n",
       "  -568.46985 \n",
       "     ⋮       \n",
       "   -71.98135 \n",
       "   460.47424 \n",
       "  -315.56506 \n",
       "  -146.51422 \n",
       "   173.94786 \n",
       "  -516.8409  \n",
       "   464.60254 \n",
       "  -178.05414 \n",
       "  -260.75806 \n",
       "  -446.14297 \n",
       "   554.69507 \n",
       "   347.88165 "
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "permutedims(torch_model[\"bert.encoder.layer.5.attention.self.value.weight\"][:cpu]()[:numpy](), (2,1)) * Array{Float32}(reshape(collect(1:768).*1.0, (768,1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PyObject tensor([  1.,   2.,   3.,   4.,   5.,   6.,   7.,   8.,   9.,  10.,  11.,  12.,\n",
       "         13.,  14.,  15.,  16.,  17.,  18.,  19.,  20.,  21.,  22.,  23.,  24.,\n",
       "         25.,  26.,  27.,  28.,  29.,  30.,  31.,  32.,  33.,  34.,  35.,  36.,\n",
       "         37.,  38.,  39.,  40.,  41.,  42.,  43.,  44.,  45.,  46.,  47.,  48.,\n",
       "         49.,  50.,  51.,  52.,  53.,  54.,  55.,  56.,  57.,  58.,  59.,  60.,\n",
       "         61.,  62.,  63.,  64.,  65.,  66.,  67.,  68.,  69.,  70.,  71.,  72.,\n",
       "         73.,  74.,  75.,  76.,  77.,  78.,  79.,  80.,  81.,  82.,  83.,  84.,\n",
       "         85.,  86.,  87.,  88.,  89.,  90.,  91.,  92.,  93.,  94.,  95.,  96.,\n",
       "         97.,  98.,  99., 100., 101., 102., 103., 104., 105., 106., 107., 108.,\n",
       "        109., 110., 111., 112., 113., 114., 115., 116., 117., 118., 119., 120.,\n",
       "        121., 122., 123., 124., 125., 126., 127., 128., 129., 130., 131., 132.,\n",
       "        133., 134., 135., 136., 137., 138., 139., 140., 141., 142., 143., 144.,\n",
       "        145., 146., 147., 148., 149., 150., 151., 152., 153., 154., 155., 156.,\n",
       "        157., 158., 159., 160., 161., 162., 163., 164., 165., 166., 167., 168.,\n",
       "        169., 170., 171., 172., 173., 174., 175., 176., 177., 178., 179., 180.,\n",
       "        181., 182., 183., 184., 185., 186., 187., 188., 189., 190., 191., 192.,\n",
       "        193., 194., 195., 196., 197., 198., 199., 200., 201., 202., 203., 204.,\n",
       "        205., 206., 207., 208., 209., 210., 211., 212., 213., 214., 215., 216.,\n",
       "        217., 218., 219., 220., 221., 222., 223., 224., 225., 226., 227., 228.,\n",
       "        229., 230., 231., 232., 233., 234., 235., 236., 237., 238., 239., 240.,\n",
       "        241., 242., 243., 244., 245., 246., 247., 248., 249., 250., 251., 252.,\n",
       "        253., 254., 255., 256., 257., 258., 259., 260., 261., 262., 263., 264.,\n",
       "        265., 266., 267., 268., 269., 270., 271., 272., 273., 274., 275., 276.,\n",
       "        277., 278., 279., 280., 281., 282., 283., 284., 285., 286., 287., 288.,\n",
       "        289., 290., 291., 292., 293., 294., 295., 296., 297., 298., 299., 300.,\n",
       "        301., 302., 303., 304., 305., 306., 307., 308., 309., 310., 311., 312.,\n",
       "        313., 314., 315., 316., 317., 318., 319., 320., 321., 322., 323., 324.,\n",
       "        325., 326., 327., 328., 329., 330., 331., 332., 333., 334., 335., 336.,\n",
       "        337., 338., 339., 340., 341., 342., 343., 344., 345., 346., 347., 348.,\n",
       "        349., 350., 351., 352., 353., 354., 355., 356., 357., 358., 359., 360.,\n",
       "        361., 362., 363., 364., 365., 366., 367., 368., 369., 370., 371., 372.,\n",
       "        373., 374., 375., 376., 377., 378., 379., 380., 381., 382., 383., 384.,\n",
       "        385., 386., 387., 388., 389., 390., 391., 392., 393., 394., 395., 396.,\n",
       "        397., 398., 399., 400., 401., 402., 403., 404., 405., 406., 407., 408.,\n",
       "        409., 410., 411., 412., 413., 414., 415., 416., 417., 418., 419., 420.,\n",
       "        421., 422., 423., 424., 425., 426., 427., 428., 429., 430., 431., 432.,\n",
       "        433., 434., 435., 436., 437., 438., 439., 440., 441., 442., 443., 444.,\n",
       "        445., 446., 447., 448., 449., 450., 451., 452., 453., 454., 455., 456.,\n",
       "        457., 458., 459., 460., 461., 462., 463., 464., 465., 466., 467., 468.,\n",
       "        469., 470., 471., 472., 473., 474., 475., 476., 477., 478., 479., 480.,\n",
       "        481., 482., 483., 484., 485., 486., 487., 488., 489., 490., 491., 492.,\n",
       "        493., 494., 495., 496., 497., 498., 499., 500., 501., 502., 503., 504.,\n",
       "        505., 506., 507., 508., 509., 510., 511., 512., 513., 514., 515., 516.,\n",
       "        517., 518., 519., 520., 521., 522., 523., 524., 525., 526., 527., 528.,\n",
       "        529., 530., 531., 532., 533., 534., 535., 536., 537., 538., 539., 540.,\n",
       "        541., 542., 543., 544., 545., 546., 547., 548., 549., 550., 551., 552.,\n",
       "        553., 554., 555., 556., 557., 558., 559., 560., 561., 562., 563., 564.,\n",
       "        565., 566., 567., 568., 569., 570., 571., 572., 573., 574., 575., 576.,\n",
       "        577., 578., 579., 580., 581., 582., 583., 584., 585., 586., 587., 588.,\n",
       "        589., 590., 591., 592., 593., 594., 595., 596., 597., 598., 599., 600.,\n",
       "        601., 602., 603., 604., 605., 606., 607., 608., 609., 610., 611., 612.,\n",
       "        613., 614., 615., 616., 617., 618., 619., 620., 621., 622., 623., 624.,\n",
       "        625., 626., 627., 628., 629., 630., 631., 632., 633., 634., 635., 636.,\n",
       "        637., 638., 639., 640., 641., 642., 643., 644., 645., 646., 647., 648.,\n",
       "        649., 650., 651., 652., 653., 654., 655., 656., 657., 658., 659., 660.,\n",
       "        661., 662., 663., 664., 665., 666., 667., 668., 669., 670., 671., 672.,\n",
       "        673., 674., 675., 676., 677., 678., 679., 680., 681., 682., 683., 684.,\n",
       "        685., 686., 687., 688., 689., 690., 691., 692., 693., 694., 695., 696.,\n",
       "        697., 698., 699., 700., 701., 702., 703., 704., 705., 706., 707., 708.,\n",
       "        709., 710., 711., 712., 713., 714., 715., 716., 717., 718., 719., 720.,\n",
       "        721., 722., 723., 724., 725., 726., 727., 728., 729., 730., 731., 732.,\n",
       "        733., 734., 735., 736., 737., 738., 739., 740., 741., 742., 743., 744.,\n",
       "        745., 746., 747., 748., 749., 750., 751., 752., 753., 754., 755., 756.,\n",
       "        757., 758., 759., 760., 761., 762., 763., 764., 765., 766., 767., 768.])"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch_tensor = torch.tensor(numpy.array(1:768), dtype=torch.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 768)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch_tensor[:unsqueeze](0)[:size]()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PyObject tensor([  1,   2,   3,   4,   5,   6,   7,   8,   9,  10,  11,  12,  13,  14,\n",
       "         15,  16,  17,  18,  19,  20,  21,  22,  23,  24,  25,  26,  27,  28,\n",
       "         29,  30,  31,  32,  33,  34,  35,  36,  37,  38,  39,  40,  41,  42,\n",
       "         43,  44,  45,  46,  47,  48,  49,  50,  51,  52,  53,  54,  55,  56,\n",
       "         57,  58,  59,  60,  61,  62,  63,  64,  65,  66,  67,  68,  69,  70,\n",
       "         71,  72,  73,  74,  75,  76,  77,  78,  79,  80,  81,  82,  83,  84,\n",
       "         85,  86,  87,  88,  89,  90,  91,  92,  93,  94,  95,  96,  97,  98,\n",
       "         99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112,\n",
       "        113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126,\n",
       "        127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140,\n",
       "        141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154,\n",
       "        155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168,\n",
       "        169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182,\n",
       "        183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196,\n",
       "        197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210,\n",
       "        211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224,\n",
       "        225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238,\n",
       "        239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252,\n",
       "        253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266,\n",
       "        267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280,\n",
       "        281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294,\n",
       "        295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308,\n",
       "        309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322,\n",
       "        323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336,\n",
       "        337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350,\n",
       "        351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364,\n",
       "        365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378,\n",
       "        379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, 392,\n",
       "        393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406,\n",
       "        407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420,\n",
       "        421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434,\n",
       "        435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 448,\n",
       "        449, 450, 451, 452, 453, 454, 455, 456, 457, 458, 459, 460, 461, 462,\n",
       "        463, 464, 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475, 476,\n",
       "        477, 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490,\n",
       "        491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504,\n",
       "        505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 516, 517, 518,\n",
       "        519, 520, 521, 522, 523, 524, 525, 526, 527, 528, 529, 530, 531, 532,\n",
       "        533, 534, 535, 536, 537, 538, 539, 540, 541, 542, 543, 544, 545, 546,\n",
       "        547, 548, 549, 550, 551, 552, 553, 554, 555, 556, 557, 558, 559, 560,\n",
       "        561, 562, 563, 564, 565, 566, 567, 568, 569, 570, 571, 572, 573, 574,\n",
       "        575, 576, 577, 578, 579, 580, 581, 582, 583, 584, 585, 586, 587, 588,\n",
       "        589, 590, 591, 592, 593, 594, 595, 596, 597, 598, 599, 600, 601, 602,\n",
       "        603, 604, 605, 606, 607, 608, 609, 610, 611, 612, 613, 614, 615, 616,\n",
       "        617, 618, 619, 620, 621, 622, 623, 624, 625, 626, 627, 628, 629, 630,\n",
       "        631, 632, 633, 634, 635, 636, 637, 638, 639, 640, 641, 642, 643, 644,\n",
       "        645, 646, 647, 648, 649, 650, 651, 652, 653, 654, 655, 656, 657, 658,\n",
       "        659, 660, 661, 662, 663, 664, 665, 666, 667, 668, 669, 670, 671, 672,\n",
       "        673, 674, 675, 676, 677, 678, 679, 680, 681, 682, 683, 684, 685, 686,\n",
       "        687, 688, 689, 690, 691, 692, 693, 694, 695, 696, 697, 698, 699, 700,\n",
       "        701, 702, 703, 704, 705, 706, 707, 708, 709, 710, 711, 712, 713, 714,\n",
       "        715, 716, 717, 718, 719, 720, 721, 722, 723, 724, 725, 726, 727, 728,\n",
       "        729, 730, 731, 732, 733, 734, 735, 736, 737, 738, 739, 740, 741, 742,\n",
       "        743, 744, 745, 746, 747, 748, 749, 750, 751, 752, 753, 754, 755, 756,\n",
       "        757, 758, 759, 760, 761, 762, 763, 764, 765, 766, 767, 768])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.matmul(torch_tensor,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "load_from_torch_classification (generic function with 1 method)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "include(\"model.jl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertPreTraining(Bert(EmbedLayer(Embedding(KFloat32[0.00618323 -0.0169476 … -0.0499978 -0.0177459; -0.0568268 -0.0541772 … -0.0468287 -0.0793887; … ; -0.0306487 -0.0289911 … -0.0284732 -0.0283612; -0.0130648 -0.0109822 … -0.0201913 0.0571289]), Embedding(KFloat32[0.0223904 -0.00928706 … 0.01999 0.00243514; -0.00443635 -0.0165968 … -0.00554573 -0.021461; … ; -0.00410777 0.0431863 … -0.0115938 0.0253736; 0.0127998 -0.019281 … -0.0258449 -0.0716339]), Embedding(KFloat32[0.00328108 -0.00180102; 0.00675674 0.00141564; … ; -0.00558636 -0.00243301; -0.00825712 -0.00647022]), LayerNormalization(KFloat32[0.921038, 0.892587, 0.840731, 0.872586, 0.910665, 0.903145, 0.928933, 0.91257, 0.954167, 0.805993  …  0.868524, 0.781676, 0.918081, 0.889764, 0.861143, 0.908655, 0.903872, 0.783843, 0.878898, 0.962423], KFloat32[-0.0262504, -0.0262299, 0.0461384, 0.097543, -0.0358597, -0.0405648, 0.0265303, -0.032672, -0.00250848, -0.00376075  …  -0.0107369, -0.0639957, -0.0192386, -0.0656198, -0.0321019, 0.00774759, -0.0453048, -0.0214143, 0.0199053, 0.029999], 1.0e-12), 256, 0.1), Encoder[Encoder(SelfAttention(Linear3D(KFloat32[-0.0125314 0.0258932 … 0.0638374 0.0389537; -0.0240974 0.0491387 … 0.0971662 0.0210878; … ; -0.0100436 0.0934063 … 0.0621778 0.0490496; 0.0195016 -0.0888619 … -0.0484028 0.00518785], KFloat32[0.596514, -0.338774, -0.420514, 0.37973, -0.307352, 0.449397, 0.049057, 0.264723, 0.255708, -0.134555  …  -0.131681, -0.042187, 0.205793, 0.0547412, 0.295678, 0.00343477, -0.236189, 0.181864, -0.115694, -0.0853654]), Linear3D(KFloat32[0.0230095 -0.0389667 … -0.0133398 -0.00706895; 0.0150291 -0.0522657 … -0.0297042 0.0176523; … ; -0.0407323 0.00926333 … 0.00476213 -0.0447775; -0.000411416 -0.0468519 … -0.0284424 0.00967882], KFloat32[0.00104263, 0.000350291, -0.00246132, -0.000105829, -0.00119185, 0.0026922, 0.00227751, 0.000840547, 0.00223686, 0.000857024  …  -0.00302352, -0.000804328, -0.00721847, -0.00118126, -0.000522741, -0.000179581, 0.00369289, -0.0055787, 0.00572705, 0.00293136]), Linear3D(KFloat32[0.0209114 -0.00499297 … -0.0388448 0.00329649; -0.0263529 -0.00667362 … -0.0128339 -0.0292286; … ; -0.00555018 0.0377352 … 0.0169142 -0.0193624; -0.0201704 -0.0319768 … -0.0515445 -0.0370621], KFloat32[0.00475926, -0.00902621, 0.00414675, 0.00209415, -0.00131872, 0.0181394, -0.0280916, 0.0257081, -0.00380245, 0.0213524  …  0.0260964, 0.00548256, -0.0191767, 0.0378439, 0.00443777, -0.00298669, -0.00416985, -0.0117145, -0.00943393, -0.0173731]), Linear3D(KFloat32[-0.00853095 7.99652e-5 … -0.0204222 0.0405484; 0.0241711 -2.86947e-6 … 0.00767818 -0.00850635; … ; -0.000886114 -0.02211 … -0.04421 -0.0382556; -0.0477372 0.0120636 … -0.0129909 -0.0216257], KFloat32[0.00275622, -0.0199607, 0.02799, -0.0126683, 0.0290275, -0.0517351, -0.0162047, 0.00514804, -0.0198569, 0.0126659  …  -0.0247672, 0.0114378, -0.00714289, -0.00634609, -0.0207949, -0.0340426, -0.0289499, -0.0248435, 0.0164384, -0.0283311]), 12, 256, 768, 64, 8, 0.1, 0.1), LayerNormalization(KFloat32[0.981005, 0.960729, 0.96616, 0.976298, 0.993649, 0.996294, 0.977041, 0.976117, 0.968032, 0.947049  …  0.931828, 0.940822, 0.964244, 0.935927, 0.945918, 0.957091, 0.967795, 0.940884, 0.944589, 0.995523], KFloat32[0.255415, -0.0321045, -0.275129, -0.395657, 0.383793, -0.477691, 0.531461, 0.191811, -0.00405172, -0.00984386  …  -0.196113, 0.135935, 0.201685, 0.0775654, -0.0735761, -0.272588, 0.0775897, 0.172075, -0.0795021, -0.120116], 1.0e-12), FeedForward(Dense(Linear3D(KFloat32[-0.000376311 0.01154 … 0.0108399 -0.0228205; -0.0796406 0.0233403 … -0.0442858 -0.002353; … ; 0.0297406 -0.0222852 … 0.0052678 0.0118438; -0.0233839 0.0257399 … 0.0130273 0.0711318], KFloat32[-0.1169, -0.100037, -0.125791, -0.130125, -0.0587654, -0.149743, -0.100815, -0.119838, -0.137706, -0.132159  …  -0.129497, -0.114819, -0.134292, -0.14221, -0.0197138, -0.139995, -0.104713, -0.113175, -0.127959, -0.0824315]), 0.0, gelu), Linear3D(KFloat32[-0.043129 -0.0244816 … 0.0236394 0.0181582; 0.0607264 -0.022502 … -0.0122041 -0.0103681; … ; -0.0632376 0.0174214 … -0.0599947 0.0572332; -0.0555597 0.00667871 … 0.00968108 0.0360241], KFloat32[-0.0554663, 0.21027, 0.0285571, 0.0362072, 0.0419793, 0.0443257, -0.0585574, 0.0577716, 0.0110956, 0.0926527  …  0.218722, -0.116133, 0.0144059, 0.132921, 0.0142233, -0.0368112, -0.0589292, 0.0378082, -0.00582394, -0.00224854]), 0.1), LayerNormalization(KFloat32[0.785673, 0.807163, 0.771382, 0.74021, 0.775586, 0.770403, 0.76655, 0.795674, 0.811372, 0.763524  …  0.797166, 0.759269, 0.783123, 0.791754, 0.784195, 0.77442, 0.779225, 0.736311, 0.762143, 0.810764], KFloat32[-0.107692, 0.00229767, 0.0534346, 0.101521, -0.103445, 0.0832797, -0.194398, -0.0862236, -0.0268219, -0.0434362  …  0.0824809, -0.125599, -0.0956205, -0.0462591, 0.0151909, 0.0483596, -0.0703633, -0.0996856, 0.0105609, 0.0346681], 1.0e-12)), Encoder(SelfAttention(Linear3D(KFloat32[0.0370278 -0.0208408 … -0.0491242 0.0692049; -0.00948053 -0.0214701 … 0.0436366 0.0326059; … ; -0.0107077 -0.0136335 … 0.00620687 -0.0661955; -0.000562126 -0.0317406 … -0.0503491 0.0181204], KFloat32[-0.260608, 0.173605, -0.241902, -0.883867, 0.416969, 0.0196286, 0.49937, 0.270704, 0.396154, 0.0389215  …  -0.0254294, -0.0552639, 0.0188127, -0.0130626, 0.0263363, 0.0451571, -0.000837979, -0.0770189, 0.126624, -0.00740442]), Linear3D(KFloat32[-0.0372935 0.0313145 … 0.00659017 0.028299; 0.0429187 0.0702135 … -0.0126181 0.0280364; … ; -0.0074265 0.00210492 … -0.0486102 -0.00133493; 0.00336592 -0.0596365 … 0.0155881 0.0550298], KFloat32[-0.00504376, 0.00136878, -0.00394327, 0.00646116, -0.00148918, -0.00190902, 0.000532792, 0.000275371, 0.000303158, 0.00140118  …  -0.00727293, 0.00317768, 0.0138626, 0.00289187, 0.00572978, 0.00607313, -0.00666384, 0.00478895, -0.00100174, -0.00184508]), Linear3D(KFloat32[-0.0186243 0.00831199 … 0.0544783 0.0493129; 0.0544522 -0.00648363 … -0.0114433 0.0156144; … ; -0.0300538 -0.0374185 … 0.0277062 0.00347097; -0.0232271 -0.00370202 … 0.0291191 0.0220723], KFloat32[0.00537398, 0.0299914, -0.00458131, -0.00479999, -0.0152428, -0.0201227, 0.105387, 0.0050875, -0.00595852, -0.0128426  …  0.0066381, 0.0481591, -0.00425096, -0.0323468, 0.0243423, -0.0463222, -0.0241348, -0.033772, 0.0408227, 0.0200426]), Linear3D(KFloat32[-0.0223945 0.040453 … -0.0360431 0.00788213; -0.00419363 0.0232656 … -0.00177737 -0.0213461; … ; 0.00198322 -0.00395896 … -0.00830607 0.0130504; -0.00933888 -0.00818775 … 0.022561 -0.0257934], KFloat32[0.028135, 0.0564571, 0.0152847, 0.018555, 0.0174515, -0.0423365, -0.0661691, -0.00157936, 0.0117425, -0.0561001  …  0.0447518, 0.0201163, -0.0512705, 0.0568477, -0.0190088, -0.0539225, -0.0389787, -0.0958418, 0.00181545, 0.0247304]), 12, 256, 768, 64, 8, 0.1, 0.1), LayerNormalization(KFloat32[0.90604, 0.873346, 0.862878, 0.877554, 0.955328, 0.949486, 0.908424, 0.911759, 0.908155, 0.872545  …  0.855257, 0.845369, 0.869542, 0.887988, 0.855211, 0.886746, 0.884908, 0.873242, 0.870944, 0.9043], KFloat32[0.0915283, 0.149914, -0.092776, -0.197814, 0.227593, -0.265666, 0.286855, 0.166128, 0.0240325, 0.110555  …  0.0513181, 0.018154, 0.0123323, 0.151684, -0.0249573, -0.208544, -0.0711839, 0.0272778, 0.0124702, 0.0160873], 1.0e-12), FeedForward(Dense(Linear3D(KFloat32[0.0295053 0.0200189 … 0.00188981 0.0304793; -0.0315026 0.00763194 … -0.00237253 -0.0144178; … ; 0.00725679 0.0438569 … 0.0375148 0.00985773; -0.0498582 0.0514479 … 0.0134948 -0.0688336], KFloat32[-0.156577, -0.108238, -0.084496, -0.0929442, -0.0849999, -0.150465, -0.105622, -0.133359, -0.142778, -0.100207  …  -0.118635, -0.114552, -0.0837703, -0.134479, -0.107785, -0.110794, -0.0540337, -0.104319, -0.0826858, -0.123327]), 0.0, gelu), Linear3D(KFloat32[-0.0192446 0.0101193 … 0.0157793 0.0621471; 0.0335926 -0.0210024 … 0.007813 -0.0175086; … ; 0.0186974 -0.0166882 … -0.0456305 -0.0220007; 0.00645953 0.0361526 … -0.0317369 -0.0435879], KFloat32[-0.0284274, 0.107856, -0.0294227, 0.0420162, 0.0104208, 0.0189943, -0.0636056, 0.10486, -0.0295812, 0.110354  …  0.131375, -0.0725058, -0.024738, 0.0887463, 0.0551387, 0.00640024, -0.0115556, 0.00339068, 0.00614762, -0.0165344]), 0.1), LayerNormalization(KFloat32[0.908577, 0.886652, 0.884023, 0.862472, 0.877384, 0.885216, 0.870277, 0.92371, 0.930716, 0.886638  …  0.894843, 0.853561, 0.879556, 0.881597, 0.88034, 0.883897, 0.906034, 0.829055, 0.892955, 0.93511], KFloat32[-0.0798401, -0.102697, 0.0357304, 0.0793252, -0.112901, 0.121691, -0.195561, -0.139463, -0.0777243, -0.0665637  …  -0.028993, -0.0617384, -0.0565754, -0.0885289, -0.0465065, 0.0760931, 0.00593424, -0.0469942, -0.0277054, -0.0433663], 1.0e-12)), Encoder(SelfAttention(Linear3D(KFloat32[0.0759656 -0.000217702 … -0.0971586 -0.01535; 0.0852017 -0.010774 … -0.00577288 -0.0323714; … ; -0.0114838 -0.0305947 … -0.0561908 -0.0293154; -0.0229191 -0.0296405 … -0.0450458 0.0333894], KFloat32[0.0254483, 0.0285178, 0.291815, -0.234706, -0.0464112, -0.0624038, -0.107604, 0.171888, 0.238523, 0.0551293  …  -0.302419, -0.0549704, 0.100568, -0.147296, -0.10907, 0.0376631, 0.0125388, -0.068721, 0.0745724, 0.0234039]), Linear3D(KFloat32[0.0640576 0.0921455 … 0.0922393 -0.0283436; 0.0199365 0.0160515 … 0.0551377 0.0150158; … ; 0.017364 -0.0342777 … 0.0305553 -0.0164251; -0.00233834 0.0804035 … 0.0260061 0.0412468], KFloat32[-0.000373189, -0.00125872, -0.00112986, -0.00158112, 0.000367756, -0.00160734, -0.00129028, 0.00213482, 0.00261693, 0.00022549  …  0.00144676, -0.000181783, 0.000379259, -0.00108257, 0.000987125, -0.00164541, -0.00260254, -0.000718777, 0.00377168, 0.00106194]), Linear3D(KFloat32[-0.010249 -0.0424589 … 0.0144307 0.0229542; -0.0150867 -0.0122535 … -0.0269445 -0.0312312; … ; -0.0258091 0.0489394 … 0.00634609 0.00521732; -0.0319188 0.00718025 … 0.0169361 -0.0533985], KFloat32[-0.0342076, 0.0458289, 0.0823005, 0.0522363, 0.615098, 0.111481, -0.0553877, -0.0772435, -0.0343578, -0.0385267  …  0.0205542, -0.00680909, 0.00020832, 0.00872338, 0.00818579, 0.00254061, 0.0105948, 0.0202389, -0.0179387, -0.0273126]), Linear3D(KFloat32[0.0105526 0.0129336 … -0.0246751 0.00645919; -0.00429616 0.0473612 … -0.00807052 -0.0187276; … ; -0.0347141 -0.0397574 … -0.019491 0.0138953; 0.0187616 0.0198496 … -0.0200336 0.0341364], KFloat32[0.0217654, 0.125197, -0.0604438, 0.0625303, 0.0151629, -0.00361173, -0.0478906, 0.0138153, -0.0403713, 0.0443183  …  0.0821696, 0.0751011, -0.0521693, 0.124177, 0.00171279, -0.060567, 0.0138211, -0.0813914, 0.088452, 0.010209]), 12, 256, 768, 64, 8, 0.1, 0.1), LayerNormalization(KFloat32[0.904127, 0.894049, 0.866781, 0.869467, 0.964117, 0.924624, 0.910701, 0.902536, 0.890728, 0.868137  …  0.850931, 0.800874, 0.885143, 0.857458, 0.839023, 0.897707, 0.879436, 0.837317, 0.832761, 0.910809], KFloat32[0.156521, 0.136634, -0.199834, -0.275356, 0.341599, -0.279864, 0.25523, 0.0596012, -0.08324, 0.0230492  …  -0.114911, -0.0212768, 0.0220296, 0.158053, 0.0404132, -0.261368, -0.0522587, -0.00124932, -0.11801, -0.00104689], 1.0e-12), FeedForward(Dense(Linear3D(KFloat32[0.00774802 -0.0117866 … -0.0479374 0.0578029; 0.00534976 -0.0227453 … 0.000142725 -0.00271624; … ; -0.0190045 -0.0427749 … 0.0504322 -0.0161743; 0.014534 -0.051181 … 0.0134826 -0.0285095], KFloat32[-0.084457, -0.134022, -0.118349, -0.0950558, -0.030923, -0.124014, -0.137814, -0.0240601, -0.0634831, -0.117704  …  -0.159201, -0.0619585, -0.178413, -0.164466, -0.149585, -0.0800646, -0.121284, -0.120686, -0.105587, -0.102823]), 0.0, gelu), Linear3D(KFloat32[-0.0755376 0.0240624 … -0.0163334 0.0156317; -0.018927 -0.0191297 … 0.0102754 -0.0200277; … ; 0.0263843 0.00811287 … -0.0124842 0.0534267; 0.0209432 -0.040833 … 0.0348546 0.0342538], KFloat32[-0.0655225, 0.0687298, -0.0304415, 0.0158824, 0.0577131, -0.018003, 0.000410703, 0.0908851, -0.00138296, 0.0508091  …  0.229309, -0.0155098, -0.0325667, 0.0813682, 0.00191509, -0.0248425, -0.0760201, -0.0753876, -0.0765503, 0.00308212]), 0.1), LayerNormalization(KFloat32[0.885658, 0.864353, 0.859903, 0.858881, 0.833035, 0.870993, 0.856105, 0.906345, 0.912519, 0.858712  …  0.874303, 0.796408, 0.861562, 0.856091, 0.871076, 0.841066, 0.864434, 0.81501, 0.854397, 0.921549], KFloat32[-0.117923, -0.116271, 0.08247, 0.0892243, -0.1534, 0.138356, -0.152333, -0.0795963, -0.0190692, -0.0535541  …  0.0399958, -0.0306943, -0.0433795, -0.109271, -0.0532372, 0.0943692, -0.00656187, -0.0218869, -0.00488333, -0.0243152], 1.0e-12)), Encoder(SelfAttention(Linear3D(KFloat32[0.066885 -0.0301923 … 0.0360809 0.0615241; -0.00644277 0.118435 … -0.0203959 0.03877; … ; -0.00331069 0.046319 … 0.0391922 0.0402902; 0.0428821 0.025767 … 0.0207964 -0.00565049], KFloat32[0.100931, -0.188687, 0.037339, 0.179891, -0.081973, 0.243539, -0.116227, -0.0723076, 0.13709, -0.257188  …  0.00476237, 0.0548668, 0.143323, -0.0681102, 0.0513389, 0.0280388, 0.0343582, -0.0182051, 0.0159308, 0.0602764]), Linear3D(KFloat32[0.0181927 0.00198243 … -0.161348 0.0568523; 0.0318997 0.0220526 … -0.00407304 0.0182913; … ; -0.0321061 0.0281612 … 0.00194999 -0.0122166; 0.0150139 -0.00827551 … -0.0383042 0.0187852], KFloat32[0.00188306, -0.00378867, -0.0102035, 0.00969956, -0.000944984, -0.00917916, -0.00670069, -0.00684818, 0.00104163, 0.0112271  …  -0.00142713, -0.00249779, -0.00193455, 0.00605527, -0.000900843, 0.00156933, 0.000383857, 0.00346227, -0.00310676, 0.00187968]), Linear3D(KFloat32[0.00176943 0.00765947 … 0.0235287 0.00311671; 0.00736573 -0.0216281 … 0.0102898 0.00820563; … ; -0.00581243 -0.012401 … -0.0340069 -0.00915108; 0.0149731 0.0357838 … 0.0207847 -0.0228814], KFloat32[0.0423584, 0.0567238, 0.00846541, 0.00521738, 0.0117638, -0.0622149, -0.0393573, 0.0516673, -0.0273843, -0.00723244  …  0.0219373, -0.0324176, -0.00440396, -0.00469132, 0.0492044, -0.0250479, 0.00450502, -0.00685256, 0.0044645, -0.0348979]), Linear3D(KFloat32[0.0108539 -0.0226728 … 0.00627325 -0.0323472; 0.0144648 -0.0819274 … 0.0160881 -0.0269953; … ; -0.0142739 0.0119856 … -0.0203131 -0.0340695; -0.0371828 -0.0167829 … -0.0158106 -0.0165717], KFloat32[0.0156309, 0.0399283, -0.0526082, 0.0218969, 0.0571426, 0.0302855, -0.0263746, 0.0138588, -0.058716, 0.034048  …  0.00334844, -0.154497, -0.124249, 0.144281, -0.051561, 0.00935872, -0.0326413, -0.105909, -0.0467202, -0.0629808]), 12, 256, 768, 64, 8, 0.1, 0.1), LayerNormalization(KFloat32[0.921292, 0.887782, 0.838676, 0.857617, 0.962362, 0.913804, 0.921576, 0.902043, 0.895311, 0.862046  …  0.870065, 0.736567, 0.90517, 0.878094, 0.851328, 0.891882, 0.87887, 0.805222, 0.818222, 0.914349], KFloat32[0.100857, 0.00823099, -0.0115411, -0.274716, 0.235342, -0.308405, 0.258829, 0.118597, -0.0300525, -0.0721424  …  -0.00273791, 0.077165, 0.129316, 0.142645, 0.042747, -0.173055, -0.130448, 0.0314804, -0.0805492, 0.0421403], 1.0e-12), FeedForward(Dense(Linear3D(KFloat32[-0.0335999 -0.0272393 … 0.0569726 -0.0384737; 0.018888 -0.0861342 … -0.0492593 0.0534945; … ; -0.0107991 0.0327088 … 0.023694 0.0334703; 0.000878946 0.0417587 … 0.0226051 0.0117068], KFloat32[-0.116791, -0.150641, -0.0797797, 0.00811763, -0.0937879, -0.11048, -0.10669, -0.0926289, -0.153968, -0.0936689  …  -0.036515, -0.141811, -0.231197, -0.0922033, -0.106215, -0.0852289, -0.111227, -0.115529, -0.157411, -0.0765214]), 0.0, gelu), Linear3D(KFloat32[-0.00880223 -0.0103997 … 0.0162181 0.00360934; 0.0143082 -0.0525052 … -0.00528428 -0.0198906; … ; 0.0314311 -0.00500574 … 0.0218464 -0.0124189; 0.0401606 -0.0116022 … 0.0548564 -0.00405615], KFloat32[-0.041524, 0.0886072, -0.0429818, 0.0227377, 0.124156, -0.033967, 0.00857181, 0.107083, -0.00642215, -0.0265879  …  0.104432, -0.00757867, 0.0114401, 0.0809984, -0.0147359, -0.0540675, -0.0901072, -0.0367929, 0.0349988, 0.0283808]), 0.1), LayerNormalization(KFloat32[0.836999, 0.840109, 0.824509, 0.829189, 0.806765, 0.814231, 0.809405, 0.852929, 0.867006, 0.815651  …  0.829026, 0.78288, 0.836679, 0.81931, 0.813041, 0.81319, 0.809587, 0.788196, 0.814266, 0.858404], KFloat32[-0.0788164, -0.0703204, -0.030957, 0.0572331, -0.103289, 0.0792928, -0.137313, -0.0913192, -0.0246541, -0.0165114  …  -0.0187094, -0.0578976, -0.0676776, -0.0871857, -0.0427582, 0.0163433, 0.00922753, -0.0323774, -0.00845516, -0.0541834], 1.0e-12)), Encoder(SelfAttention(Linear3D(KFloat32[0.0707842 0.0493507 … 0.056876 0.0297896; 0.0133138 -0.0273627 … 0.045083 -0.00132379; … ; -0.0416553 -0.00932119 … 0.0456476 0.00172883; -0.0370186 0.0294864 … -0.032772 -0.0174887], KFloat32[0.071113, -0.135187, -0.0485137, -0.0041652, 0.122899, -0.0274352, -0.00097572, -0.00823274, -0.0836812, -0.220541  …  0.0418395, 0.018047, -0.204099, 0.105871, 0.0927693, 0.106618, -0.0635341, 0.0590473, 0.00108998, 0.284727]), Linear3D(KFloat32[0.0807765 -0.045803 … -0.0446431 0.0132814; 0.0687294 -0.0229906 … 0.00546503 -0.0132964; … ; 0.0228461 0.047706 … -0.0717003 0.0393485; -0.0285311 0.0551969 … 0.0246282 0.0661069], KFloat32[-0.00588925, -0.00235002, 0.00281122, 0.00173234, -0.00546912, -0.00175447, -0.00444366, -0.00383022, -0.00218411, 0.00746224  …  0.00501635, -0.00155223, -0.00461728, 0.00361112, 0.000726643, -0.00557767, -0.0013016, 0.0131573, -0.00346985, -0.000398656]), Linear3D(KFloat32[0.0733598 0.00346096 … 0.0448469 0.00741321; 0.0617461 0.040462 … -0.0443935 -0.0281799; … ; 0.0132693 0.0225334 … -0.00837859 -0.0216455; -0.00991072 -0.00557414 … 0.109162 0.0192416], KFloat32[-0.0140475, -0.0285213, 0.0327146, 0.023278, -0.0393354, 0.0307359, 0.0142984, -0.00352714, 0.0157257, 0.0225164  …  0.0119436, 0.0457327, -0.0488078, 0.0242576, 0.0121584, -0.0913673, 0.0430819, -0.042235, -0.0549698, 0.0616456]), Linear3D(KFloat32[0.0321065 0.0299032 … -0.00890582 0.0133935; -0.0331082 -0.00750723 … 0.0146349 -0.0298823; … ; -0.0080715 -0.039069 … -0.0332673 0.0142988; -0.0105115 0.0012897 … -0.0194941 -0.0471456], KFloat32[0.0316142, 0.00595848, -0.0275855, -0.0396005, 0.000811242, 0.0476166, -0.0318267, -0.0324383, -0.0372905, 0.023873  …  0.00242296, -0.0460171, -0.0315085, 0.0294118, -0.0140608, -0.000906953, 0.0218056, -0.034442, 0.0195686, -0.0212355]), 12, 256, 768, 64, 8, 0.1, 0.1), LayerNormalization(KFloat32[0.897672, 0.879212, 0.819994, 0.819605, 0.957994, 0.872092, 0.89473, 0.887392, 0.876437, 0.839696  …  0.840927, 0.738816, 0.890177, 0.862094, 0.833858, 0.8771, 0.870749, 0.759147, 0.788387, 0.880433], KFloat32[0.0127199, 0.00734239, -0.0309609, -0.237056, 0.297509, -0.112122, 0.220547, 0.0562941, 0.00898308, -0.102405  …  -0.116804, 0.0744098, 0.0184153, 0.198659, 0.044474, -0.269793, -0.105838, -0.0172507, 0.036461, 0.0069042], 1.0e-12), FeedForward(Dense(Linear3D(KFloat32[0.011497 -0.00959824 … -0.0366418 -0.0127675; 0.0549355 0.0099402 … -0.0443466 -0.0209024; … ; 0.0116751 -0.0362203 … 0.0303408 -0.00476002; -0.014382 0.0703796 … 0.0510249 0.018158], KFloat32[-0.0937494, -0.117761, -0.118221, -0.00108828, -0.134517, -0.0940222, -0.0948405, -0.139225, -0.115523, -0.204285  …  -0.177183, -0.0956005, -0.0758371, 0.0113024, -0.041066, -0.0770856, -0.0826751, -0.0578993, -0.139732, -0.105456]), 0.0, gelu), Linear3D(KFloat32[-0.0362976 0.0293012 … -0.0125552 -0.0128176; 0.015134 0.00652288 … 0.0643495 0.00857149; … ; -0.000812842 0.0519178 … -0.0114406 -0.0429863; -0.0614464 -0.0507425 … -0.0289173 -0.0139343], KFloat32[-0.0555554, 0.0504975, 0.0113411, 0.0544813, 0.088264, 0.000197722, -0.0119283, 0.0771945, -0.0264195, -0.0112834  …  0.0846865, -0.0229498, -0.0150063, 0.0491452, -0.0374006, -0.0150699, -0.0814351, -0.059123, -0.0463643, 0.00848171]), 0.1), LayerNormalization(KFloat32[0.879553, 0.864118, 0.837087, 0.850229, 0.81027, 0.847874, 0.835805, 0.885869, 0.88623, 0.863625  …  0.855546, 0.790329, 0.861397, 0.830061, 0.859429, 0.809108, 0.864894, 0.812749, 0.857476, 0.896309], KFloat32[-0.0425801, -0.076427, -0.0134896, 0.0533873, -0.128034, 0.00855373, -0.125383, -0.0716734, -0.0314538, 0.0153915  …  0.0107066, -0.0537924, -0.0524468, -0.0966409, -0.0690476, 0.0742452, -0.0319214, -0.0287913, -0.0416268, -0.0554131], 1.0e-12)), Encoder(SelfAttention(Linear3D(KFloat32[-0.00703597 0.00097537 … -0.00423259 0.0174123; -0.0285358 -0.00936906 … 0.002896 0.0378083; … ; 0.0102814 0.0164477 … 0.0241533 0.0520526; 0.0135769 -0.0857086 … -0.021946 0.0800711], KFloat32[-0.0100324, -0.0184221, -0.0458246, 0.00853763, 0.0351795, 0.0413976, 0.0360579, 0.128114, 0.000118834, 0.0449305  …  0.0271423, -0.0505524, -0.0293804, -0.224029, -0.0495308, 0.0202956, -0.00938978, -0.0209216, -0.355761, -0.00516334]), Linear3D(KFloat32[0.00216522 0.0240097 … 0.003771 0.026735; 0.0220188 -0.0870823 … -0.00873408 -0.0193323; … ; 0.0225929 0.0207692 … -0.0514878 -0.00568433; 0.0533203 0.013963 … 0.00835304 -0.0117913], KFloat32[-0.0108711, -0.0117642, 0.000165407, -0.0116235, 0.00946288, -0.011528, 0.00118585, -0.00551473, -0.000733038, 0.00575767  …  0.00109014, 0.00634691, -0.0106609, -0.00927198, 0.00359072, 0.00747549, 0.00126911, -0.00255274, -0.0176024, -0.00532156]), Linear3D(KFloat32[0.0555403 -0.00437965 … 0.00575841 -0.0511; 0.0244745 -0.0270834 … 0.0464034 -0.00683464; … ; 0.0213199 -0.00891732 … 0.0228202 0.0236682; -0.0339596 -0.00585846 … -0.0256106 0.0219551], KFloat32[0.0498581, -0.0314158, -0.00567899, 0.00579905, 0.0414394, -0.0207871, -0.0106538, 0.0102934, 0.0401719, -0.0153665  …  0.0463233, -0.0193844, -0.0441503, 0.0139379, -0.00181875, -0.00152454, -0.0153797, -0.0167332, -0.0303927, -0.0106458]), Linear3D(KFloat32[-0.00946778 0.00210877 … 0.034589 0.00835275; 0.0465848 0.0240714 … 0.0637352 -0.0128467; … ; -0.0183065 0.0275782 … -0.0717007 0.00880504; 0.0116488 0.0115241 … -0.0216155 -0.00148541], KFloat32[0.000154294, 0.0217987, -0.0376636, 0.00779527, 0.0110158, 0.00900967, -0.0338065, 0.0139691, -0.0400911, -0.00863933  …  0.0358283, 0.00949225, -0.0449934, 0.0235681, -0.0261035, -0.0199313, 0.036595, -0.00453109, -0.00268802, -0.0282772]), 12, 256, 768, 64, 8, 0.1, 0.1), LayerNormalization(KFloat32[0.897715, 0.881989, 0.823586, 0.811437, 0.974489, 0.906607, 0.884648, 0.880992, 0.872106, 0.848586  …  0.852349, 0.751214, 0.8852, 0.842541, 0.82353, 0.881825, 0.875622, 0.759031, 0.811465, 0.904252], KFloat32[-0.0688961, -0.0150687, 0.102099, -0.245643, 0.294576, -0.0994807, 0.0936654, 0.139206, -0.0308536, -0.106517  …  -0.087723, 0.0458529, 0.043178, 0.10691, -0.0158028, -0.142907, -0.222188, 0.0357236, -0.0094081, 0.0408247], 1.0e-12), FeedForward(Dense(Linear3D(KFloat32[-0.0221165 0.0627425 … 0.0282371 0.01288; 0.081766 0.036988 … -0.0704142 0.0441318; … ; 0.0390884 -0.028572 … 0.0239541 0.0471209; 0.00140126 -0.0520392 … 0.012313 -0.0396623], KFloat32[-0.10015, -0.282124, -0.137595, -0.118036, -0.107364, -0.0885115, -0.0968743, -0.0380096, -0.0465242, -0.140026  …  -0.12818, -0.122244, -0.0611703, -0.172408, -0.0908077, -0.0707623, -0.13998, 0.431418, -0.0648138, -0.11109]), 0.0, gelu), Linear3D(KFloat32[0.067354 -0.0596034 … 0.0236187 -0.0239624; -0.0150209 0.0915779 … -0.0508774 0.00138836; … ; -0.046135 0.0391662 … -0.059455 -0.0071027; -0.00840838 0.0684632 … 0.0297561 0.012013], KFloat32[-0.000914299, 0.0639834, 0.000402025, 0.0384584, 0.0618297, -0.00320292, 0.0456076, 0.0628884, -0.0407698, 0.0093179  …  0.133968, -0.035967, -0.081569, 0.0326926, -0.0495605, -0.0312476, -0.0615295, -0.017486, -0.0354496, 0.0192363]), 0.1), LayerNormalization(KFloat32[0.865178, 0.853618, 0.828574, 0.822509, 0.798748, 0.848673, 0.845135, 0.865426, 0.877865, 0.84218  …  0.848693, 0.785289, 0.866089, 0.846731, 0.837512, 0.822483, 0.838173, 0.825997, 0.828336, 0.878828], KFloat32[-0.00107029, -0.0527674, -0.0613477, 0.0659776, -0.194544, 0.00024867, -0.0848779, -0.111905, -0.0217896, 0.0243767  …  0.0318716, -0.0410186, -0.0608758, -0.0834275, -0.0532214, 0.0589733, 0.0254245, -0.0326826, -0.0410644, -0.0618449], 1.0e-12)), Encoder(SelfAttention(Linear3D(KFloat32[-0.0320905 0.0631225 … -0.00997725 -0.00409498; -0.0235327 -0.0484869 … -0.0305471 0.0158107; … ; -0.0424441 0.0188584 … 0.00984728 -0.00208451; 0.00573743 -0.0389573 … 0.02388 0.0305132], KFloat32[-0.177626, 0.283176, 0.177058, -0.117357, 0.163422, -0.119488, -0.264227, -0.13899, -0.31371, -0.471069  …  -0.0234524, -0.0294037, -0.0421488, -0.171979, -0.00416284, -0.0753574, -0.00251671, 0.042349, -0.00283091, -0.361992]), Linear3D(KFloat32[0.0638853 0.0316505 … -0.043016 -0.0164335; 0.0101442 -0.0363272 … -0.0408674 0.031772; … ; 0.00612606 0.0730237 … 0.00740017 0.00792483; 0.0101818 -0.0487617 … -0.00422673 0.0102856], KFloat32[-0.004246, -0.000118961, 0.00460458, -0.00315649, 0.00434239, -0.000256675, -0.00506765, -0.00139198, -0.00540428, 0.0165043  …  -0.00175301, -0.00267699, 0.00349895, 0.00286292, -0.00177116, -0.00309696, -0.00310136, -0.00234223, -0.0052737, -0.000388134]), Linear3D(KFloat32[-0.0112262 0.00498519 … 0.0237067 0.0357379; 0.00115284 0.0015998 … -0.00560849 -0.0474511; … ; -0.0265157 0.0191543 … 0.00443737 0.0331378; 0.034277 -0.0086046 … 0.0138841 -0.0151574], KFloat32[-0.0376277, 0.0164517, 0.000885425, -0.0120121, -0.003617, 0.0504331, 0.012638, -0.00318629, 0.0357315, 0.0262525  …  -0.0460782, -0.0689359, 0.0354768, 0.0700535, 0.019437, -5.22319e-5, -0.0335336, -0.0619196, -0.0285223, -0.0197717]), Linear3D(KFloat32[0.0151619 -0.0261395 … 0.00606884 -0.00783651; -0.000703872 0.0124088 … 0.0119803 -0.0180423; … ; -0.0298908 0.0156308 … 0.0357927 -0.00764106; 0.00498726 0.0380366 … -0.0390197 0.0162486], KFloat32[-0.0084483, -0.056118, -0.0518198, -0.019386, -0.0219611, 0.0482919, -0.0319447, -0.0340057, -0.0269769, -0.0466023  …  0.0300536, -0.0405738, 0.000299813, 0.00979882, 0.00638778, 0.0186082, 0.030515, -0.0345119, -0.0139122, -0.0299993]), 12, 256, 768, 64, 8, 0.1, 0.1), LayerNormalization(KFloat32[0.898563, 0.875281, 0.800635, 0.786749, 0.928416, 0.872507, 0.867194, 0.865825, 0.847492, 0.836186  …  0.809868, 0.711962, 0.882626, 0.823361, 0.821927, 0.856895, 0.812276, 0.750555, 0.806233, 0.889248], KFloat32[-0.0679422, 0.0352994, 0.162577, -0.15743, 0.153056, -0.0459599, 0.16078, 0.048005, -0.0657152, -0.014104  …  0.02209, 0.034399, 0.0615105, 0.0766888, -0.0336425, -0.232901, -0.184738, 0.150854, -0.0378862, 0.0157306], 1.0e-12), FeedForward(Dense(Linear3D(KFloat32[0.0469478 -0.0391501 … 0.053504 0.0183964; 0.0557571 0.04502 … -0.0625994 0.00970282; … ; -0.0510314 0.0774716 … -0.00813778 -0.00744252; 0.00322046 -0.0119369 … 0.0614438 0.0548934], KFloat32[-0.087062, -0.16334, -0.107593, -0.100631, -0.177566, -0.129952, -0.0990657, -0.0530325, -0.0862519, -0.256029  …  -0.0707951, -0.0442083, -0.242616, -0.0838288, -0.0830784, -0.105785, -0.078577, -0.145777, -0.170239, -0.186494]), 0.0, gelu), Linear3D(KFloat32[0.00997404 2.45137e-5 … -0.00279634 0.0536866; -0.0197475 0.0182786 … 0.00982973 0.0289547; … ; 0.01007 0.0124394 … -0.00316106 0.0313432; -0.0143475 0.0425193 … 0.018453 0.0170202], KFloat32[-0.0322116, 0.0631164, 0.0317852, 0.0224458, 0.0890892, 0.0327693, 0.0664198, 0.139394, -0.0170194, 0.00735451  …  0.14323, -0.13455, -0.0806511, 0.0814679, -0.0591698, 0.0369674, -0.0691738, 0.0250644, -0.00860303, 0.00378411]), 0.1), LayerNormalization(KFloat32[0.87567, 0.867783, 0.815866, 0.829422, 0.83233, 0.855686, 0.838475, 0.867819, 0.883889, 0.855821  …  0.848331, 0.827132, 0.84988, 0.839781, 0.857761, 0.838229, 0.848125, 0.797158, 0.833975, 0.870615], KFloat32[-0.0127295, -0.0304893, -0.0904985, 0.0445355, -0.144895, -0.024197, -0.0994711, -0.0862123, -4.30946e-5, -0.0167518  …  -0.0362838, -0.0539902, -0.0869694, -0.0842818, -0.0399579, 0.095863, 0.0143196, -0.0584756, -0.0262519, -0.0513966], 1.0e-12)), Encoder(SelfAttention(Linear3D(KFloat32[0.00896527 -0.0645792 … 0.00241922 0.0233109; -0.00648313 -0.0214259 … -0.0137526 -0.0709136; … ; 0.00666649 0.0373848 … 0.0931021 -0.00359787; -0.0609829 -0.00721878 … 0.0153918 -0.0128817], KFloat32[-0.000628368, -0.00427145, 0.327254, 0.0482903, -0.0101103, 0.025975, -0.0671861, -0.0414595, -0.0326851, -0.0614032  …  0.0467369, 0.118603, 0.609178, -0.0507981, -0.0659099, 0.0460094, 0.0249538, -0.234628, -0.0180062, -0.0482316]), Linear3D(KFloat32[0.0284992 -0.00149629 … 0.00437037 0.00443021; -0.0258194 -0.0553238 … -0.0204206 -0.0106635; … ; -0.0752375 -0.0526777 … 0.00642409 0.0215944; -0.0028737 0.0558756 … -0.0567617 0.0032094], KFloat32[-0.000740833, 0.00732449, 0.000985629, 0.00346116, 0.000438284, 0.0114086, -0.00579523, 0.00672868, -0.00161431, -0.00181688  …  -0.00115321, -0.00153366, -0.00313612, -0.00171685, 0.000849927, -0.00669145, 0.00362517, 0.00852484, -0.00966807, 0.00172212]), Linear3D(KFloat32[-0.00224545 0.0346811 … -0.0485482 0.023758; -0.044981 -0.00812104 … 0.00525021 -0.01517; … ; -0.00805686 0.0270245 … -0.0067748 -0.0343746; 0.0288855 -0.0217475 … 0.0378505 -0.0925177], KFloat32[-0.0197343, 0.0140509, 0.00234974, -0.0342493, 0.00541747, -0.0124649, -0.0159743, -0.00417507, 0.000201565, -0.00477054  …  0.0157086, 0.0382228, -0.0695548, -0.169293, 0.00342002, 0.0159958, 0.025315, 0.0128415, 0.0901567, -0.04497]), Linear3D(KFloat32[-0.0539293 0.0285253 … 0.0495934 -0.0264026; 0.0103668 -0.000212049 … 0.0325584 -0.00972839; … ; -0.013565 -0.0137611 … 0.0168915 0.0110641; 0.019315 -0.0283956 … 0.0556689 0.0158965], KFloat32[0.00350443, -0.0573286, -0.061193, -0.0371589, 0.0362978, -0.011181, -0.0321375, 0.0405997, -0.0192735, -0.046595  …  0.0392835, -0.0137947, -0.00819177, -0.0010802, -0.00391018, -0.0133678, 0.0951157, -0.0907368, -0.0410951, -0.0382667]), 12, 256, 768, 64, 8, 0.1, 0.1), LayerNormalization(KFloat32[0.881622, 0.876344, 0.793083, 0.761243, 0.929209, 0.848779, 0.861767, 0.862238, 0.85239, 0.841594  …  0.820177, 0.695762, 0.861209, 0.812049, 0.795916, 0.857464, 0.828999, 0.768391, 0.796986, 0.887361], KFloat32[-0.029431, 0.0463614, 0.0540025, -0.0473204, 0.168531, -0.0262122, 0.180307, 0.104632, -0.0197971, -0.102608  …  -0.0678709, -0.0596223, -0.0339695, 0.0124973, 6.53852e-6, -0.0386522, -0.23865, 0.108274, 0.0248636, -0.0158561], 1.0e-12), FeedForward(Dense(Linear3D(KFloat32[0.0484802 0.00256627 … -0.0751743 0.0176594; -0.10155 -0.0617603 … 0.00568311 -0.0040159; … ; 0.0483959 0.00350528 … 0.0571018 0.0211979; -0.0367129 -0.0303013 … 0.0136973 -0.0329795], KFloat32[-0.131688, -0.0892379, -0.0619609, -0.0810045, -0.0866559, -0.15773, -0.0676666, -0.104892, -0.0658851, -0.112908  …  -0.145807, -0.067306, -0.210487, -0.0881774, -0.159211, -0.155998, -0.215355, -0.033194, -0.229607, -0.0633825]), 0.0, gelu), Linear3D(KFloat32[-0.0313085 -0.109376 … 0.00434493 0.0139654; -0.00634838 -0.0383636 … -0.014755 0.0214562; … ; -0.0210953 0.0286534 … 0.119088 0.00249492; -0.043793 -0.0162001 … 0.0472122 0.0114457], KFloat32[-0.0415286, 0.0380192, 0.0558934, -0.0478377, 0.025836, -0.0515832, 0.0506327, 0.0854503, -0.0831816, -0.0146496  …  0.106682, -0.161671, -0.0999822, 0.104364, 0.0592195, -0.000923089, -0.13732, 3.75149e-5, -0.0372628, -0.0522225]), 0.1), LayerNormalization(KFloat32[0.843361, 0.821854, 0.805293, 0.822919, 0.782836, 0.818317, 0.789613, 0.841527, 0.84722, 0.819969  …  0.834522, 0.818159, 0.847986, 0.824122, 0.812012, 0.816405, 0.818372, 0.776499, 0.809847, 0.844429], KFloat32[-0.0321487, -0.0450338, -0.0561477, -0.0175777, -0.172329, -0.0380185, -0.104658, -0.0754746, -0.0290361, -0.0243589  …  -0.0250188, -0.0214711, -0.0438003, -0.0897225, -0.041106, 0.0152172, 0.0289193, -0.0823017, -0.0584448, -0.0382355], 1.0e-12)), Encoder(SelfAttention(Linear3D(KFloat32[0.0388164 0.0607835 … -0.0185454 0.00434548; 0.0271046 -0.00956456 … -0.0104098 0.0527196; … ; 0.00510517 -0.0323042 … -0.0457763 -0.0116207; -0.0208955 0.0673651 … 0.0122475 0.0220666], KFloat32[-0.258701, -0.247156, 0.0246178, 0.00929726, 0.190205, -0.111973, 0.438829, 0.139277, 0.51057, -0.20131  …  0.283882, -0.0612375, -0.0172269, -0.306779, 0.113691, -0.181302, 0.20953, -0.0201566, -0.0364933, -0.183534]), Linear3D(KFloat32[-0.00756848 -0.00338899 … 0.0153478 -0.018448; 0.0253631 -0.0545311 … -0.00652499 0.0231719; … ; -0.0174846 -0.0528538 … 0.0174606 -0.0827516; -0.0691879 0.00997738 … 0.0458532 0.0261125], KFloat32[4.84552e-5, 0.00345531, -0.00175954, -0.00612634, -0.00043859, -0.010083, -0.00324462, -0.0176641, -0.00546734, -0.00456329  …  0.00725202, 0.00075686, 0.00991323, -0.00775879, 0.00407166, -0.000156444, 0.00167567, 0.000527598, -0.00631207, 0.00110785]), Linear3D(KFloat32[-0.00407655 -0.0315303 … -0.0013821 -0.0282779; -0.0304675 -0.00780581 … 0.00791398 0.030217; … ; 0.0219425 0.0258201 … 0.0142248 -0.00987703; -0.00699568 -0.035392 … -0.0445342 -0.0111146], KFloat32[0.024169, 0.00723856, -0.03573, 0.00280749, -0.026287, 0.00806716, -0.00525383, -0.0275214, 0.0373534, 0.0383706  …  0.0200117, -0.00722892, 0.0336776, -0.0215702, 0.0505285, 0.0265787, -0.00585558, -0.018169, -0.00959817, 0.0118585]), Linear3D(KFloat32[0.0362245 -0.0137367 … 0.0115601 -0.0013892; 0.0445233 0.0170755 … -0.0127798 -0.0228715; … ; 0.0325878 0.0313821 … 0.0302142 0.0159314; -0.002754 -0.00911497 … -0.00327504 0.0411723], KFloat32[0.0131591, -0.0557545, -0.0630387, 0.00300814, 0.027906, -0.0747646, -0.0230393, 0.00505034, -0.0158796, -0.0362536  …  0.0351204, 0.0502527, 0.00152056, 0.0236999, -0.108068, 0.0417671, 0.0916544, -0.0332293, 0.0206049, -0.006822]), 12, 256, 768, 64, 8, 0.1, 0.1), LayerNormalization(KFloat32[0.898273, 0.89862, 0.815009, 0.783865, 0.904057, 0.857047, 0.899352, 0.866709, 0.86916, 0.867383  …  0.838021, 0.762782, 0.835373, 0.813659, 0.830329, 0.848087, 0.847626, 0.784163, 0.806893, 0.888964], KFloat32[-0.062369, 0.043276, 0.089764, -0.151134, -0.054906, -0.0103397, 0.260581, 0.097807, -0.0870408, -0.181704  …  -0.0812599, -0.135191, -0.0470538, -0.0741931, 0.0915758, -0.128805, -0.250249, -0.00319491, -0.0889768, -0.0569763], 1.0e-12), FeedForward(Dense(Linear3D(KFloat32[0.00463911 0.0240151 … -0.0112991 0.0802082; -0.0136311 0.00828379 … 0.0590412 0.0246431; … ; 0.00985721 0.0859319 … -0.00196432 0.0228469; 0.00299473 -0.0308479 … 0.0706927 -0.0199022], KFloat32[-0.0791842, -0.172961, -0.106286, -0.115164, -0.131361, -0.020113, -0.122497, 0.0145773, -0.12476, -0.125177  …  -0.124953, -0.101673, -0.0986047, -0.0852359, -0.0423655, -0.0538767, -0.124611, -0.127897, -0.178045, -0.180434]), 0.0, gelu), Linear3D(KFloat32[-0.0301868 0.0325206 … 0.0209579 -0.0130637; -0.0101236 0.0209914 … 0.0311992 0.00960506; … ; -0.029839 4.30147e-6 … -0.0387052 -0.0431132; -0.0170483 0.00593194 … 0.0385936 0.0212801], KFloat32[-0.0912109, -0.0451974, 0.0529481, -0.0976375, 0.116918, 0.0701795, 0.0488588, 0.133395, -0.0716097, -0.00940306  …  0.070759, -0.137781, -0.0952152, 0.00487934, 0.0521733, -0.0662693, -0.0787949, -0.0418508, -0.0363682, -0.0731502]), 0.1), LayerNormalization(KFloat32[0.849397, 0.853644, 0.830864, 0.843624, 0.824219, 0.846206, 0.804246, 0.84158, 0.853735, 0.833238  …  0.853754, 0.834877, 0.849305, 0.852848, 0.850947, 0.820988, 0.805521, 0.847078, 0.826297, 0.858821], KFloat32[-0.0629248, -0.0711301, -0.0580446, -0.00312697, -0.107833, -0.022097, -0.133269, -0.0813468, 0.000947313, 0.0241789  …  -0.00692475, 0.00971595, -0.0569504, -0.0772161, -0.0526174, 0.0480974, 0.0344029, -0.0474425, -0.0228142, -0.0208941], 1.0e-12)), Encoder(SelfAttention(Linear3D(KFloat32[0.0715192 0.0228642 … 0.0018771 0.010974; -0.0430378 -0.0591152 … -0.0579609 0.0189026; … ; 0.0232648 0.0633709 … 0.0260652 -0.0304687; -0.0721848 -0.000211997 … -0.0275993 -0.0220579], KFloat32[-0.155194, 0.419576, -0.329738, -0.0296491, -0.2639, -0.0124473, 0.287049, 0.292418, -0.0906055, 0.412139  …  -0.0767008, -0.0480644, -0.0386395, -0.140976, 0.07246, 0.101828, 0.0485731, -0.00439472, -0.0466305, -0.0198388]), Linear3D(KFloat32[-0.000647281 -0.000743133 … 0.0523733 0.01391; -0.00523023 -0.0537896 … -0.0609509 0.0205155; … ; -0.0436089 -0.00546788 … -0.0625646 -0.0249456; 0.0308901 0.0213209 … 0.0212718 -0.0124143], KFloat32[-0.0051125, 0.00834389, 0.0037761, 0.00929363, -0.00419275, -0.00804659, 0.00358508, 0.005272, -0.00335078, 0.00376932  …  -0.014526, -0.00830259, -0.00344179, -0.0117241, -0.0126948, 0.0118298, -0.00238022, 0.00463774, -0.00644765, 0.00038889]), Linear3D(KFloat32[-0.0139388 0.0235448 … 0.0209995 0.0660312; -0.0402653 0.0260353 … 0.0423156 -0.0111658; … ; 0.0194042 0.00421942 … -0.0367257 -0.0495049; -0.00925736 0.0475911 … 0.00384488 0.01306], KFloat32[-0.0432247, -0.0200408, 0.00323672, -0.000899292, 0.0509734, -0.0313252, -0.0391894, 0.00579211, 0.00121049, 0.00186705  …  -0.00502531, 0.0111507, 0.00790766, -0.0207992, -0.0289624, -0.0109599, 0.0239687, -0.0129929, -0.0178019, -0.0167052]), Linear3D(KFloat32[0.000760623 0.0110061 … 0.0160549 -0.0246738; 0.00244885 -0.0099471 … -0.00364059 -0.0280702; … ; 0.00147388 -0.036718 … -0.00477767 0.0155861; 0.0303591 -0.0412512 … -0.010384 0.0109737], KFloat32[0.0212502, 0.0493468, -0.0671334, -0.0158188, 0.0119973, -0.0537279, -0.0296846, -0.0607311, 0.0258002, -0.0373451  …  0.0507148, 0.0449629, -0.0357749, 0.0241432, -0.0840274, 0.00712703, 0.106785, -0.0639751, -0.060085, -0.0649474]), 12, 256, 768, 64, 8, 0.1, 0.1), LayerNormalization(KFloat32[0.857594, 0.829514, 0.798337, 0.773125, 0.896423, 0.83883, 0.864829, 0.865001, 0.841054, 0.838779  …  0.830951, 0.775382, 0.818042, 0.775587, 0.801966, 0.828543, 0.823921, 0.7732, 0.808528, 0.862775], KFloat32[-0.094489, -0.065302, 0.0724158, -0.106003, 0.0923605, 0.0956846, 0.173821, 0.176316, -0.12405, -0.172564  …  -0.090833, -0.160664, 0.0624419, -0.139933, 0.0653048, -0.166307, -0.175211, -0.0546064, -0.0749023, -0.0699072], 1.0e-12), FeedForward(Dense(Linear3D(KFloat32[0.0487061 -0.0118265 … 0.0137912 -0.0579595; 0.00784316 -0.0229906 … -0.00967215 -0.0346685; … ; -0.0039208 0.0609452 … 0.0861011 -0.0546284; 0.0227673 -0.0353637 … 0.0426296 -0.00971514], KFloat32[-0.108791, -0.00584135, -0.0941776, -0.105397, -0.143535, -0.119502, -0.127852, -0.124293, -0.0540293, -0.112188  …  -0.100539, -0.0836761, -0.0544823, -0.108502, -0.101315, -0.107486, -0.15039, -0.0250884, -0.0697383, -0.096078]), 0.0, gelu), Linear3D(KFloat32[0.0506214 0.00763107 … -0.0210749 -0.0381577; 0.0100306 0.0272092 … -0.0173806 -0.0254556; … ; 0.00549996 0.00046856 … 0.0432146 -0.0188744; -0.0273861 -0.00706908 … 0.0187039 0.00153592], KFloat32[-0.0883273, -0.0204082, -0.0364724, -0.0830269, 0.0955854, 0.0626112, 0.06784, 0.0652947, -0.0507099, -0.0689095  …  0.0548709, -0.0256723, 0.0332503, -0.0447851, 0.137993, -0.00790642, -0.0567217, 0.0317385, 0.0205113, 0.00850145]), 0.1), LayerNormalization(KFloat32[0.835835, 0.840334, 0.777607, 0.812151, 0.786886, 0.824662, 0.783895, 0.783048, 0.823728, 0.79853  …  0.820425, 0.797225, 0.818381, 0.820798, 0.80959, 0.791895, 0.794933, 0.804462, 0.822817, 0.837176], KFloat32[-0.038307, -0.0210817, -0.0758258, -0.00878752, -0.0992655, -0.0640754, -0.115336, -0.0920016, 0.0066929, 0.013534  …  -0.0235416, 0.041162, -0.0602311, -0.022866, -0.0730742, 0.0513098, 0.0323097, -0.0140102, -0.0248398, -0.0272981], 1.0e-12)), Encoder(SelfAttention(Linear3D(KFloat32[-0.00846596 0.0330938 … -0.0113931 0.0117777; -0.00904127 0.0272398 … 0.0501928 0.0319319; … ; -0.0382526 0.00629995 … 0.028472 -0.109675; 0.0681221 -0.0304994 … 0.0166265 -0.0336947], KFloat32[-0.0400402, -0.027469, -0.0458579, -0.0078563, -0.0557169, -0.0605462, -0.00215188, 0.0686311, -0.0223629, -0.0287055  …  -0.00639185, 0.0471284, -0.0203689, 0.00467536, -0.0199282, -0.132879, -0.0467959, -0.279584, 0.00949014, -0.0180372]), Linear3D(KFloat32[-0.0415346 -0.0212556 … -0.0274155 0.0674779; 0.0583549 -0.0505614 … 0.0257912 0.0631988; … ; -0.0764208 0.00300912 … 0.0244161 -0.0650166; -0.0259351 -0.08072 … -0.0778932 -0.0115442], KFloat32[-0.00693039, 0.0085596, 0.00670912, -0.00524555, -0.000174295, 0.0050662, 0.00418257, 0.00185772, -0.00101529, 0.00119562  …  -0.00107656, 0.00569142, -0.00113544, 0.00514889, 0.00990483, -0.0101674, -0.00164622, -0.002075, -0.00216864, 0.0041604]), Linear3D(KFloat32[0.0314202 0.0414027 … -0.00861006 -0.0924014; 0.0505874 0.0663632 … -0.00320221 0.083238; … ; -0.00500523 -0.00505953 … -0.0992578 0.0251702; -0.0905331 -0.0179972 … -0.0114624 -0.0173277], KFloat32[-0.0154709, -0.0121216, -0.00774924, -0.00259434, -0.000927358, 0.0252472, -0.00685565, 0.0100812, 0.0178414, -0.0328056  …  0.0342542, -0.0247867, -0.0126417, 0.0399945, -0.00207709, 0.00777719, 0.0566921, 0.0223318, -0.00432853, -0.0143106]), Linear3D(KFloat32[-0.0374263 -0.0337741 … 0.0236563 -0.00476478; 0.0336529 -0.0223491 … 0.00123772 -0.0466988; … ; -0.01775 -0.0061594 … 0.0353524 -0.0210365; 0.015367 -0.0795335 … -0.0357039 0.00336914], KFloat32[0.0225386, 0.014959, 0.0542274, 0.0298167, 0.0159205, 0.0176375, 0.0244215, -0.0689132, 0.0297022, 0.0566074  …  -0.00590228, -0.00464236, -0.0843023, 0.0275345, -0.017384, -0.0811144, -0.00279512, 0.0370788, -0.0891135, -0.132582]), 12, 256, 768, 64, 8, 0.1, 0.1), LayerNormalization(KFloat32[0.857921, 0.822616, 0.845871, 0.775152, 0.886226, 0.833409, 0.832226, 0.879852, 0.832022, 0.836092  …  0.828863, 0.833162, 0.84515, 0.795485, 0.817476, 0.8566, 0.825461, 0.813878, 0.819559, 0.842449], KFloat32[-0.109651, -0.0417605, 0.0362388, -0.0636018, 0.158169, 0.023189, 0.0174584, 0.150409, -0.109643, -0.0316043  …  -0.117351, -0.0167175, 0.116653, -0.0667549, 0.00410766, -0.127346, -0.0299201, -0.078612, -0.050465, -0.0595335], 1.0e-12), FeedForward(Dense(Linear3D(KFloat32[-0.0613653 -0.0327641 … 0.0240241 0.0818172; 0.0733787 0.0279311 … 0.0178987 -0.0321581; … ; -0.0278835 -0.0285588 … 0.00333484 0.0334873; -0.0295649 0.0164237 … 0.0065991 0.0520008], KFloat32[-0.144958, -0.063249, -0.145886, -0.152788, -0.103942, 0.0448957, -0.102309, -0.122734, -0.0889336, -0.0325621  …  -0.12772, -0.0674632, -0.0898458, -0.167649, -0.136661, -0.126413, -0.115664, -0.0492113, -0.112593, -0.103311]), 0.0, gelu), Linear3D(KFloat32[0.0229222 -0.0331473 … 0.0166346 -0.00178451; 0.0304881 0.0276935 … -0.00662481 -0.00486354; … ; -0.0501759 -0.0102409 … 0.00936838 -0.0614161; -0.0296648 -0.0155245 … 0.0226335 0.022399], KFloat32[-0.0186336, 0.111696, -0.0298404, -0.0339797, 0.103479, 0.1251, 0.0886984, 0.0852141, -0.0823288, -0.116562  …  -0.0102634, -0.0301755, 0.114603, -0.0972063, 0.172549, -0.0835289, -0.093207, 0.0334924, 0.0381817, 0.00640153]), 0.1), LayerNormalization(KFloat32[0.835903, 0.866589, 0.79692, 0.818541, 0.836397, 0.832215, 0.812946, 0.797133, 0.83847, 0.814139  …  0.82734, 0.814174, 0.863259, 0.844337, 0.83415, 0.806062, 0.81293, 0.827399, 0.82871, 0.839287], KFloat32[0.0184391, -0.0192821, -0.0826565, -0.00922328, -0.0757732, -0.0849442, -0.048473, -0.105045, -0.00103142, -0.0221743  …  -0.015187, -0.0145156, -0.0608686, -0.00666481, -0.0518845, 0.0409791, 0.00173667, -0.0298248, -0.0567055, -0.0422205], 1.0e-12)), Encoder(SelfAttention(Linear3D(KFloat32[-0.0619166 -0.00119244 … 0.0456402 -0.0570816; -0.0930355 -0.0264885 … -0.0705614 0.01517; … ; 0.0546556 0.00179513 … -0.0823758 0.00746658; -0.0446889 0.0322623 … 0.0442187 -0.043325], KFloat32[0.0452436, 0.12612, 0.215809, 0.0964397, -0.209281, 0.234274, -0.358509, 0.0569348, -0.435072, 0.122037  …  0.0714506, 0.0529339, -0.0835622, 0.287433, 0.096056, -0.154332, -0.197719, 0.259118, 0.181829, 0.179459]), Linear3D(KFloat32[0.0214798 -0.0154309 … 0.0707313 -0.0289973; -0.0195324 -0.0528643 … 0.0509169 -0.00265497; … ; 0.00124895 -0.0267105 … -0.0224189 0.0897006; 0.0471457 0.0482504 … 0.000925252 -0.0569181], KFloat32[-0.000784431, -0.000662038, -0.000104968, 0.00119019, -0.00179686, -0.00275173, -0.000776681, 0.00258377, -0.0225535, -0.0026923  …  -0.00278521, -0.0119263, 0.00207065, 0.00312377, 0.003316, 0.0100499, 0.000155324, -0.00759888, 0.00313334, -0.00141902]), Linear3D(KFloat32[0.0372145 -0.0383612 … 0.0173978 -0.0509606; -0.0207322 -0.0274311 … 0.0096083 -0.0291448; … ; -0.0470994 -0.0079893 … 0.120101 -0.0107127; 0.0115737 -0.0166647 … 0.00340503 -0.0572358], KFloat32[0.0193677, -0.00475728, 0.00103245, -0.000514294, -0.00533322, 0.00755855, -0.0113693, -0.00703755, 0.00471488, -0.00458319  …  0.000203516, -0.0191094, 0.00400823, 0.0029437, -0.0166149, -0.0119975, -0.00789729, 0.00339477, -0.0155796, 0.0160248]), Linear3D(KFloat32[0.045823 -0.0371547 … -0.0448876 -0.0306689; 0.0363964 -0.0177871 … -0.00241771 -0.0218722; … ; -0.016208 -0.0752646 … 0.0865968 0.00749161; -0.0973066 0.0104237 … -0.0553478 0.0267396], KFloat32[0.0267571, 0.00660395, -0.000254276, 0.0233143, -0.00932865, -0.0577665, 0.000551676, -0.0176162, 0.0469589, 0.0399452  …  0.0412048, 0.0155685, -0.060947, 0.0124328, -0.0328537, 0.0127199, 0.0336913, 0.00536097, -0.0532224, -0.0549582]), 12, 256, 768, 64, 8, 0.1, 0.1), LayerNormalization(KFloat32[0.856396, 0.805148, 0.852187, 0.821726, 0.855675, 0.826754, 0.862809, 0.849582, 0.84667, 0.868172  …  0.86737, 0.866919, 0.796145, 0.782494, 0.831994, 0.843429, 0.855304, 0.840911, 0.839397, 0.835407], KFloat32[-0.0235748, 0.0210148, -0.0239803, -0.0273365, 0.103877, 0.0460757, 0.0272399, -0.0187669, -0.110665, -0.122998  …  -0.057848, 0.0302805, 0.101279, -0.122537, -0.00456612, -0.14115, -0.0452224, -0.0381125, 0.0463717, 0.0103002], 1.0e-12), FeedForward(Dense(Linear3D(KFloat32[0.0565045 0.0685513 … 0.0343543 -0.0173157; -0.00684705 -0.014927 … 0.00669027 0.00147132; … ; -0.00764741 0.0488817 … 0.066738 0.0545332; -0.0201364 -0.0664263 … 0.0633109 -0.00316789], KFloat32[-0.104775, -0.0592018, -0.0553662, -0.0901755, -0.0934389, -0.0730384, -0.0782455, -0.0777856, -0.0677402, -0.058786  …  -0.0829975, -0.0547986, -0.0440764, -0.0652548, -0.0593083, -0.0217005, -0.215905, -0.110135, -0.049059, -0.124741]), 0.0, gelu), Linear3D(KFloat32[-0.0112698 0.0579318 … -0.0234224 0.0264506; 0.00958657 -0.0164245 … -0.0243531 -0.00394748; … ; -0.00766256 0.0475985 … -0.0279663 0.0825156; -0.0333418 -0.0144371 … -0.0194319 0.00789958], KFloat32[-0.0659266, 0.0426318, 0.0124363, 0.00275178, 0.0541394, 0.0475896, 0.0217408, -0.0549929, -0.0204092, -0.0990393  …  0.0210347, -0.047856, -0.0472294, 0.0161939, 0.038242, -0.00222726, -0.0653064, 0.0515407, 0.0283886, 0.0443353]), 0.1), LayerNormalization(KFloat32[0.639673, 0.641097, 0.664543, 0.613195, 0.636797, 0.630183, 0.660904, 0.634321, 0.630064, 0.617583  …  0.643693, 0.643947, 0.615546, 0.602717, 0.633976, 0.63567, 0.632704, 0.614622, 0.616857, 0.628513], KFloat32[0.0234814, -0.0171893, 0.0682789, 0.0113935, -0.0423639, -0.00102498, 0.00307995, -0.0218328, 0.0304409, -0.0165133  …  0.0726081, -0.0240185, -0.0640335, -0.0142857, 0.0079084, -0.0481675, 0.00325446, 0.00707768, 0.0273389, -0.0319619], 1.0e-12))], KnetArray{Float32,N} where N), Pooler(Linear(KFloat32[0.00756929 -0.0378011 … -0.00601825 0.0181501; 0.0120643 0.00841361 … -0.00788617 0.0163892; … ; 0.00878611 -0.00200486 … -0.00652385 -0.0116807; 0.0312671 0.0705206 … 0.0485129 0.0105209], KFloat32[-0.0341992, -0.00248577, 0.0567474, 0.0191413, -0.00750162, -2.86559e-5, 0.0219396, 0.00680262, 0.0375916, -0.0971814  …  -0.000831233, 0.0169847, -0.0336673, 0.0465352, -0.0112774, 0.0522394, 0.010307, 0.016399, -0.0166942, 0.0473651])), NSPHead(Linear(KFloat32[-0.0210782 -0.0298697 … -0.0291932 0.00926572; 0.0132354 0.0372161 … 0.035094 -0.0266885], KFloat32[0.0243201, -0.00920289])), MLMHead(Dense(Linear3D(KFloat32[0.351745 0.00516173 … -0.0392233 0.0286469; 0.0128208 0.275913 … -0.0175868 0.00771347; … ; 0.00793999 -0.0442648 … 0.193474 -0.038447; -0.0716141 0.0548792 … -0.0733149 0.366621], KFloat32[0.0701037, 0.0920184, 0.0455005, 0.0297336, 0.0553089, 0.0397923, 0.0403409, 0.0252468, 0.0331813, 0.125963  …  0.0926675, 0.0590998, 0.0571389, 0.0316408, 0.0347013, 0.0191983, 0.04481, 0.0468275, 0.0579494, 0.0444784]), 0.0, gelu), LayerNormalization(KFloat32[2.51703, 2.70333, 2.65458, 2.77146, 2.49534, 2.73382, 2.61742, 2.76009, 2.73197, 2.63018  …  2.61433, 2.37725, 2.56413, 2.76293, 2.54514, 2.72719, 2.37566, 2.58617, 2.73862, 2.98347], KFloat32[-0.50624, 0.0924518, 0.0338049, 0.117695, -0.499471, 0.0730249, 0.149177, 0.0629309, -0.128307, 0.129261  …  0.138374, -0.153489, -0.252505, 0.11565, -0.203978, -0.183053, -0.226679, -0.210253, 0.0107535, 0.190375], 1.0e-12), Linear3D(KFloat32[0.00618323 -0.0568268 … -0.0306487 -0.0130648; -0.0169476 -0.0541772 … -0.0289911 -0.0109822; … ; -0.0499978 -0.0468287 … -0.0284732 -0.0201913; -0.0177459 -0.0793887 … -0.0283612 0.0571289], KFloat32[-0.90455, -0.914618, -0.930487, -0.959663, -0.924124, -0.870831, -0.91987, -0.883486, -0.904187, -0.976599  …  -1.31019, -1.28476, -1.38072, -1.38533, -1.2995, -1.31533, -1.40002, -1.28598, -1.31441, -1.12316])))"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = load_from_torch_pretraining(model, config.num_encoder, config.atype, torch_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.5112249f0"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model(dtrn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2×768 Array{Float32,2}:\n",
       " -0.0210782  -0.0298697  -0.0114933   0.0265236  …  -0.0291932   0.00926572\n",
       "  0.0132354   0.0372161   0.0114068  -0.0150693      0.035094   -0.0266885 "
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch_model[\"cls.seq_relationship.weight\"][:cpu]()[:numpy]()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2×768 Array{Float32,2}:\n",
       "  0.0132354   0.0372161   0.0114068  -0.0150693  …   0.035094   -0.0266885 \n",
       " -0.0210782  -0.0298697  -0.0114933   0.0265236     -0.0291932   0.00926572"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch_model[\"cls.seq_relationship.weight\"][:cpu]()[:numpy]()[end:-1:1,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11.019350194610766"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Knet.log(30522) + Knet.log(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "true"
      ]
     },
     "execution_count": 229,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check if there is weight sharing\n",
    "torch_model[\"cls.predictions.decoder.weight\"][:cpu]()[:numpy]() == torch_model[\"bert.embeddings.word_embeddings.weight\"][:cpu]()[:numpy]()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Unimportant stuff after this part"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  1.480823 seconds (3.64 M allocations: 163.294 MiB, 4.27% gc time)\n"
     ]
    }
   ],
   "source": [
    "#=\n",
    "asd = randn(2,3,4,5)\n",
    "dsa = randn(3,2,4,5)\n",
    "@time begin\n",
    "    qwe = zeros(2,2,4,5)\n",
    "    for i in 1:5\n",
    "        qwe[:, :, :, i] = bmm(asd[:, :, :, i], dsa[:, :, :, i])\n",
    "    end\n",
    "end\n",
    "=#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  0.064232 seconds (184.66 k allocations: 9.384 MiB)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "2×2×4×5 Array{Float64,4}:\n",
       "[:, :, 1, 1] =\n",
       "  0.115702   0.84845\n",
       " -1.28604   -1.00778\n",
       "\n",
       "[:, :, 2, 1] =\n",
       " -0.117182   0.363062\n",
       "  4.44089   -0.717041\n",
       "\n",
       "[:, :, 3, 1] =\n",
       " -0.733266  -1.53797\n",
       " -1.17601   -1.41673\n",
       "\n",
       "[:, :, 4, 1] =\n",
       " 0.791245   -0.0872022\n",
       " 0.0146814   0.0400762\n",
       "\n",
       "[:, :, 1, 2] =\n",
       " -1.13423  0.766976\n",
       "  2.13047  0.910996\n",
       "\n",
       "[:, :, 2, 2] =\n",
       " 1.41911    2.82969 \n",
       " 0.716323  -0.676642\n",
       "\n",
       "[:, :, 3, 2] =\n",
       "  1.1102      -1.53823\n",
       " -0.00522836   1.55034\n",
       "\n",
       "[:, :, 4, 2] =\n",
       " -3.75843   0.621332\n",
       "  1.44112  -1.03046 \n",
       "\n",
       "[:, :, 1, 3] =\n",
       " -0.760699  1.55247\n",
       "  0.272704  3.22196\n",
       "\n",
       "[:, :, 2, 3] =\n",
       " 0.12309  2.14021\n",
       " 1.64758  2.65152\n",
       "\n",
       "[:, :, 3, 3] =\n",
       " -0.815294   1.14375 \n",
       " -0.615639  -0.989944\n",
       "\n",
       "[:, :, 4, 3] =\n",
       " -1.07816     -1.03071 \n",
       " -0.00645643   0.314534\n",
       "\n",
       "[:, :, 1, 4] =\n",
       " -0.366951  -0.496234\n",
       "  2.55274    0.690053\n",
       "\n",
       "[:, :, 2, 4] =\n",
       " -0.176301  -0.385411\n",
       " -0.480657   1.25897 \n",
       "\n",
       "[:, :, 3, 4] =\n",
       "  2.53924   2.71935\n",
       " -2.18906  -1.85169\n",
       "\n",
       "[:, :, 4, 4] =\n",
       " -0.585549  -0.480815\n",
       "  1.91477   -0.159747\n",
       "\n",
       "[:, :, 1, 5] =\n",
       "  0.554726  -0.869307\n",
       " -1.44895    1.90506 \n",
       "\n",
       "[:, :, 2, 5] =\n",
       " -0.20791   0.686798\n",
       " -0.630707  2.3288  \n",
       "\n",
       "[:, :, 3, 5] =\n",
       " 0.596997  -0.0937985\n",
       " 0.700099  -0.213198 \n",
       "\n",
       "[:, :, 4, 5] =\n",
       " 0.216494  0.526591\n",
       " 5.54772   2.13989 "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# @time qwe2 = reshape(bmm(reshape(asd, (2,3,20)), reshape(dsa, (3,2,20))), (2,2,4,5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "true"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# qwe == qwe2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# I change this function in order to ignore the indexes with labels -1 in nll function. We don't need this part anymore.\n",
    "#=\n",
    "function findindices(y,a::AbstractArray{<:Integer}; dims=1)\n",
    "    n = length(a[a.!=-1]) # ignore indexes with -1\n",
    "    indices = Vector{Int}(undef,n)\n",
    "    if dims == 1                   # instances in first dimension\n",
    "        y1 = size(y,1)\n",
    "        y2 = div(length(y),y1)\n",
    "        #if n != y2; throw(DimensionMismatch()); end\n",
    "        @inbounds for j=1:n\n",
    "            if a[j] != -1\n",
    "                indices[j] = (j-1)*y1 + a[j]\n",
    "            end\n",
    "        end\n",
    "    elseif dims == 2               # instances in last dimension\n",
    "        y2 = size(y,ndims(y))\n",
    "        y1 = div(length(y),y2)\n",
    "        #if n != y1; throw(DimensionMismatch()); end\n",
    "        @inbounds for j=1:n\n",
    "            if a[j] != -1\n",
    "                indices[j] = (a[j]-1)*y1 + j\n",
    "            end\n",
    "        end\n",
    "    else\n",
    "        error(\"findindices only supports dims = 1 or 2\")\n",
    "    end\n",
    "    return indices\n",
    "end\n",
    "=#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dict{Any,Any} with 4 entries:\n",
       "  \"dense.weight\" => PyObject tensor([[ 0.0263, -0.0085,  0.0108,  ..., -0.0319,…\n",
       "  \"dense.bias\"   => PyObject tensor([ 5.0000e-03, -9.3551e-03,  1.1515e-02, -1.…\n",
       "  \"final.weight\" => PyObject tensor([[ 0.0118, -0.0328,  0.0302,  ...,  0.0087,…\n",
       "  \"final.bias\"   => PyObject tensor([ 0.0352, -0.0279])"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "linear_model = torch.load(\"/scratch/users/omutlu/hyperpartisan/data_20181122_big/linear_model.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "768×1 Array{Float32,2}:\n",
       "   1.0\n",
       "   2.0\n",
       "   3.0\n",
       "   4.0\n",
       "   5.0\n",
       "   6.0\n",
       "   7.0\n",
       "   8.0\n",
       "   9.0\n",
       "  10.0\n",
       "  11.0\n",
       "  12.0\n",
       "  13.0\n",
       "   ⋮  \n",
       " 757.0\n",
       " 758.0\n",
       " 759.0\n",
       " 760.0\n",
       " 761.0\n",
       " 762.0\n",
       " 763.0\n",
       " 764.0\n",
       " 765.0\n",
       " 766.0\n",
       " 767.0\n",
       " 768.0"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mutable struct Asd <: Layer\n",
    "    dense::Linear\n",
    "    final::Linear\n",
    "end\n",
    "\n",
    "Asd() = Asd(Linear(768,768), Linear(768,2))\n",
    "\n",
    "function (a::Asd)(x)\n",
    "    x = a.dense(x)\n",
    "    display(x)\n",
    "    return a.final(x)\n",
    "end\n",
    "\n",
    "Array{Float32}(reshape(collect(1:768).*1.0, (768,1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Asd(Linear(P(Array{Float32,2}(768,768)), P(Array{Float32,1}(768))), Linear(P(Array{Float32,2}(2,768)), P(Array{Float32,1}(2))))"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "asd_model = Asd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2-element Array{Float32,1}:\n",
       "  0.0351763  \n",
       " -0.027943168"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "asd_model.dense.w = linear_model[\"dense.weight\"][:cpu]()[:numpy]()\n",
    "asd_model.dense.b = linear_model[\"dense.bias\"][:cpu]()[:numpy]()\n",
    "asd_model.final.w = linear_model[\"final.weight\"][:cpu]()[:numpy]()\n",
    "asd_model.final.b = linear_model[\"final.bias\"][:cpu]()[:numpy]()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "768×1 Array{Float32,2}:\n",
       " -301.81982 \n",
       "  338.263   \n",
       " -545.907   \n",
       " -253.97137 \n",
       "   69.617035\n",
       " -321.05923 \n",
       "  208.37355 \n",
       " -365.06345 \n",
       "   41.59743 \n",
       " -103.13887 \n",
       "  416.80682 \n",
       "  133.91138 \n",
       " -206.91249 \n",
       "    ⋮       \n",
       "   26.830832\n",
       "  212.42296 \n",
       " -206.62547 \n",
       "  -16.682762\n",
       "  315.7369  \n",
       " -324.698   \n",
       "  -98.1218  \n",
       "  194.63629 \n",
       "   80.866425\n",
       "  138.81798 \n",
       "  172.95903 \n",
       " -297.8899  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "2×1 Array{Float32,2}:\n",
       " -232.30048 \n",
       "   20.705103"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "asd_model(Array{Float32}(reshape(collect(1:768).*1.0, (768,1))))\n",
    "# EVERY LINEAR IS OK -> 3D dene!!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dict{Any,Any} with 1 entry:\n",
       "  \"word_embeddings.weight\" => PyObject tensor([[-0.3275, -1.8525,  0.4429, -0.5…"
      ]
     },
     "execution_count": 196,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embed_model = torch.load(\"/scratch/users/omutlu/hyperpartisan/data_20181122_big/embed_model.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10×64 Array{Float32,2}:\n",
       " -0.327533    0.17811   -0.504147   …   0.266249   0.948373   0.454536 \n",
       " -1.8525     -1.02624    0.730247       1.22722   -0.226292   0.698327 \n",
       "  0.442899    1.36549   -0.0350884     -0.705415   1.63154   -1.31326  \n",
       " -0.565772   -0.583261  -0.231603       0.734134  -0.922967   0.946676 \n",
       " -0.739279    1.10309    1.32591       -1.47574    0.118688   0.0749492\n",
       "  1.25015     0.456525   0.169865   …  -0.691988   1.15282   -0.472395 \n",
       "  0.0826522   0.448301  -0.167629       1.03884   -0.976023  -0.219711 \n",
       "  0.178917   -0.674799  -1.07465       -0.919094   0.728958  -0.591924 \n",
       " -0.360179   -0.685848   2.03066        1.02952   -0.555284   0.472736 \n",
       " -0.44771     1.54605   -0.573169       0.559609   0.045501  -1.58461  "
      ]
     },
     "execution_count": 197,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "asd_model = Embedding(64,10)\n",
    "asd_model.w = permutedims(embed_model[\"word_embeddings.weight\"][:cpu]()[:numpy](), (2,1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "64×10 Array{Float32,2}:\n",
       " -0.327533   -1.8525       0.442899   …   0.178917   -0.360179  -0.44771  \n",
       "  0.17811    -1.02624      1.36549       -0.674799   -0.685848   1.54605  \n",
       " -0.504147    0.730247    -0.0350884     -1.07465     2.03066   -0.573169 \n",
       "  0.386169   -0.396257     0.848314      -1.46865     1.03246    2.09795  \n",
       " -0.807839    0.548595     0.0551342     -2.4487     -1.45526    0.0600894\n",
       "  0.317731    0.758683     1.92686    …  -2.25161     0.749815   0.592579 \n",
       "  1.91857     0.429779    -0.270597      -0.41556    -0.132503   0.439184 \n",
       "  2.27921    -0.00234205   0.411843      -0.141696    1.09544    0.479317 \n",
       "  2.44634    -0.166409    -1.09532       -0.25081     1.53878   -1.6418   \n",
       "  0.0465411   1.68927     -1.27962       -0.219757   -0.547628   0.022649 \n",
       "  0.198385   -0.30309      1.56886    …  -0.261564   -0.800133  -0.820916 \n",
       " -0.380725    0.131117    -0.345388      -0.787145    0.322748   2.61584  \n",
       " -2.21218    -0.63758     -0.0996587      0.910331    1.38684   -0.136687 \n",
       "  ⋮                                   ⋱                                   \n",
       " -0.32901    -0.770494     0.512024      -0.406462   -1.30555   -0.635841 \n",
       "  0.935165   -0.0205195   -2.45046       -1.02307    -0.249505  -0.63487  \n",
       "  0.195528    1.52472      0.253939       0.123944   -0.525301  -0.239052 \n",
       " -1.34798    -0.737864    -1.00106    …  -1.16006    -0.37065    1.7247   \n",
       "  1.59213     0.79344     -0.860818       0.674576    1.42948   -0.45398  \n",
       "  1.24832     0.0154065   -0.119792       1.45004     0.785695  -0.0868203\n",
       "  0.280714   -0.369093    -0.791205       0.367633   -0.848382   0.449621 \n",
       " -0.848662    2.77524      2.53621        0.0467806   0.725441  -0.328411 \n",
       "  0.123004   -1.31756     -0.209354   …  -0.433657   -0.65018    0.0114383\n",
       "  0.266249    1.22722     -0.705415      -0.919094    1.02952    0.559609 \n",
       "  0.948373   -0.226292     1.63154        0.728958   -0.555284   0.045501 \n",
       "  0.454536    0.698327    -1.31326       -0.591924    0.472736  -1.58461  "
      ]
     },
     "execution_count": 201,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embed_model[\"word_embeddings.weight\"][:cpu]()[:numpy]()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5×2 Array{Int64,2}:\n",
       " 11  44\n",
       " 33  12\n",
       " 16  14\n",
       " 13  11\n",
       " 55  43"
      ]
     },
     "execution_count": 202,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "asd = zeros(Int64, 5,2)\n",
    "asd[:,1] = [11,33,16,13,55]\n",
    "asd[:,2] = [44,12,14,11,43]\n",
    "asd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10×5×2 Array{Float32,3}:\n",
       "[:, :, 1] =\n",
       "  0.198385      0.259268   -0.312076    -2.21218      0.195528\n",
       " -0.30309       0.323448   -0.403771    -0.63758      1.52472 \n",
       "  1.56886      -0.0422995  -0.00417269  -0.0996587    0.253939\n",
       " -0.790627     -0.121948    0.975851     0.251527    -1.90247 \n",
       " -0.000929175  -0.740367    0.00304474   0.939455     0.842466\n",
       " -1.4012        0.452447   -1.46052     -0.128605     1.40901 \n",
       " -1.0822        1.09097     0.510383    -0.00162251   1.40591 \n",
       " -0.261564      0.614126   -1.766        0.910331     0.123944\n",
       " -0.800133     -0.198369   -1.1094       1.38684     -0.525301\n",
       " -0.820916     -1.0173      0.859505    -0.136687    -0.239052\n",
       "\n",
       "[:, :, 2] =\n",
       "  1.51673    -0.380725     0.469571    0.198385      1.03165  \n",
       " -0.210208    0.131117    -0.832091   -0.30309       0.978226 \n",
       "  1.6988     -0.345388    -1.30723     1.56886       1.26134  \n",
       " -0.0121622   0.386055    -0.499968   -0.790627      0.856667 \n",
       " -0.0786632  -0.578785    -0.657167   -0.000929175   0.617462 \n",
       " -0.843933   -0.00765016   0.255106   -1.4012       -0.803283 \n",
       " -0.657063   -0.431888    -0.194098   -1.0822       -0.954424 \n",
       " -0.87596    -0.787145    -0.369441   -0.261564     -0.967158 \n",
       " -0.527348    0.322748    -0.0815277  -0.800133      2.07211  \n",
       "  0.633131    2.61584     -0.85339    -0.820916     -0.0556995"
      ]
     },
     "execution_count": 203,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "asd_model(asd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3×10 Array{Float64,2}:\n",
       "  0.805224   0.0205297  -1.64533   …   0.420204   0.12264   -1.06714\n",
       "  0.268687   1.21235    -0.197448     -1.20452    1.73878   -1.59637\n",
       " -0.226314  -0.294014   -0.957525     -0.731563  -0.629339   1.42835"
      ]
     },
     "execution_count": 212,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "asd = randn(10,3)\n",
    "dsa = permutedims(asd, (2,1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4×3 Array{Float64,2}:\n",
       "  0.313378   -0.557817   0.594091 \n",
       "  0.0205297   1.21235   -0.294014 \n",
       " -0.58133     0.335587  -0.0179976\n",
       "  0.12264     1.73878   -0.629339 "
      ]
     },
     "execution_count": 217,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "asd[[5,2,4,9],:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3×4 Array{Float64,2}:\n",
       "  0.313378   0.0205297  -0.58133     0.12264 \n",
       " -0.557817   1.21235     0.335587    1.73878 \n",
       "  0.594091  -0.294014   -0.0179976  -0.629339"
      ]
     },
     "execution_count": 218,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dsa[:,[5,2,4,9]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dict{Any,Any} with 201 entries:\n",
       "  \"bert.encoder.layer.2.at… => PyObject tensor([[ 0.0469,  0.0819,  0.0672,  ..…\n",
       "  \"bert.encoder.layer.6.ou… => PyObject tensor([0.8681, 0.8663, 0.8156, 0.8230,…\n",
       "  \"bert.encoder.layer.3.ou… => PyObject tensor([0.8274, 0.8302, 0.8236, 0.8177,…\n",
       "  \"bert.encoder.layer.5.at… => PyObject tensor([[ 0.0046,  0.0253, -0.0487,  ..…\n",
       "  \"bert.encoder.layer.0.at… => PyObject tensor([[ 0.0119,  0.0013, -0.0088,  ..…\n",
       "  \"bert.encoder.layer.0.at… => PyObject tensor([[-0.0169,  0.0255, -0.0256,  ..…\n",
       "  \"bert.encoder.layer.4.in… => PyObject tensor([-0.0872, -0.1183, -0.1181,  ...…\n",
       "  \"bert.encoder.layer.4.at… => PyObject tensor([-5.8873e-03, -2.3518e-03,  2.81…\n",
       "  \"bert.encoder.layer.3.at… => PyObject tensor([0.9145, 0.8843, 0.8339, 0.8423,…\n",
       "  \"bert.encoder.layer.9.at… => PyObject tensor([0.8507, 0.8258, 0.7929, 0.7660,…\n",
       "  \"bert.encoder.layer.11.a… => PyObject tensor([[-0.0414, -0.0008, -0.0386,  ..…\n",
       "  \"bert.encoder.layer.10.a… => PyObject tensor([ 0.0213,  0.0152,  0.0528,  0.0…\n",
       "  \"bert.encoder.layer.10.o… => PyObject tensor([[ 0.0291, -0.0296, -0.0143,  ..…\n",
       "  \"bert.pooler.dense.weigh… => PyObject tensor([[-0.0009, -0.0371, -0.0149,  ..…\n",
       "  \"bert.encoder.layer.4.ou… => PyObject tensor([0.8738, 0.8576, 0.8342, 0.8449,…\n",
       "  \"bert.encoder.layer.8.at… => PyObject tensor([ 5.5696e-05,  3.4639e-03, -1.76…\n",
       "  \"bert.encoder.layer.6.at… => PyObject tensor([[-0.0171,  0.0046, -0.0515,  ..…\n",
       "  \"classifier.weight\"       => PyObject tensor([[-0.0063, -0.0054, -0.0078,  ..…\n",
       "  \"bert.embeddings.word_em… => PyObject tensor([[-0.0102, -0.0615, -0.0265,  ..…\n",
       "  \"bert.encoder.layer.10.a… => PyObject tensor([-3.6148e-02, -1.4108e-02, -3.76…\n",
       "  \"bert.encoder.layer.3.at… => PyObject tensor([[ 0.0591, -0.0266,  0.0031,  ..…\n",
       "  \"bert.encoder.layer.10.a… => PyObject tensor([[ 0.0006,  0.0370, -0.0083,  ..…\n",
       "  \"bert.encoder.layer.8.at… => PyObject tensor([ 0.0292,  0.0059, -0.0363,  0.0…\n",
       "  \"bert.encoder.layer.2.ou… => PyObject tensor([-0.0630,  0.0621, -0.0307,  0.0…\n",
       "  \"bert.encoder.layer.6.at… => PyObject tensor([[-0.0263,  0.0487, -0.0149,  ..…\n",
       "  ⋮                         => ⋮"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "using PyCall\n",
    "@pyimport torch\n",
    "@pyimport numpy\n",
    "torch_model = torch.load(\"/scratch/users/omutlu/dl_course/project/model-64-32.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2×768 Array{Float32,2}:\n",
       " -0.00625429  -0.00539395  …  0.0189996   0.0146921    0.00993014\n",
       " -0.0177308   -0.0229973      0.0122508  -0.00701508  -0.0259408 "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch_model[\"classifier.weight\"][:cpu]()[:numpy]()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 1.0.3",
   "language": "julia",
   "name": "julia-1.0"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.0.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}