Hidden FC Layers.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gensim\n",
    "W2V_PATH = 'word2vec/GoogleNews-vectors-negative300.bin'\n",
    "w2v = gensim.models.KeyedVectors.load_word2vec_format(W2V_PATH, binary=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.test.is_gpu_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp_data = pd.read_csv('data/train.csv')\n",
    "split_num = int(len(temp_data)*0.8)\n",
    "test_data = temp_data.iloc[split_num:]\n",
    "train_data = temp_data.iloc[:split_num]\n",
    "print(len(train_data))\n",
    "print(len(test_data))\n",
    "\n",
    "train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "import re\n",
    "\n",
    "def clean_punc(input_string):\n",
    "    proc_string = input_string.replace('<',' <less ')\n",
    "    proc_string = proc_string.replace('>',' <greater> ')\n",
    "    proc_string = re.sub(\"https?:\\/\\/(www\\.)?[-a-zA-Z0-9@:%._\\+~#=]{2,256}\\.[a-z]{2,6}\\b([-a-zA-Z0-9@:%_\\+.~#?&//=]*)\",' <url> ',proc_string)\n",
    "    proc_string = proc_string.replace(' <less ',' <less> ')\n",
    "    proc_string = proc_string.replace('?',' <question> ')\n",
    "    proc_string = proc_string.replace('...',' <suspension> ')\n",
    "    proc_string = proc_string.replace('. ',' <period> ')\n",
    "    proc_string = proc_string if not proc_string.endswith('.') else proc_string[:-1]\n",
    "    proc_string = proc_string.replace('/',' <slash> ')\n",
    "    proc_string = proc_string.replace('\\\\',' <backslash> ')\n",
    "    proc_string = proc_string.replace('; ',' <semicolon> ')\n",
    "    proc_string = proc_string.replace(': ',' <colon> ')\n",
    "    proc_string = proc_string.replace(', ',' <comma> ')\n",
    "    proc_string = proc_string.replace('!',' <exclame> ')\n",
    "    proc_string = proc_string.replace('\\n',' <newline> ')\n",
    "    proc_string = proc_string.replace(' - ',' <dash> ')\n",
    "    proc_string = proc_string.replace('\"\"',' <quote> ')\n",
    "    proc_string = proc_string.replace('\"',' <quote> ')\n",
    "    proc_string = proc_string.replace('(',' <openbracket> ')\n",
    "    proc_string = proc_string.replace(')',' <closebracket> ')\n",
    "    return proc_string\n",
    "\n",
    "def clean_word(input_word):\n",
    "    out_word = input_word.lower()\n",
    "    if ( out_word.startswith(\"'\") and out_word.endswith(\"'\")):\n",
    "        out_word = out_word[1:-1]\n",
    "    \n",
    "    if len(out_word)>0:\n",
    "        out_word = out_word if not out_word[-1] in ['.',':',';',','] else out_word[:-1]\n",
    "    \n",
    "    return out_word\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "comments = [clean_punc(comment) for comment in train_data.comment_text]\n",
    "comment_words = []\n",
    "for comment in comments:\n",
    "    comment_words.append ([word for word in comment.split()])\n",
    "flatten = lambda l: [item for sublist in l for item in sublist]\n",
    "\n",
    "flat_comments = flatten(comment_words)\n",
    "\n",
    "word_counts = collections.Counter()\n",
    "for word in flat_comments:\n",
    "    word_counts[word]+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_comments = [clean_punc(comment) for comment in test_data.comment_text]\n",
    "test_comment_words = []\n",
    "for comment in test_comments:\n",
    "    test_comment_words.append ([word for word in comment.split()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels_train = train_data.as_matrix(columns=['toxic','severe_toxic','obscene','threat','insult','identity_hate'])\n",
    "labels_test = test_data.as_matrix(columns=['toxic','severe_toxic','obscene','threat','insult','identity_hate'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(word_counts))\n",
    "\n",
    "very_common = [word for word,_ in word_counts.most_common(100)]\n",
    "\n",
    "very_common[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_words = [word for word in enumerate(word_counts.keys()) if word_counts[word]>4]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average: 79.68248260951307\n",
      "5 Percentile : 7.0\n",
      "10 Percentile : 10.0\n",
      "15 Percentile : 13.0\n",
      "20 Percentile : 17.0\n",
      "25 Percentile : 20.0\n",
      "30 Percentile : 24.0\n",
      "35 Percentile : 28.0\n",
      "40 Percentile : 33.0\n",
      "45 Percentile : 37.0\n",
      "50 Percentile : 43.0\n",
      "55 Percentile : 49.0\n",
      "60 Percentile : 56.0\n",
      "65 Percentile : 64.0\n",
      "70 Percentile : 74.0\n",
      "75 Percentile : 88.0\n",
      "80 Percentile : 106.0\n",
      "85 Percentile : 134.0\n",
      "90 Percentile : 178.0\n",
      "95 Percentile : 269.0\n",
      "100 Percentile : 4950.0\n"
     ]
    }
   ],
   "source": [
    "comment_lens = [len(comment) for comment in comment_words]\n",
    "print(\"Average: {}\".format(sum(comment_lens)/float(len(comment_lens))))\n",
    "for perc in range(5,101,5):\n",
    "    print(\"{0} Percentile : {1}\".format(perc,np.percentile(comment_lens,perc)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'wikipedia' in w2v.vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def map_word(in_word):\n",
    "    out_vector = np.zeros(307)\n",
    "    if in_word.isupper():\n",
    "        out_vector[300] = 1 #Flag shouting\n",
    "    if in_word.islower():\n",
    "        out_vector[301] = 1 #Flag normal text\n",
    "    work_word = in_word.lower()\n",
    "    if work_word in very_common:\n",
    "        out_vector[302] = 1 #Flag 100 most common words\n",
    "    \n",
    "    if work_word[0] == '<':\n",
    "        out_vector[303] = 1 #Flag punctuation we replaced and return\n",
    "        return out_vector\n",
    "    \n",
    "    if work_word in w2v.vocab and work_word in filtered_words:\n",
    "        out_vector[:300] = w2v[work_word]\n",
    "        return out_vector\n",
    "    \n",
    "    if work_word[0] == work_word[-1] and work_word[0] in ['_','*',\"'\"]:\n",
    "        out_vector[304] = 1 #Flag words with emphasis\n",
    "        work_word = work_word[1:-1]\n",
    "    \n",
    "    if len(work_word)>0:\n",
    "        work_word = work_word if not work_word[-1] in ['.',':',';',',',\"'\"] else work_word[:-1]\n",
    "\n",
    "    if work_word in w2v.vocab and work_word in filtered_words:\n",
    "        out_vector[:300] = w2v[work_word]\n",
    "        return out_vector\n",
    "    \n",
    "    out_vector[305] = 1 #Flag unknown words\n",
    "    return out_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.21875    -0.12207031 -0.00296021  0.02429199  0.08300781 -0.01977539\n",
      "  0.00396729 -0.09570312  0.11035156 -0.37109375  0.12451172 -0.54296875\n",
      " -0.09912109  0.08544922 -0.16894531 -0.10205078  0.22753906 -0.07421875\n",
      " -0.03015137 -0.35742188 -0.11523438 -0.01171875  0.27148438 -0.01049805\n",
      " -0.22070312 -0.17578125 -0.18847656  0.18554688 -0.08007812 -0.05615234\n",
      " -0.05151367 -0.11132812 -0.24609375 -0.09912109 -0.14550781  0.08447266\n",
      " -0.12792969  0.29882812  0.24609375  0.10449219  0.12402344 -0.07324219\n",
      "  0.15625     0.59765625  0.28125     0.00970459 -0.171875   -0.25585938\n",
      " -0.24511719 -0.171875   -0.24121094 -0.10302734 -0.17578125 -0.05834961\n",
      "  0.18945312 -0.08349609  0.11279297  0.07470703 -0.27148438 -0.3203125\n",
      "  0.12158203 -0.04052734  0.13378906 -0.18457031  0.01904297 -0.19433594\n",
      " -0.203125   -0.24414062  0.16113281  0.02490234 -0.11035156  0.16015625\n",
      " -0.23632812 -0.19628906 -0.14550781  0.10546875  0.07177734 -0.14257812\n",
      " -0.03857422  0.20703125  0.30078125  0.06591797  0.14160156 -0.09228516\n",
      "  0.3125      0.09667969  0.04296875  0.31640625  0.19726562  0.04272461\n",
      " -0.04833984 -0.09228516 -0.26757812 -0.11914062  0.03710938  0.26367188\n",
      " -0.01989746  0.01098633  0.30273438  0.24902344 -0.2890625   0.18945312\n",
      " -0.01281738  0.07714844  0.07714844  0.21875    -0.59765625 -0.1015625\n",
      "  0.09863281 -0.11523438 -0.40625     0.01867676 -0.05859375 -0.125\n",
      "  0.109375   -0.08007812 -0.17871094 -0.29296875  0.19726562  0.10449219\n",
      "  0.16113281 -0.33398438 -0.22558594 -0.10791016 -0.11230469  0.07910156\n",
      " -0.20214844  0.03637695 -0.13378906 -0.0037384  -0.41015625 -0.18652344\n",
      " -0.08984375  0.26367188 -0.1953125  -0.27929688 -0.31835938 -0.02246094\n",
      " -0.25585938  0.01544189  0.10546875 -0.09960938  0.01660156 -0.00756836\n",
      "  0.02514648 -0.18847656 -0.09082031 -0.14648438 -0.04956055 -0.30664062\n",
      "  0.14453125 -0.15820312 -0.33398438 -0.05615234 -0.1328125  -0.18066406\n",
      "  0.16503906  0.00177765 -0.15527344 -0.16601562  0.08203125  0.34375\n",
      " -0.33203125 -0.2421875   0.30664062 -0.26171875  0.15136719  0.17285156\n",
      " -0.47851562  0.03564453  0.01806641  0.01556396  0.13769531  0.00169373\n",
      " -0.00897217  0.04882812 -0.07421875 -0.53515625  0.6328125  -0.18164062\n",
      "  0.04516602 -0.21679688 -0.32226562  0.07177734  0.203125    0.12597656\n",
      " -0.17382812  0.20117188 -0.1484375   0.06738281  0.16308594  0.01916504\n",
      " -0.15722656 -0.21972656  0.26757812  0.265625   -0.27929688 -0.38085938\n",
      " -0.0703125  -0.14648438 -0.09521484  0.2734375   0.04833984  0.11669922\n",
      " -0.05566406  0.04296875  0.04150391 -0.15429688 -0.46679688  0.22460938\n",
      " -0.06835938  0.10302734 -0.3125      0.08154297 -0.12597656  0.18359375\n",
      "  0.10791016  0.03112793  0.17578125 -0.01287842  0.04077148 -0.19335938\n",
      "  0.24121094  0.13183594  0.12304688  0.1796875   0.12353516  0.27148438\n",
      "  0.08300781  0.26171875  0.14355469 -0.07226562 -0.04467773 -0.01831055\n",
      "  0.07226562  0.16894531  0.14160156  0.00646973  0.28125    -0.09570312\n",
      "  0.296875    0.07714844  0.07226562 -0.16894531 -0.01446533  0.18554688\n",
      " -0.01495361  0.55078125 -0.12890625 -0.06982422  0.06689453  0.12988281\n",
      "  0.17480469  0.02197266  0.06225586 -0.33984375 -0.37109375  0.1796875\n",
      "  0.11425781 -0.13378906  0.3984375  -0.02685547  0.11035156 -0.25976562\n",
      "  0.2890625   0.08984375 -0.05004883 -0.20605469 -0.0222168  -0.34375\n",
      " -0.17382812  0.07324219  0.14550781  0.11376953 -0.06030273 -0.11035156\n",
      "  0.21191406 -0.20703125  0.0168457   0.3359375   0.23925781  0.4140625\n",
      " -0.41015625  0.18261719  0.08642578 -0.12207031 -0.02758789  0.27929688\n",
      " -0.03295898  0.14550781  0.32226562  0.10205078 -0.17578125  0.37304688\n",
      " -0.09179688 -0.14257812 -0.00221252 -0.35351562 -0.25195312 -0.11621094\n",
      "  1.          0.          0.          0.          1.          0.\n",
      "  0.        ]\n"
     ]
    }
   ],
   "source": [
    "print(map_word('*WIKIPEDIA*'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_comment(input_comment):\n",
    "    result_matrix = np.zeros((comment_length,embed_size))\n",
    "    if (len(input_comment) == 0):\n",
    "        return result_matrix\n",
    "    \n",
    "    input_comment = input_comment[:comment_length]\n",
    "    temp_matrix = [map_word(word) for word in input_comment]\n",
    "    result_matrix[-len(input_comment):,:] = temp_matrix\n",
    "    return result_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(250, 307)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Test the process comment function\n",
    "np.array(process_comment(comment_words[21])).shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Approach using live generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from random import shuffle\n",
    "\n",
    "def get_batches(comment_words,labels,batch_size):\n",
    "    \n",
    "    num_inputs = len(comment_words)\n",
    "    num_batches = num_inputs//batch_size\n",
    "    \n",
    "    labels_true = []\n",
    "    offset = []\n",
    "    for i in range(0,6):\n",
    "        labels_true.append([])\n",
    "        offset.append(0)\n",
    "\n",
    "    for i,label in enumerate(labels):\n",
    "        for ii in range(0,6):\n",
    "            if label[ii] == 1:\n",
    "                labels_true[ii].append(i)\n",
    "    \n",
    "    no_labels = [i for i,label in enumerate(labels) if sum(label)==0]\n",
    "    shuffle(no_labels)\n",
    "    \n",
    "    offset.append(0)\n",
    "    group_size = batch_size // 8\n",
    "    \n",
    "    for ii in range(0,num_batches):\n",
    "        indicies = set()\n",
    "        for i in range(0,6):\n",
    "            for iii in range(0,group_size):\n",
    "                indicies.add(labels_true[i][offset[i]])\n",
    "                offset[i]+=1\n",
    "                if (offset[i]==len(labels_true[i])):\n",
    "                    offset[i]=0\n",
    "                    shuffle(labels_true[i])\n",
    "                \n",
    "        num_remaining = batch_size - len(indicies)\n",
    "        for iii in range(0,num_remaining):\n",
    "            indicies.add(no_labels[offset[6]])\n",
    "            offset[6]+=1\n",
    "            if offset[6] == len(no_labels):\n",
    "                offset[6]=0\n",
    "                shuffle(no_labels)\n",
    "            \n",
    "        features = []\n",
    "        for iii in indicies:\n",
    "            features.append(process_comment(comment_words[iii]))\n",
    "        \n",
    "        return_labels = [labels[i] for i in indicies]\n",
    "        yield features, return_labels\n",
    "\n",
    "def get_test_batches(comment_words,labels,batch_size):\n",
    "    num_inputs = len(comment_words)\n",
    "    num_batches = num_inputs//batch_size\n",
    "    if (num_inputs > num_batches * batch_size):\n",
    "        num_batches += 1\n",
    "        \n",
    "    for ii in range(0,num_batches):\n",
    "        end = ii * batch_size + batch_size if ii * batch_size + batch_size <= num_inputs else num_inputs - 1\n",
    "        indicies = [0] * batch_size\n",
    "        indicies[:end-ii*batch_size] = range(ii * batch_size,end)\n",
    "        features = []\n",
    "        for iii in indicies:\n",
    "            features.append(process_comment(comment_words[iii]))\n",
    "        \n",
    "        return_labels = [labels[i] for i in indicies]\n",
    "        yield features, return_labels\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels_true = []\n",
    "for i in range(0,6):\n",
    "    labels_true.append([])\n",
    "\n",
    "for i,label in enumerate(labels_train):\n",
    "    for ii in range(0,6):\n",
    "        if label[ii] == 1:\n",
    "            labels_true[ii].append(i)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[12257, 1284, 6780, 386, 6295, 1100]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[len(the_list) for the_list in labels_true]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hyper-paramters\n",
    "layer_size = 256\n",
    "layer_count = 3\n",
    "hidden_fc_layers = [200]\n",
    "keep_prob_training = 0.6\n",
    "learning_rate = 0.0001\n",
    "epochs = 20\n",
    "batch_size=128\n",
    "comment_length = 100\n",
    "embed_size = 307\n",
    "n_labels = 6\n",
    "checkpoint_path = 'a3cp1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Building a graph and placeholders\n",
    "graph = tf.Graph()\n",
    "\n",
    "with graph.as_default():\n",
    "    inputs_ = tf.placeholder(tf.float32,[None,comment_length,embed_size],name='inputs')\n",
    "    labels_ = tf.placeholder(tf.float32,[None,None],name='outputs')\n",
    "    keep_prob_ = tf.placeholder(tf.float32,name='keep_prob')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build the network\n",
    "with graph.as_default():\n",
    "    lstm_cell = tf.contrib.rnn.BasicLSTMCell(num_units=layer_size)\n",
    "    drop = tf.contrib.rnn.DropoutWrapper(cell=lstm_cell,input_keep_prob=keep_prob_)\n",
    "    network = drop\n",
    "    for _ in range(layer_count):\n",
    "        network = tf.contrib.rnn.MultiRNNCell([network])\n",
    "\n",
    "    initial_state = network.zero_state(batch_size,tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Forward pass\n",
    "with graph.as_default():\n",
    "    outputs, final_state = tf.nn.dynamic_rnn(network,inputs_,initial_state=initial_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get outputs\n",
    "with graph.as_default():\n",
    "    predictions = tf.contrib.layers.flatten(outputs)\n",
    "    for size in hidden_fc_layers:\n",
    "        predictions = tf.contrib.layers.fully_connected(predictions, size, activation_fn=None)\n",
    "        predictions = tf.nn.leaky_relu(predictions,alpha=0.2)\n",
    "    predictions = tf.contrib.layers.fully_connected(predictions, n_labels, activation_fn=tf.sigmoid)\n",
    "    cost = tf.losses.sigmoid_cross_entropy(labels_, predictions)\n",
    "    \n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Determine accuracy on test set\n",
    "with graph.as_default():\n",
    "    validation_metrics_var_scope = \"validation_metrics\"\n",
    "    binary_pred = tf.cast(tf.round(predictions), tf.bool)\n",
    "    binary_labels = tf.cast(labels_, tf.bool)\n",
    "    accuracy = tf.reduce_sum(tf.cast(tf.equal(binary_pred,binary_labels),tf.int32))\n",
    "    correct_pos = tf.reduce_sum(tf.cast(tf.logical_and(binary_pred,binary_labels),tf.int32),axis=0)\n",
    "    false_pos = tf.reduce_sum(tf.cast(tf.logical_and(binary_pred,tf.logical_not(binary_labels)),tf.int32),axis=0)\n",
    "    false_neg = tf.reduce_sum(tf.cast(tf.logical_and(tf.logical_not(binary_pred),binary_labels),tf.int32),axis=0)\n",
    "    correct_neg = tf.reduce_sum(tf.cast(tf.logical_and(tf.logical_not(binary_pred),tf.logical_not(binary_labels)),tf.int32),axis=0)\n",
    "    auc = tf.metrics.auc(labels=labels_,predictions=predictions,name=validation_metrics_var_scope)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting...\n",
      "INFO:tensorflow:Restoring parameters from a3cp1/epoch0iter200.ckpt\n",
      "Restored checkpoint from a3cp1/epoch0iter200.ckpt.\n",
      "Epoch: 0/20 Iteration: 200/19940 Train loss: 0.619During epoch 0\n",
      "  Val acc      : 0.18093109130859375\n",
      "  AuC          : 0.8819448947906494\n",
      "  Correct pos  :   316      0    139      0    136      0\n",
      "  False neg    :    81     43     74     10     69     38\n",
      "  Correct neg  :  3343   4053   3807   4086   3777   4058\n",
      "  False pos    :   356      0     76      0    114      0\n",
      "\n",
      "Epoch: 0/20 Iteration: 400/19940 Train loss: 0.609During epoch 0\n",
      "  Val acc      : 0.1809539794921875\n",
      "  AuC          : 0.90392005443573\n",
      "  Correct pos  :   333      0    168      0    155      0\n",
      "  False neg    :    64     43     45     10     50     38\n",
      "  Correct neg  :  3342   4053   3766   4086   3757   4058\n",
      "  False pos    :   357      0    117      0    134      0\n",
      "\n",
      "Epoch: 0/20 Iteration: 600/19940 Train loss: 0.617During epoch 0\n",
      "  Val acc      : 0.18126678466796875\n",
      "  AuC          : 0.9044803977012634\n",
      "  Correct pos  :   328      0    171      0    157      0\n",
      "  False neg    :    69     43     42     10     48     38\n",
      "  Correct neg  :  3381   4053   3760   4086   3765   4058\n",
      "  False pos    :   318      0    123      0    126      0\n",
      "\n",
      "Epoch: 0/20 Iteration: 800/19940 Train loss: 0.608During epoch 0\n",
      "  Val acc      : 0.18169403076171875\n",
      "  AuC          : 0.9054816961288452\n",
      "  Correct pos  :   329      0    166      0    164      0\n",
      "  False neg    :    68     43     47     10     41     38\n",
      "  Correct neg  :  3427   4053   3785   4086   3747   4058\n",
      "  False pos    :   272      0     98      0    144      0\n",
      "\n",
      "Epoch: 1/20 Iteration: 1000/19940 Train loss: 0.618During epoch 1\n",
      "  Val acc      : 0.18262481689453125\n",
      "  AuC          : 0.8908983469009399\n",
      "  Correct pos  :   305      0    161      0    148      0\n",
      "  False neg    :    92     43     52     10     57     38\n",
      "  Correct neg  :  3535   4053   3804   4086   3787   4058\n",
      "  False pos    :   164      0     79      0    104      0\n",
      "\n",
      "Epoch: 1/20 Iteration: 1200/19940 Train loss: 0.609During epoch 1\n",
      "  Val acc      : 0.182830810546875\n",
      "  AuC          : 0.8834781646728516\n",
      "  Correct pos  :   302      0    142      0    139      0\n",
      "  False neg    :    95     43     71     10     66     38\n",
      "  Correct neg  :  3556   4053   3818   4086   3810   4058\n",
      "  False pos    :   143      0     65      0     81      0\n",
      "\n",
      "Epoch: 1/20 Iteration: 1400/19940 Train loss: 0.603During epoch 1\n",
      "  Val acc      : 0.1828155517578125\n",
      "  AuC          : 0.8932780623435974\n",
      "  Correct pos  :   306      0    153      0    139      0\n",
      "  False neg    :    91     43     60     10     66     38\n",
      "  Correct neg  :  3548   4053   3813   4086   3806   4058\n",
      "  False pos    :   151      0     70      0     85      0\n",
      "\n",
      "Epoch: 1/20 Iteration: 1600/19940 Train loss: 0.611During epoch 1\n",
      "  Val acc      : 0.182037353515625\n",
      "  AuC          : 0.9073960781097412\n",
      "  Correct pos  :   325      0    183      0    166      0\n",
      "  False neg    :    72     43     30     10     39     38\n",
      "  Correct neg  :  3472   4053   3753   4086   3764   4058\n",
      "  False pos    :   227      0    130      0    127      0\n",
      "\n",
      "Epoch: 1/20 Iteration: 1800/19940 Train loss: 0.604During epoch 1\n",
      "  Val acc      : 0.182403564453125\n",
      "  AuC          : 0.9043751955032349\n",
      "  Correct pos  :   327      0    169      0    170      0\n",
      "  False neg    :    70     43     44     10     35     38\n",
      "  Correct neg  :  3480   4053   3804   4086   3761   4058\n",
      "  False pos    :   219      0     79      0    130      0\n",
      "\n",
      "Epoch: 2/20 Iteration: 2000/19940 Train loss: 0.609During epoch 2\n",
      "  Val acc      : 0.02353426288579038\n",
      "  AuC          : 0.8802485466003418\n",
      "  Correct pos  :  2251      0   1132      0   1097      0\n",
      "  False neg    :   786    311    537     92    485    305\n",
      "  Correct neg  : 28035  31689  30060  31908  29907  31695\n",
      "  False pos    :   928      0    271      0    511      0\n",
      "\n",
      "Epoch: 2/20 Iteration: 2200/19940 Train loss: 0.606During epoch 2\n",
      "  Val acc      : 0.181915283203125\n",
      "  AuC          : 0.909542441368103\n",
      "  Correct pos  :   337      0    179      0    167      0\n",
      "  False neg    :    60     43     34     10     38     38\n",
      "  Correct neg  :  3408   4053   3786   4086   3770   4058\n",
      "  False pos    :   291      0     97      0    121      0\n",
      "\n",
      "Epoch: 2/20 Iteration: 2400/19940 Train loss: 0.610During epoch 2\n",
      "  Val acc      : 0.1821441650390625\n",
      "  AuC          : 0.9067018628120422\n",
      "  Correct pos  :   337      0    159      0    170      0\n",
      "  False neg    :    60     43     54     10     35     38\n",
      "  Correct neg  :  3428   4053   3824   4086   3759   4058\n",
      "  False pos    :   271      0     59      0    132      0\n",
      "\n",
      "Epoch: 2/20 Iteration: 2600/19940 Train loss: 0.590During epoch 2\n",
      "  Val acc      : 0.18304443359375\n",
      "  AuC          : 0.8912787437438965\n",
      "  Correct pos  :   305      0    169      0    156      0\n",
      "  False neg    :    92     43     44     10     49     38\n",
      "  Correct neg  :  3554   4053   3811   4086   3800   4058\n",
      "  False pos    :   145      0     72      0     91      0\n",
      "\n",
      "Epoch: 2/20 Iteration: 2800/19940 Train loss: 0.604During epoch 2\n",
      "  Val acc      : 0.18304443359375\n",
      "  AuC          : 0.8903486132621765\n",
      "  Correct pos  :   305      0    168      0    152      0\n",
      "  False neg    :    92     43     45     10     53     38\n",
      "  Correct neg  :  3547   4053   3815   4086   3808   4058\n",
      "  False pos    :   152      0     68      0     83      0\n",
      "\n",
      "Epoch: 3/20 Iteration: 3000/19940 Train loss: 0.608During epoch 3\n",
      "  Val acc      : 0.1833648681640625\n",
      "  AuC          : 0.8733048439025879\n",
      "  Correct pos  :   281      0    153      0    149      0\n",
      "  False neg    :   116     43     60     10     56     38\n",
      "  Correct neg  :  3615   4053   3833   4086   3806   4058\n",
      "  False pos    :    84      0     50      0     85      0\n",
      "\n",
      "Epoch: 3/20 Iteration: 3200/19940 Train loss: 0.596During epoch 3\n",
      "  Val acc      : 0.1829376220703125\n",
      "  AuC          : 0.8919846415519714\n",
      "  Correct pos  :   308      0    167      0    156      0\n",
      "  False neg    :    89     43     46     10     49     38\n",
      "  Correct neg  :  3547   4053   3815   4086   3788   4058\n",
      "  False pos    :   152      0     68      0    103      0\n",
      "\n",
      "Epoch: 3/20 Iteration: 3400/19940 Train loss: 0.604During epoch 3\n",
      "  Val acc      : 0.1829376220703125\n",
      "  AuC          : 0.8870043754577637\n",
      "  Correct pos  :   298      0    171      0    156      0\n",
      "  False neg    :    99     43     42     10     49     38\n",
      "  Correct neg  :  3568   4053   3799   4086   3789   4058\n",
      "  False pos    :   131      0     84      0    102      0\n",
      "\n",
      "Epoch: 3/20 Iteration: 3600/19940 Train loss: 0.597During epoch 3\n",
      "  Val acc      : 0.18302154541015625\n",
      "  AuC          : 0.8940317034721375\n",
      "  Correct pos  :   306      0    176      0    159      0\n",
      "  False neg    :    91     43     37     10     46     38\n",
      "  Correct neg  :  3559   4053   3805   4086   3787   4058\n",
      "  False pos    :   140      0     78      0    104      0\n",
      "\n",
      "Epoch: 3/20 Iteration: 3800/19940 Train loss: 0.603During epoch 3\n",
      "  Val acc      : 0.183258056640625\n",
      "  AuC          : 0.8862273693084717\n",
      "  Correct pos  :   300      0    170      0    151      0\n",
      "  False neg    :    97     43     43     10     54     38\n",
      "  Correct neg  :  3590   4053   3803   4086   3809   4058\n",
      "  False pos    :   109      0     80      0     82      0\n",
      "\n",
      "Epoch: 4/20 Iteration: 4000/19940 Train loss: 0.612During epoch 4\n",
      "  Val acc      : 0.02353463888453705\n",
      "  AuC          : 0.8905243873596191\n",
      "  Correct pos  :  2348      0   1263      0   1225      0\n",
      "  False neg    :   689    311    406     92    357    305\n",
      "  Correct neg  : 27987  31689  29991  31908  29671  31695\n",
      "  False pos    :   976      0    340      0    747      0\n",
      "\n",
      "Epoch: 4/20 Iteration: 4200/19940 Train loss: 0.600During epoch 4\n",
      "  Val acc      : 0.18257904052734375\n",
      "  AuC          : 0.8990344405174255\n",
      "  Correct pos  :   322      0    176      0    167      0\n",
      "  False neg    :    75     43     37     10     38     38\n",
      "  Correct neg  :  3506   4053   3792   4086   3771   4058\n",
      "  False pos    :   193      0     91      0    120      0\n",
      "\n",
      "Epoch: 4/20 Iteration: 4400/19940 Train loss: 0.591During epoch 4\n",
      "  Val acc      : 0.18289947509765625\n",
      "  AuC          : 0.8966649770736694\n",
      "  Correct pos  :   310      0    173      0    155      0\n",
      "  False neg    :    87     43     40     10     50     38\n",
      "  Correct neg  :  3547   4053   3800   4086   3791   4058\n",
      "  False pos    :   152      0     83      0    100      0\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 4/20 Iteration: 4600/19940 Train loss: 0.597During epoch 4\n",
      "  Val acc      : 0.18288421630859375\n",
      "  AuC          : 0.895698070526123\n",
      "  Correct pos  :   308      0    173      0    154      0\n",
      "  False neg    :    89     43     40     10     51     38\n",
      "  Correct neg  :  3553   4053   3800   4086   3786   4058\n",
      "  False pos    :   146      0     83      0    105      0\n",
      "\n",
      "Epoch: 4/20 Iteration: 4800/19940 Train loss: 0.598During epoch 4\n",
      "  Val acc      : 0.183258056640625\n",
      "  AuC          : 0.8851990103721619\n",
      "  Correct pos  :   303      0    165      0    148      0\n",
      "  False neg    :    94     43     48     10     57     38\n",
      "  Correct neg  :  3584   4053   3818   4086   3805   4058\n",
      "  False pos    :   115      0     65      0     86      0\n",
      "\n",
      "Epoch: 5/20 Iteration: 5000/19940 Train loss: 0.613During epoch 5\n",
      "  Val acc      : 0.18279266357421875\n",
      "  AuC          : 0.8960103988647461\n",
      "  Correct pos  :   316      0    162      0    159      0\n",
      "  False neg    :    81     43     51     10     46     38\n",
      "  Correct neg  :  3520   4053   3820   4086   3785   4058\n",
      "  False pos    :   179      0     63      0    106      0\n",
      "\n",
      "Epoch: 5/20 Iteration: 5200/19940 Train loss: 0.603During epoch 5\n",
      "  Val acc      : 0.1828155517578125\n",
      "  AuC          : 0.8957093358039856\n",
      "  Correct pos  :   319      0    172      0    159      0\n",
      "  False neg    :    78     43     41     10     46     38\n",
      "  Correct neg  :  3521   4053   3807   4086   3787   4058\n",
      "  False pos    :   178      0     76      0    104      0\n",
      "\n",
      "Epoch: 5/20 Iteration: 5400/19940 Train loss: 0.599During epoch 5\n",
      "  Val acc      : 0.18303680419921875\n",
      "  AuC          : 0.8839399218559265\n",
      "  Correct pos  :   298      2    161      0    158      0\n",
      "  False neg    :    99     41     52     10     47     38\n",
      "  Correct neg  :  3581   4051   3826   4086   3770   4058\n",
      "  False pos    :   118      2     57      0    121      0\n",
      "\n",
      "Epoch: 5/20 Iteration: 5600/19940 Train loss: 0.597During epoch 5\n",
      "  Val acc      : 0.18296051025390625\n",
      "  AuC          : 0.9101561307907104\n",
      "  Correct pos  :   316     20    163      0    152      0\n",
      "  False neg    :    81     23     50     10     53     38\n",
      "  Correct neg  :  3549   4036   3818   4086   3783   4058\n",
      "  False pos    :   150     17     65      0    108      0\n",
      "\n",
      "Epoch: 5/20 Iteration: 5800/19940 Train loss: 0.595During epoch 5\n",
      "  Val acc      : 0.18291473388671875\n",
      "  AuC          : 0.9109674692153931\n",
      "  Correct pos  :   318     14    168      0    153      0\n",
      "  False neg    :    79     29     45     10     52     38\n",
      "  Correct neg  :  3544   4037   3810   4086   3789   4056\n",
      "  False pos    :   155     16     73      0    102      2\n",
      "\n",
      "Epoch: 6/20 Iteration: 6000/19940 Train loss: 0.605During epoch 6\n",
      "  Val acc      : 0.023509446968510105\n",
      "  AuC          : 0.9088672399520874\n",
      "  Correct pos  :  2454    145   1311      0   1191      0\n",
      "  False neg    :   583    166    358     92    391    305\n",
      "  Correct neg  : 27743  31512  29875  31908  29745  31692\n",
      "  False pos    :  1220    177    456      0    673      3\n",
      "\n",
      "Epoch: 6/20 Iteration: 6200/19940 Train loss: 0.587During epoch 6\n",
      "  Val acc      : 0.18297576904296875\n",
      "  AuC          : 0.9096677303314209\n",
      "  Correct pos  :   313     15    168      0    149      0\n",
      "  False neg    :    84     28     45     10     56     38\n",
      "  Correct neg  :  3552   4036   3800   4086   3807   4057\n",
      "  False pos    :   147     17     83      0     84      1\n",
      "\n",
      "Epoch: 6/20 Iteration: 6400/19940 Train loss: 0.585During epoch 6\n",
      "  Val acc      : 0.18321990966796875\n",
      "  AuC          : 0.9017293453216553\n",
      "  Correct pos  :   299     14    164      0    157      0\n",
      "  False neg    :    98     29     49     10     48     38\n",
      "  Correct neg  :  3590   4037   3820   4084   3794   4056\n",
      "  False pos    :   109     16     63      2     97      2\n",
      "\n",
      "Epoch: 6/20 Iteration: 6600/19940 Train loss: 0.594During epoch 6\n",
      "  Val acc      : 0.18300628662109375\n",
      "  AuC          : 0.9021214842796326\n",
      "  Correct pos  :   321     13    160      0    152      0\n",
      "  False neg    :    76     30     53     10     53     38\n",
      "  Correct neg  :  3529   4042   3831   4085   3798   4056\n",
      "  False pos    :   170     11     52      1     93      2\n",
      "\n",
      "Epoch: 6/20 Iteration: 6800/19940 Train loss: 0.586During epoch 6\n",
      "  Val acc      : 0.183502197265625\n",
      "  AuC          : 0.8878135681152344\n",
      "  Correct pos  :   291     11    166      0    144      0\n",
      "  False neg    :   106     32     47     10     61     38\n",
      "  Correct neg  :  3611   4043   3821   4084   3825   4056\n",
      "  False pos    :    88     10     62      2     66      2\n",
      "\n",
      "Epoch: 7/20 Iteration: 7000/19940 Train loss: 0.599During epoch 7\n",
      "  Val acc      : 0.1828765869140625\n",
      "  AuC          : 0.9146817326545715\n",
      "  Correct pos  :   316     21    163      0    161      0\n",
      "  False neg    :    81     22     50     10     44     38\n",
      "  Correct neg  :  3539   4029   3823   4084   3778   4056\n",
      "  False pos    :   160     24     60      2    113      2\n",
      "\n",
      "Epoch: 7/20 Iteration: 7200/19940 Train loss: 0.597During epoch 7\n",
      "  Val acc      : 0.18329620361328125\n",
      "  AuC          : 0.9045299887657166\n",
      "  Correct pos  :   305     17    167      0    156      0\n",
      "  False neg    :    92     26     46     10     49     38\n",
      "  Correct neg  :  3584   4038   3818   4084   3798   4058\n",
      "  False pos    :   115     15     65      2     93      0\n",
      "\n",
      "Epoch: 7/20 Iteration: 7400/19940 Train loss: 0.600During epoch 7\n",
      "  Val acc      : 0.18315887451171875\n",
      "  AuC          : 0.8959149122238159\n",
      "  Correct pos  :   303     12    166      0    138      0\n",
      "  False neg    :    94     31     47     10     67     38\n",
      "  Correct neg  :  3581   4037   3807   4084   3821   4058\n",
      "  False pos    :   118     16     76      2     70      0\n",
      "\n",
      "Epoch: 7/20 Iteration: 7600/19940 Train loss: 0.582During epoch 7\n",
      "  Val acc      : 0.18331146240234375\n",
      "  AuC          : 0.8978018164634705\n",
      "  Correct pos  :   304     20    163      0    149      0\n",
      "  False neg    :    93     23     50     10     56     38\n",
      "  Correct neg  :  3591   4029   3820   4084   3809   4058\n",
      "  False pos    :   108     24     63      2     82      0\n",
      "\n",
      "Epoch: 7/20 Iteration: 7800/19940 Train loss: 0.597During epoch 7\n",
      "  Val acc      : 0.18328857421875\n",
      "  AuC          : 0.9013495445251465\n",
      "  Correct pos  :   309     11    163      0    150      0\n",
      "  False neg    :    88     32     50     10     55     38\n",
      "  Correct neg  :  3582   4039   3815   4084   3813   4058\n",
      "  False pos    :   117     14     68      2     78      0\n",
      "\n",
      "Epoch: 8/20 Iteration: 8000/19940 Train loss: 0.594During epoch 8\n",
      "  Val acc      : 0.02355030549898167\n",
      "  AuC          : 0.900409996509552\n",
      "  Correct pos  :  2330    131   1326      0   1161      0\n",
      "  False neg    :   707    180    343     92    421    305\n",
      "  Correct neg  : 28086  31553  29890  31904  29828  31693\n",
      "  False pos    :   877    136    441      4    590      2\n",
      "\n",
      "Epoch: 8/20 Iteration: 8200/19940 Train loss: 0.591During epoch 8\n",
      "  Val acc      : 0.18326568603515625\n",
      "  AuC          : 0.9015793204307556\n",
      "  Correct pos  :   306     15    169      0    147      0\n",
      "  False neg    :    91     28     44     10     58     38\n",
      "  Correct neg  :  3593   4033   3804   4084   3812   4058\n",
      "  False pos    :   106     20     79      2     79      0\n",
      "\n",
      "Epoch: 8/20 Iteration: 8400/19940 Train loss: 0.593During epoch 8\n",
      "  Val acc      : 0.18314361572265625\n",
      "  AuC          : 0.903624951839447\n",
      "  Correct pos  :   311     16    162      0    148      0\n",
      "  False neg    :    86     27     51     10     57     38\n",
      "  Correct neg  :  3565   4034   3819   4084   3808   4058\n",
      "  False pos    :   134     19     64      2     83      0\n",
      "\n",
      "Epoch: 8/20 Iteration: 8600/19940 Train loss: 0.598During epoch 8\n",
      "  Val acc      : 0.18317413330078125\n",
      "  AuC          : 0.8990839123725891\n",
      "  Correct pos  :   309     14    166      0    148      0\n",
      "  False neg    :    88     29     47     10     57     38\n",
      "  Correct neg  :  3572   4039   3812   4084   3807   4058\n",
      "  False pos    :   127     14     71      2     84      0\n",
      "\n",
      "Epoch: 8/20 Iteration: 8800/19940 Train loss: 0.594During epoch 8\n",
      "  Val acc      : 0.18338775634765625\n",
      "  AuC          : 0.8953045606613159\n",
      "  Correct pos  :   302     18    167      0    149      0\n",
      "  False neg    :    95     25     46     10     56     38\n",
      "  Correct neg  :  3598   4030   3821   4085   3809   4058\n",
      "  False pos    :   101     23     62      1     82      0\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 9/20 Iteration: 8980/19940 Train loss: 0.602"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-29-f295fb97cfb1>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     27\u001b[0m         \u001b[0mstate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minitial_state\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 29\u001b[0;31m         \u001b[0;32mfor\u001b[0m \u001b[0mii\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mget_batches\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcomment_words\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlabels_train\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     30\u001b[0m             feed = {inputs_:x,\n\u001b[1;32m     31\u001b[0m                     \u001b[0mlabels_\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-17-2f52daea08a6>\u001b[0m in \u001b[0;36mget_batches\u001b[0;34m(comment_words, labels, batch_size)\u001b[0m\n\u001b[1;32m     43\u001b[0m         \u001b[0mfeatures\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0miii\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mindicies\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 45\u001b[0;31m             \u001b[0mfeatures\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocess_comment\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcomment_words\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0miii\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     46\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     47\u001b[0m         \u001b[0mreturn_labels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mindicies\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-27-a992cdc8defb>\u001b[0m in \u001b[0;36mprocess_comment\u001b[0;34m(input_comment)\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0minput_comment\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput_comment\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mcomment_length\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m     \u001b[0mtemp_matrix\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mmap_word\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mword\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mword\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minput_comment\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m     \u001b[0mresult_matrix\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_comment\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtemp_matrix\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mresult_matrix\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-27-a992cdc8defb>\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0minput_comment\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput_comment\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mcomment_length\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m     \u001b[0mtemp_matrix\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mmap_word\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mword\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mword\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minput_comment\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m     \u001b[0mresult_matrix\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_comment\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtemp_matrix\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mresult_matrix\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-13-927d935f33be>\u001b[0m in \u001b[0;36mmap_word\u001b[0;34m(in_word)\u001b[0m\n\u001b[1;32m     13\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mout_vector\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m     \u001b[0;32mif\u001b[0m \u001b[0mwork_word\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mw2v\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvocab\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mwork_word\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfiltered_words\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mout_vector\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m300\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mw2v\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mwork_word\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mout_vector\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "#Training\n",
    "with graph.as_default():\n",
    "    saver = tf.train.Saver()\n",
    "    \n",
    "n_batches = len(comment_words)//batch_size\n",
    "\n",
    "val_acc = []\n",
    "false_pos_list = []\n",
    "\n",
    "last_checkpoint = tf.train.latest_checkpoint(checkpoint_path)\n",
    "\n",
    "print(\"Starting...\")\n",
    "\n",
    "iteration = 0\n",
    "with tf.Session(graph=graph) as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    \n",
    "    validation_metrics_vars = tf.get_collection(tf.GraphKeys.LOCAL_VARIABLES, scope=validation_metrics_var_scope)\n",
    "    validation_metrics_init_op = tf.variables_initializer(var_list=validation_metrics_vars, name='validation_metrics_init')\n",
    "    sess.run(validation_metrics_init_op)\n",
    "    \n",
    "    if last_checkpoint != None:\n",
    "        saver.restore(sess,last_checkpoint)\n",
    "        print(\"Restored checkpoint from {}.\".format(last_checkpoint))    \n",
    "    \n",
    "    for e in range(epochs):\n",
    "        state = sess.run(initial_state)\n",
    "            \n",
    "        for ii,(x,y) in enumerate(get_batches(comment_words,labels_train,batch_size),1):\n",
    "            feed = {inputs_:x,\n",
    "                    labels_:y,\n",
    "                    keep_prob_:keep_prob_training,\n",
    "                    initial_state:state}\n",
    "            \n",
    "            loss, state, _ = sess.run([cost,final_state,optimizer],feed_dict=feed)\n",
    "            iteration += 1\n",
    "            \n",
    "            if iteration%10==0:\n",
    "                print(\"\\rEpoch: {}/{}\".format(e, epochs),\n",
    "                      \"Iteration: {}/{}\".format(iteration, n_batches*epochs),\n",
    "                      \"Train loss: {:.3f}\".format(loss),end='')\n",
    "\n",
    "            if iteration%200==0:\n",
    "                val_acc.clear()\n",
    "                false_pos_list.clear()\n",
    "                total_correct_pos = 0\n",
    "                total_false_pos = 0\n",
    "                total_correct_neg = 0\n",
    "                total_false_neg = 0\n",
    "                \n",
    "                val_state = sess.run(initial_state)\n",
    "                sess.run(validation_metrics_init_op)\n",
    "                if iteration%2000==0:\n",
    "                    test_subset_x,test_subset_y = test_comment_words,labels_test\n",
    "                else:\n",
    "                    test_subset_x,test_subset_y = test_comment_words[:4096],labels_test[:4096]\n",
    "                \n",
    "                for x, y in get_test_batches(test_subset_x, test_subset_y, batch_size):\n",
    "                    feed = {inputs_: x,\n",
    "                            labels_: y,\n",
    "                            keep_prob_: 1,\n",
    "                            initial_state: val_state}\n",
    "                    \n",
    "                    auc_val, n_correct_pos, n_correct_neg, n_false_pos, n_false_neg, val_state, batch_acc = sess.run([auc, correct_pos, correct_neg, false_pos, false_neg, final_state,accuracy], feed_dict=feed)\n",
    "                    #print(predictions)\n",
    "                    val_acc.append(batch_acc/len(test_subset_y))\n",
    "                    auc_value = auc_val[1]\n",
    "                    total_correct_pos += n_correct_pos\n",
    "                    total_false_pos += n_false_pos\n",
    "                    total_correct_neg += n_correct_neg\n",
    "                    total_false_neg += n_false_neg\n",
    "                print(\"During epoch {}\".format(e))\n",
    "                print(\"  Val acc      : {}\".format(np.mean(val_acc)))\n",
    "                print(\"  AuC          : {}\".format(auc_value))\n",
    "                print(\"  Correct pos  : {}\".format('  '.join(['{:5}'.format(x) for x in total_correct_pos])))\n",
    "                print(\"  False neg    : {}\".format('  '.join(['{:5}'.format(x) for x in total_false_neg])))\n",
    "                print(\"  Correct neg  : {}\".format('  '.join(['{:5}'.format(x) for x in total_correct_neg])))\n",
    "                print(\"  False pos    : {}\\n\".format('  '.join(['{:5}'.format(x) for x in total_false_pos])))\n",
    "                \n",
    "                \n",
    "                saver.save(sess, \"{}/epoch{}iter{}.ckpt\".format(checkpoint_path,e,iteration))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "submit_data = pd.read_csv('data/test.csv')\n",
    "\n",
    "submit_comments = [clean_punc(comment) for comment in submit_data.comment_text]\n",
    "submit_comment_words = []\n",
    "#submit_comments = submit_comments[:204]\n",
    "for comment in submit_comments:\n",
    "    submit_comment_words.append ([word for word in comment.split()])\n",
    "    \n",
    "label_placeholder = np.zeros([len(submit_comments),n_labels])\n",
    "results = []\n",
    "\n",
    "with tf.Session(graph=graph) as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    \n",
    "    last_checkpoint = tf.train.latest_checkpoint(checkpoint_path)\n",
    "    saver.restore(sess,last_checkpoint)\n",
    "\n",
    "    for x, y in get_test_batches(submit_comment_words, label_placeholder, batch_size):\n",
    "        feed = {inputs_: x,\n",
    "            keep_prob_: 1,\n",
    "            initial_state: val_state}\n",
    "        #print (x)\n",
    "        pred, val_state = sess.run([predictions, final_state], feed_dict=feed)\n",
    "\n",
    "        for the_pred in pred:\n",
    "            results.append(the_pred)\n",
    "        \n",
    "        print(\"\\rDone: {}/{}\".format(len(results), len(label_placeholder)),end='')\n",
    "\n",
    "results = results[:len(submit_comment_words)]\n",
    "\n",
    "submission = pd.concat([submit_data['id'],pd.DataFrame(results,columns=['toxic','severe_toxic','obscene','threat','insult','identity_hate'])],axis=1)\n",
    "\n",
    "submission.to_csv('submission.csv',index=False, float_format='%.4f')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}