tf_classify.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This file is for running experiments to classify the Davidson hate speech data. It contains\n",
    "- one layer NN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tf_custom_models import RNN, OneLayerNN, SoftmaxClassifier\n",
    "from utility import train_and_eval_auc, HATEBASE_FIELDS\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.manifold import TSNE\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.metrics import jaccard_similarity_score\n",
    "from sklearn.metrics import roc_auc_score as AUC\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import os\n",
    "from os.path import join as pjoin\n",
    "\n",
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "import logging\n",
    "import json\n",
    "import itertools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "EMBEDDING_SIZE = 100\n",
    "STATE_SIZE = 100\n",
    "GLOVE_SIZE = 1193514\n",
    "GLOVE_PATH = \"data/glove/glove.twitter.27B.%dd.txt\" % EMBEDDING_SIZE\n",
    "\n",
    "HB_DIR = \"data/hatebase/\"\n",
    "EMBED_PATH = HB_DIR + \"embeddings.%dd.dat\" % EMBEDDING_SIZE\n",
    "HIDDEN_EMBED_PATH = HB_DIR + \"embeddings.hidden.%dd.dat\" % EMBEDDING_SIZE\n",
    "HB_PATH = HB_DIR + \"lexicon.csv\"\n",
    "HB_VOCAB_PATH = HB_DIR + \"vocab.dat\"\n",
    "\n",
    "DATA_DIR = \"data/twitter_davidson/\"\n",
    "VOCAB_PATH = DATA_DIR + \"data/twitter_davidson/vocab.dat\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x_path = pjoin(DATA_DIR, \"train.%dd.vec\") % EMBEDDING_SIZE\n",
    "train_y_path = pjoin(DATA_DIR, \"train.y\")\n",
    "test_x_path = pjoin(DATA_DIR, \"test.%dd.vec\") % EMBEDDING_SIZE\n",
    "test_y_path = pjoin(DATA_DIR, \"test.y\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_dataset(x_file, y_file):\n",
    "    with open(x_file, 'rb') as x_file, open(y_file, 'rb') as y_file:\n",
    "        data_x = pd.read_csv( x_file, header = None, quoting = 0, dtype = np.float64 )\n",
    "        data_y = pd.read_csv( y_file, header = 0, quoting = 0, usecols = ['hate_speech'], dtype = np.int32)\n",
    "        return data_x.values, data_y.values.ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x, train_y = load_dataset(train_x_path, train_y_path)\n",
    "test_x, test_y = load_dataset(test_x_path, test_y_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration 1000: loss: 0.155419558287 "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.85703685  0.14296315]\n",
      " [ 0.8153461   0.1846539 ]\n",
      " [ 0.7612256   0.2387744 ]\n",
      " ..., \n",
      " [ 0.80026539  0.19973461]\n",
      " [ 0.79036303  0.20963697]\n",
      " [ 0.76689819  0.23310181]]\n",
      "AUC: 0.631445309132\n"
     ]
    }
   ],
   "source": [
    "EPOCHS = 1000\n",
    "tf.reset_default_graph()\n",
    "nn = OneLayerNN(max_iter=EPOCHS, h=100)\n",
    "train_and_eval_auc( train_x, train_y, test_x, test_y, nn )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# rnn = RNN(max_iter=EPOCHS)\n",
    "# train_and_eval_auc( train_x, train_y, test_x, test_y, rnn )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}