sentiment_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#Sentiment analysis for tweets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "@Ilkan Esiyok - 13 Apr 2016"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>TIMING STARTS</b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import time\n",
    "start_time = time.time() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "TRAIN_DATA_FILE_PATH = ''\n",
    "DEV_DATA_FILE_PATH = ''\n",
    "TEST_DATA_FILE_PATH = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from __future__ import division\n",
    "%matplotlib inline\n",
    "import re\n",
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "from nltk.corpus import words\n",
    "import json\n",
    "import nltk\n",
    "import numpy as np\n",
    "import itertools\n",
    "from nltk.tokenize.regexp import WordPunctTokenizer\n",
    "from nltk.corpus import stopwords\n",
    "from sklearn.feature_extraction import DictVectorizer\n",
    "from sklearn.metrics import accuracy_score, classification_report\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from scipy import sparse\n",
    "from random import randrange\n",
    "from nltk.corpus import brown\n",
    "import math\n",
    "import operator\n",
    "import gensim \n",
    "from nltk.data import find\n",
    "import operator\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from nltk.corpus import sentiwordnet as swn\n",
    "from nltk.corpus import wordnet as wn\n",
    "from nltk.corpus import opinion_lexicon\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>HELPER METHODS</b>\n",
    "\n",
    "Includes the methods which are required for preprocessing. The explanations are given for each method on top of their definition."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class Helper:\n",
    "    \n",
    "    corpus = None\n",
    "    \n",
    "    def __init__(self, corpus):\n",
    "        self.corpus = corpus\n",
    "    \n",
    "    #lemmatization only used in max_match method\n",
    "    def lemmatize(self, word):\n",
    "        lemmatizer = WordNetLemmatizer()\n",
    "        lemma = lemmatizer.lemmatize(word,'v')\n",
    "        if lemma == word:\n",
    "            lemma = lemmatizer.lemmatize(word,'n')\n",
    "        return lemma\n",
    "\n",
    "\n",
    "    #max_match method is used in order to process hashtags\n",
    "    def max_match_applier(self, hashtag):\n",
    "        hashtag = hashtag.group().lstrip('#')\n",
    "        return self.max_match(hashtag)\n",
    "\n",
    "    #max_match method, basically works as :\n",
    "    #moving through words from last character to first character and stopping when a word matches-> a word in the corpus\n",
    "    def max_match(self, hashtag):\n",
    "        #you should convert it to a python set before you use it TODO\n",
    "\n",
    "        sentence = re.split(r'(\\d+)', hashtag)#split with the numbers, (if there is)\n",
    "        word_list = ''\n",
    "        for word in sentence:\n",
    "            if word.isdigit():\n",
    "                word_list += word + ' '\n",
    "            else:\n",
    "                length = len(word)\n",
    "                i = 0\n",
    "                while i < length :\n",
    "                    for j in range(length, i, -1):\n",
    "                        ht = word[i : j]\n",
    "                        if(self.lemmatize(ht.lower()) in corpus):#lower operation here doesn't change the original letter\n",
    "                            word_list += ht + ' '\n",
    "                            i = j-1\n",
    "                            break\n",
    "                    i+=1\n",
    "\n",
    "        return word_list.rstrip()\n",
    "\n",
    "    #removes hashtags in the text\n",
    "    def remove_hashtags(self, sentence):\n",
    "        regexp = r'(^|\\W)#(\\w+)'\n",
    "        return re.sub(regexp, self.max_match_applier, sentence)    \n",
    "\n",
    "    #tokenizes the words in sentences\n",
    "    def word_tokenizer(self, sentence, wordpunct_tokenizer):\n",
    "        return wordpunct_tokenizer.tokenize(sentence)\n",
    "\n",
    "    #splits text into sentences\n",
    "    def sent_segmenter(self, tweet, sent_segmenter):   \n",
    "        return sent_segmenter.tokenize(tweet)\n",
    "\n",
    "    #removes user names from the text\n",
    "    def remove_usernames(self, tweet):\n",
    "        regexp = r'(^|\\W)@(\\w+)'\n",
    "        return re.sub(regexp, '', tweet, flags=re.IGNORECASE)   \n",
    "\n",
    "    #improvement method - removes user names from the text\n",
    "    def remove_usernames_imp(self, tweet):\n",
    "        regexp = r'(^|\\W)@(\\w+)'\n",
    "        return re.sub(regexp, ' ', tweet, flags=re.IGNORECASE)\n",
    "\n",
    "    #removes urls from text\n",
    "    def remove_urls(self, tweet):\n",
    "        regexp = r'(^|\\W)(http|https)://.+?($|\\s)'\n",
    "        return re.sub(regexp, '', tweet, flags=re.IGNORECASE)\n",
    "\n",
    "    #improvement method - removes urls from text\n",
    "    def remove_urls_imp(self, tweet):\n",
    "        regexp = r'(^|\\W)(http|https)://.+?($|\\s)'\n",
    "        return re.sub(regexp, ' ', tweet, flags=re.IGNORECASE)\n",
    "\n",
    "    #improvement method - this method only removes . and , characters from the text\n",
    "    def remove_punct_imp(self, tweet):\n",
    "        regexp = r'[\\.,]'\n",
    "        return re.sub(regexp, ' ', tweet, flags=re.IGNORECASE)\n",
    "\n",
    "    #lower case\n",
    "    def lower_case(self, tweet):\n",
    "        return tweet.lower()\n",
    "\n",
    "    #remove stop words in text\n",
    "    def remove_stop_words(self, tweet, stop_words_set, f_dict):\n",
    "        for word in tweet:    \n",
    "            if word not in stop_words_set:\n",
    "                f_dict.append(word)\n",
    "        return f_dict  \n",
    "\n",
    "    #improvement method - removes numbers in text, except ones which are joined with letters like '25th'\n",
    "    def remove_numbers_imp(self, tweet):\n",
    "        regexp = r'(^|\\b).[0-9]+(\\b|$)'\n",
    "        return re.sub(regexp, ' ', tweet, flags=re.IGNORECASE)\n",
    "\n",
    "    #improvement method - removes multiple spaces in text\n",
    "    def remove_spaces_imp(self, tweet):\n",
    "        regexp = r'  +'\n",
    "        return re.sub(regexp, ' ', tweet, flags=re.IGNORECASE)\n",
    "\n",
    "    #improvement method - removes single punctiations in the text, except ! and ? which might be informative in sentiment analysis context\n",
    "    def remove_spec_puncts_imp(self, tweet):\n",
    "        #! and ? might be informative, so we won't remove them\n",
    "        regexp = r'(^|\\s)\\(|\\)|\\[|\\]|\\{|\\}|\\*|&|@|-|\\\\|\\/|=|\\+|:|;|<|>|_|\\^|%|\\$|#|(\\s|$)'\n",
    "        return re.sub(regexp, ' ', tweet, flags=re.IGNORECASE)\n",
    "\n",
    "    #improvement method - removes words which are like letters in the text : example 'm', 'a'\n",
    "    def remove_single_words_imp(self, tweet):\n",
    "        regexp = r'(^|\\b)[a-zA-Z0-9](\\b|$)'\n",
    "        return re.sub(regexp, ' ', tweet, flags=re.IGNORECASE)\n",
    "\n",
    "    #returns the frequency of words in text\n",
    "    def get_BOW(self, text):\n",
    "        BOW = {}\n",
    "        for word in text:\n",
    "            BOW[word] = BOW.get(word,0) + 1\n",
    "        return BOW\n",
    "\n",
    "    #removes words that appear less than n times in whole text\n",
    "    def remove_less_than_n_times(self, BOW_list, n):\n",
    "        temp_dict = sum((Counter(x) for x in BOW_list ), Counter())      \n",
    "        return BOW_list\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#the corpus helps us to identify if the word is in the corpus\n",
    "corpus = set(words.words())\n",
    "helper = Helper(corpus)#cretes an instance from Helper class"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>PREPROCESSING</b>\n",
    "\n",
    "Includes preprocessing methods and works with helper methods above."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#tokenizes the words in sentence\n",
    "wordpunct_tokenizer = WordPunctTokenizer()\n",
    "#splits text into sentences\n",
    "sent_segmenter = nltk.data.load('tokenizers/punkt/english.pickle')\n",
    "#stop_words corpus\n",
    "stop_words_set = set(stopwords.words())\n",
    "#vectorizer, helps to create sparse matrixes\n",
    "vectorizer = DictVectorizer() \n",
    "\n",
    "class Pre_Processor:\n",
    "\n",
    "    helper = None\n",
    "    \n",
    "    def __init__(self, helper):\n",
    "        self.helper = helper\n",
    "    \n",
    "    #original preprocess method\n",
    "    def preprocess(self, tweet):\n",
    "        tweet = helper.remove_urls(tweet)\n",
    "        tweet = helper.remove_usernames(tweet)\n",
    "        tweet = helper.remove_hashtags(tweet)\n",
    "        tweet = helper.lower_case(tweet)\n",
    "        sentences = helper.sent_segmenter(tweet, sent_segmenter)\n",
    "        words_list = []\n",
    "        for sentence in sentences:#we may have multiple sentences\n",
    "            words_list.append(helper.word_tokenizer(sentence, wordpunct_tokenizer))\n",
    "\n",
    "        return words_list\n",
    "\n",
    "    #improved preprocess method\n",
    "    def preprocess_improvement(self, tweet):\n",
    "        tweet = helper.remove_urls_imp(tweet)\n",
    "        tweet = helper.remove_usernames_imp(tweet)\n",
    "        tweet = helper.remove_hashtags(tweet)\n",
    "        tweet = helper.lower_case(tweet)\n",
    "\n",
    "        sentences = helper.sent_segmenter(tweet, sent_segmenter)\n",
    "\n",
    "        words_list = []\n",
    "        for sentence in sentences:#we may have multiple sentences\n",
    "            sentence = helper.remove_punct_imp(sentence)\n",
    "            #sentence = helper.max_match(sentence)\n",
    "            sentence = helper.remove_numbers_imp(sentence)\n",
    "            sentence = helper.remove_single_words_imp(sentence)\n",
    "            sentence = helper.remove_spec_puncts_imp(sentence)\n",
    "            sentence = helper.remove_spaces_imp(sentence)\n",
    "            words_list.append(helper.word_tokenizer(sentence, wordpunct_tokenizer))\n",
    "\n",
    "        return words_list\n",
    "\n",
    "    #original preprocess_file, gets the file and returns tweets, classes and dictionary\n",
    "    def preprocess_file(self, filename):\n",
    "\n",
    "        tweets =[] \n",
    "        labels = []\n",
    "        word_dict = {}\n",
    "        f = open(filename)\n",
    "        for line in f:\n",
    "            tweet_dict = json.loads(line)\n",
    "            words_list = self.preprocess(tweet_dict['text'])\n",
    "            tweets.append(words_list)\n",
    "            labels.append(int(tweet_dict[\"label\"]))\n",
    "            for words in words_list:\n",
    "                for word in words:\n",
    "                    word_dict[word] = word_dict.get(word,0) + 1\n",
    "\n",
    "        return tweets, labels, word_dict\n",
    "\n",
    "    #improved preprocess_file, gets the file and returns tweets, classes and dictionary  \n",
    "    #calls preprocess_improvement method,\n",
    "    def preprocess_file_improvement(self, filename):\n",
    "\n",
    "        tweets =[] \n",
    "        labels = []\n",
    "        word_dict = {}\n",
    "        f = open(filename)\n",
    "        for line in f:\n",
    "            tweet_dict = json.loads(line)\n",
    "            words_list = self.preprocess_improvement(tweet_dict['text'])\n",
    "            tweets.append(words_list)\n",
    "            labels.append(int(tweet_dict[\"label\"]))\n",
    "            for words in words_list:\n",
    "                for word in words:\n",
    "                    word_dict[word] = word_dict.get(word,0) + 1\n",
    "\n",
    "        return tweets, labels, word_dict\n",
    "\n",
    "    #creates feature dictionaries from tweets\n",
    "    #if remove_stop_words is True : it removes stop words from text\n",
    "    #if n is the number of minimum frquency of a word in whole file, if n =0 this control isn't applied \n",
    "    def convert_to_feature_dicts(self, tweets, remove_stop_words, n, wordDict):\n",
    "        feature_dicts = []\n",
    "\n",
    "        for tweet in tweets:\n",
    "            long_sentence = list(itertools.chain(*tweet))\n",
    "            if remove_stop_words:\n",
    "                for word in long_sentence:\n",
    "                    if word in stop_words_set:\n",
    "                        long_sentence.remove(word)\n",
    "                    else:\n",
    "                        if n > 0 and int(wordDict[word]) < n :\n",
    "                            long_sentence.remove(word)\n",
    "            else:\n",
    "                if n > 0 and wordDict[word] < n:\n",
    "                            long_sentence.remove(word)\n",
    "\n",
    "            BOW = helper.get_BOW(long_sentence)\n",
    "            feature_dicts.append(BOW)\n",
    "\n",
    "        return feature_dicts\n",
    "\n",
    "    #classifiers uses this method to get predictions, accuracy_score and report\n",
    "    def classifying(self, classifier, data, labels ):\n",
    "\n",
    "        predictions = classifier.predict(data)\n",
    "        print accuracy_score(labels, predictions)  \n",
    "        print classification_report(labels, predictions) \n",
    "        \n",
    "\n",
    "        \n",
    "    #applies grid search to a classsifier and finds out best parameters\n",
    "    def tuning(self, classifier, parameters, dev_data, dev_labels):\n",
    "        import matplotlib.pyplot as plt\n",
    "        fig = plt.figure()\n",
    "        \n",
    "        \n",
    "        gs = GridSearchCV(classifier, parameters, n_jobs=1)\n",
    "        gs.fit(dev_data, dev_labels)\n",
    "        #print gs.grid_scores_ \n",
    "        print 'Best score: ' + str(gs.best_score_)#finds the best score according to the params\n",
    "        best_parameters = gs.best_estimator_.get_params()\n",
    "        print 'Parameters : '\n",
    "        for p_name in sorted(parameters.keys()):\n",
    "            print(\"  %s: %r\" % (p_name, best_parameters[p_name]))\n",
    "        \n",
    "        y = []\n",
    "        x = []\n",
    "        #gs.grid_scores_ contains every possiblity\n",
    "        for param_s, mean_score, _ in gs.grid_scores_: \n",
    "#             print \"%0.4f for %r\" % (mean_score, param_s)\n",
    "            y.append(mean_score)\n",
    "            x.append(param_s)\n",
    "#reference for bar chart : \n",
    "#http://stackoverflow.com/questions/11617719/how-to-plot-a-very-simple-bar-chart-python-matplotlib-using-input-txt-file\n",
    "        N = len(y)\n",
    "        width = 0.45\n",
    "        indice = np.arange(len(y))\n",
    "        plt.bar(indice, y, width=width)\n",
    "        plt.xticks(indice + width / 2, x)\n",
    "\n",
    "        fig.autofmt_xdate()#for showing x axis better\n",
    "        plt.show()\n",
    "        \n",
    "        #predictions = gs.predict(test_data)\n",
    "        #print classification_report(test_labels, predictions)    \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pre_processing = Pre_Processor(helper)#creates an instance from Pre_Processor class"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>PRE - PROCESSING STARTS</b>\n",
    "\n",
    "We will call preprocessing block and send the data file paths."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#training data preprocessing\n",
    "train_tweets, train_labels, train_word_dict = pre_processing.preprocess_file(TRAIN_DATA_FILE_PATH)\n",
    "#converting tweets to list of vector dictionaries\n",
    "train_feature_dicts = pre_processing.convert_to_feature_dicts(train_tweets, True, 2, train_word_dict) \n",
    "#creating sparse matrix from dictionaries\n",
    "train_sparse = vectorizer.fit_transform(train_feature_dicts).toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#dev data preprocessing -- same process except we don't remove stop-words and words which are less than n\n",
    "dev_tweets, dev_labels, dev_word_dict = pre_processing.preprocess_file(DEV_DATA_FILE_PATH)\n",
    "dev_feature_dicts = pre_processing.convert_to_feature_dicts(dev_tweets, False, 0, dev_word_dict)\n",
    "dev_sparse = vectorizer.transform(dev_feature_dicts)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>TUNING AND CLASSIFYING</b>\n",
    "\n",
    "We will train both DecisonTreeClassifier and LogisticRegressionClassifier, and will try to identify best available parameters for the best classifications."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.428649535265\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "         -1       0.29      0.25      0.27       360\n",
      "          0       0.42      0.56      0.48       700\n",
      "          1       0.52      0.40      0.45       769\n",
      "\n",
      "avg / total       0.44      0.43      0.42      1829\n",
      "\n",
      "--- DT Classification time : 137.391000032 seconds ---\n"
     ]
    }
   ],
   "source": [
    "#decision tree classifier\n",
    "time1 = time.time()\n",
    "dt_classifier = DecisionTreeClassifier()\n",
    "dt_classifier.fit(train_sparse, train_labels)\n",
    "pre_processing.classifying(dt_classifier, dev_sparse, dev_labels)\n",
    "print(\"--- DT Classification time : %s seconds ---\" % (time.time() - time1))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score: 0.453253143794\n",
      "Parameters : \n",
      "  max_depth: 13\n",
      "  max_features: 'auto'\n",
      "  random_state: 28\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmQAAAGECAYAAACYvTyjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XvcZWP9//HX+76HGYzzsZzPp1AOUyJfUg4ldBJKJRUK\nlZIOdJQUoi+llCQ/pVKOoaJukhKppJxDTvF1ypmZ8fn9cV17Zs12z8w9M3vfa133ej8fj+sxe+29\n1pr3uLa9rrXWta5LEYGZmZmZ1Weg7gBmZmZmbecGmZmZmVnN3CAzMzMzq5kbZGZmZmY1c4PMzMzM\nrGZukJmZmZnVzA0ysxaRtIOkGyXdLOnQYT7/H0mPSro2l8PqyGlm1jbj6g5gZqND0gBwIrAtcC9w\ntaRzI+LGrlUvj4idRz2gmVmL+QqZWXtMAm6JiDsjYjJwJrDLMOtpdGOZmZkbZGbtsTxwV2X57vxe\nt80l/VXSLyStNzrRzMzazbcszazqz8BKEfGUpB2Bc4C1as5kZjbmuUFm1h73ACtVllfI700TEU9U\nXl8k6ZuSloiIh6vrSfIkuGZmcyEihu0W4luWZu1xNbCGpJUlzQ/sDpxXXUHSspXXkwB1N8Y6ImKW\n5bOf/exs12lCcU7nbHJxzrGVcVZ8hcysJSJiqqQDgF+RTsZOiYgbJO2bPo6TgbdI2h+YDDwNvK2+\nxGZm7eEGmVmLRMTFwNpd73278vobwDdGO5eZWdv5lqWZ9cXWW29dd4QRcc7ecs7ecs7eaXpGze6e\npplZN0nh3w4zszkjiXCnfjMzM7NmcoPMzMzMrGZukJmZmZnVzA0yMzMzs5q5QWZmZmZWMzfIzMzM\nzGrmBpmZmZlZzdwgMzMzM6uZG2RmZmZmNXODzMzMzKxmbpCZmZmZ1cwNMjMzM7OauUFmZmZmVjM3\nyMzMzEbZcsutgqSelOWWW6Xuf471gCKi7gxmVhhJ4d8Os7knCejV/0PC/z+WQRIRoeE+8xUyMzMz\ns5q5QWZmZmZWMzfIzMzMzGrmBpmZmZlZzdwgMzMzM6uZG2RmZmZmNXODzMzMzKxmbpCZ2ZjlwTfN\nrBQeGNbM5lgpA8N68E1rKn8328kDw5qZmZk1mBtkZmY2Ir26Bezbv9ZrY6F7gm9Zmtkc8y3Ldurd\nf0//t/R3s7dK+e/pW5ZmZmZmDeYGmZmZmVnN3CAzMzMzq5kbZGYtImkHSTdKulnSobNYbzNJkyW9\naTTzmVmzjIXO8qVwg8ysJSQNACcC2wPrA3tIWmcm6x0F/HI2+/OPdI/4oGdNdf/9d5I6y897Sfuy\nmXGDzKw9JgG3RMSdETEZOBPYZZj1DgTOAh6Y9e78I90rPuj1jhu3VqpxdQcws1GzPHBXZfluUiNt\nGkkvBnaNiG0kzfCZWQmmN257sa9hRycw6ws3yMys6nig2rdsFkekz1Veb52LmZl1DA0NMTQ0NKJ1\nPTCsWUtIegXwuYjYIS9/AoiI+EplnX91XgJLAU8C74+I87r2FaUMwuicvVPCwLDt+28JzllWzpkN\nDOsrZGbtcTWwhqSVgfuA3YE9qitExGqd15JOBc7vboyZmVnvuUFm1hIRMVXSAcCvSA/0nBIRN0ja\nN30cJ3dvMuohzcxayrcszWyO+ZZlb7UvZwkZwTmds9c8l6WZmZlZg7lBZmZmZlYzN8jMzMzMauYG\nmZmZmVnN3CAzMzMzq5kbZGZmZmY1c4PMzMzMrGZukJmZmZnVzA0yMzMzs5q5QWZmZmZWMzfIzMzM\nzGrmBpmZmZlZzdwgMzMzM6uZG2RmZmZmNXODzMzMzKxmbpCZmZmZ1cwNMjMzM7OauUFmZmZmVjM3\nyMzMzMxq5gaZmZmZWc3cIDMzMzOrmRtkZmZmZjVzg8zMzMysZm6QmZmZmdXMDTIzMzOzmrlBZmZm\nZlYzN8jMzMzMauYGmZmZmVnN3CAzMzMzq5kbZGZmZmY1c4PMzMzMrGZukJm1iKQdJN0o6WZJhw7z\n+c6S/ibpL5KukfTqOnKambWNIqLuDGY2CiQNADcD2wL3AlcDu0fEjZV1FoyIp/LrDYCzI2KNYfYV\n0KvfDtGv3yFJOGfv9C5nCRnBOZ2z1yQRERruM18hM2uPScAtEXFnREwGzgR2qa7QaYxlE4EHRzGf\nmVlruUFm1h7LA3dVlu/O781A0q6SbgAuBA4apWxmZq02ru4AZtYsEXEOcI6kLYHTgbWHX/Nzlddb\n52JmZh1DQ0MMDQ2NaF33ITNrCUmvAD4XETvk5U8AERFfmcU2twGTIuKhrvfdh6yH2pezhIzgnM7Z\na+5DZmaQOvGvIWllSfMDuwPnVVeQtHrl9cYA3Y0xMzPrPd+yNGuJiJgq6QDgV6STsVMi4gZJ+6aP\n42TgzZLeCTwHPAm8rb7EZmbt4VuWZjbHfMuyt9qXs4SM4JzO2Wu+ZWlmZmbWYG6QmZmZmdXMDTIz\nMzOzmrlBZmZmZlYzN8jMzMzMauYGmZmZmVnN3CAzMzMzq5kbZGZmZmY1c4PMzMzMrGZukJmZmZnV\nzA0yMzMzs5q5QWZmZmZWMzfIzMzMzGrmBpmZmZlZzdwgMzMzM6uZG2RmZmZmNXODzMzMzKxmbpCZ\nmZmZ1cwNMjMzM7OauUFmZmZmVjM3yMzMzMxq5gaZmZmZWc3cIDMzMzOrmRtkZmZmZjVzg8zMzMys\nZm6QmZmZmdXMDTIzMzOzmrlBZmZmZlYzN8jMzMzMauYGmZmZmVnN3CAzMzMzq5kbZGYtImkHSTdK\nulnSocN8vqekv+VyhaQN6shpZtY2bpCZtYSkAeBEYHtgfWAPSet0rfYvYKuI2Ag4AvjO6KY0M2sn\nN8jM2mMScEtE3BkRk4EzgV2qK0TEHyPiv3nxj8Dyo5zRzKyV3CAza4/lgbsqy3cz6wbXe4GL+prI\nzMwAGFd3ADNrHknbAHsDW9adxcysDdwgM2uPe4CVKssr5PdmIGlD4GRgh4h4ZOa7+1zl9da5mJlZ\nx9DQEENDQyNaVxHR3zRm1giSBoGbgG2B+4A/AXtExA2VdVYCLgX2iog/zmJfAb367RD9+h2ShHP2\nTu9ylpARnNM5e00SEaHhPvMVMrOWiIipkg4AfkXqP3pKRNwgad/0cZwMHA4sAXxT6RduckRMqi+1\nmVk7+AqZmc0xXyHrrfblLCEjOKdz9tqsrpD5KUszMzOzmrlBZmZmZlYzN8jMzMzMauYGmZmZmVnN\n3CAzMzMzq5kbZGZmZmY1c4PMzMzMrGZukJmZmZnVzA0yMzMzs5q5QWZmZmZWMzfIzMzMzGrmBpmZ\nmZlZzdwgMzMzM6uZG2RmZmZmNXODzMzMzKxmbpCZmZmZ1cwNMjMzM7OauUFmZmZmVjM3yMzMzMxq\n5gaZmZmZWc3cIDMzMzOrmRtkZmZmZjVzg8zMzMysZm6QmZmZmdXMDTIzMzOzmrlBZmZmZlYzN8jM\nzMzMauYGmZmZmVnN3CAzMzMzq5kbZGZmZmY1c4PMzMzMrGZukJm1iKQdJN0o6WZJhw7z+dqSrpT0\njKSD68hoZtZG4+oOYGajQ9IAcCKwLXAvcLWkcyPixspqDwEHArvWENHMrLV8hcysPSYBt0TEnREx\nGTgT2KW6QkQ8GBF/BqbUEdDMrK3cIDNrj+WBuyrLd+f3zMysZm6QmZmZmdXMfcjM2uMeYKXK8gr5\nvbn0ucrrrXMxM7OOoaEhhoaGRrSuIqK/acysESQNAjeROvXfB/wJ2CMibhhm3c8CT0TEsTPZV0Cv\nfjtEv36HJOGcvdO7nCVkBOd0zl6TRERouM98hcysJSJiqqQDgF+RuiucEhE3SNo3fRwnS1oWuAZY\nGHhe0oeA9SLiifqSm5mNfb5CZmZzzFfIeqt9OUvICM7pnL02qytk7tRvZmZmVjM3yMzMzMxq5gaZ\nmZmZWc3cIDMzMzOrmRtkZmZmZjVzg8zMzMysZm6QmZmZmdXMDTIzMzOzmrlBZmZmZlYzN8jMzMzM\nauYGmZmZmVnN3CAzMzMzq5kbZGZmZmY1c4PMzMzMrGZukJmZmZnVzA0yMzMzs5q5QWZmZmZWMzfI\nzMzMzGrmBpmZmZlZzdwgMzMzM6uZG2RmZmZmNXODzMzMzKxmbpCZmZmZ1cwNMjMzM7OauUFmZmZm\nVjM3yMzMzMxq5gaZmZmZWc3cIDMzMzOrmRtkZmZmZjVzg8zMzMysZm6QmZmZmdXMDTIzMzOzmrlB\nZtYiknaQdKOkmyUdOpN1/lfSLZL+Kumlc/+3Dc39pqNqqO4AIzRUd4ARGqo7wAgN1R1ghIbqDjBC\nQ3UHGIGhugPMkhtkZi0haQA4EdgeWB/YQ9I6XevsCKweEWsC+wLfmvu/cWjuNx1VQ3UHGKGhugOM\n0FDdAUZoqO4AIzRUd4ARGqo7wAgM1R1gltwgM2uPScAtEXFnREwGzgR26VpnF+AHABFxFbCopGVH\nN6aZWfu4QWbWHssDd1WW787vzWqde4ZZx8zMekwRUXcGMxsFkt4MbB8R78/L7wAmRcRBlXXOB74c\nEVfm5UuAj0fEtV378g+HmdlciAgN9/640Q5iZrW5B1ipsrxCfq97nRVns85Mf1DMzGzu+JalWXtc\nDawhaWVJ8wO7A+d1rXMe8E4ASa8AHo2I+0c3pplZ+/gKmVlLRMRUSQcAvyKdjJ0SETdI2jd9HCdH\nxIWSXifpVuBJYO86M5uZtYX7kJmZmZnVzLcszczMzGrmBpmZzRNJRXTwLyWn9U4pdV5KTusvN8jM\nbI5Jep+k70haafZr16egnNtKWqruHLNTQs6C6ryUnI2vcygn56y4QWZmI9J1Fr82MAH4NLBuPYmG\nV0rODklfAr4JfCZPXdXIKyal5KSAOs8an7OUOi8l5+y4QWZmsyVpMWDa2WdEfCwi9gIGgc0lLVBb\nuIqCclYPFl8DXgrcBOwoaamICEmD9aSbroSckpaQtEpnucF1XkrOan0eRwPrHMrJOSfcIDOzWZL0\neeAS4CRJ+3QdOH4BbAXsJemdkmobSqegnB8Hjpe0M0BEPBQRT5PGiXueND4cQK1n+CXklHQY8Fvg\nKEmfaHCdl5LzU8Apkt4HEBEPNq3OoZycc8oNMjMblqTxkk4B1gG2Bn4MvAx4RWediDgb+H/AbsD7\ngamSRvV3paCcL5P0Z2A94B/AvpI+WFnlWuBi4E2SngS2G818HSXkzHV+Mum237bAqcBawCKdqyIN\nqfNScm4s6WrS/0NnAbtJendlldrrHMrJObc8MKyZDSsinpX0I+CfEfGEpAuB9wBTIN0yiIipEfFr\nSf8DXB9pYMNRHdywlJykSdqPjYgf5lyPANtIGhcRUyJiCnBxvk14YERcOMr5ismZ6/zIiLgjZxwg\nTfM1ntQv60lJqrvOS8kJLAB8JCKuyDk3JzVuOv+O2us8KyXnXPEVMjMDUv8rScdIelfnvYi4JCLu\nlTQ+Ip4EHgcWzp9NrWw+GXippFWdc4ackypv/YZ09t7xYuCZiJjS1VfraeBRSa/K++nrbZcScnbX\neW7E3JFf70nq33Y/cBTwBUhTT+TNa/tuNjxntc6vjIgrJC0g6TjgUOAASV/N63faCnV/NxuZs1fc\nIDMzJC0PDJFu870tL0/rOJvP9Bcmnd1fPcwujgU2BJZ0TpC0O3AX8AdJ4/PbT0fEw10Hh0cgHZQr\n798EXAe8Jh/Q+3a1pIScw9V5V4ft3wDr5Q7yXwFWk7RhZRe1fTcbmnO4Ou/U6XLANTnDkcDekraI\niOfz53V/NxuXs5fcIDMzIuIe4BOkzrB/BQ7I71evLq0L3BgR/ydpJ+UOtfkAvSLpYDLROQFYBtgS\n+BFwYn5vIGftHBy2B36Xs23UeT8i7gMWB54fhQNJ43POrs4j4j+Vv/9h0tXRO3PeARry3WxKToav\nc+WMt0fEGRHx33xl73vAtCtUDfhuNjFn70SEi4tLiwt5TtvK8hbA+cDL8/J8+c8dSWedpwNXAq+s\nbLMh8Pq25+xkBJbIfy4DPAasm5cH85+LAueQHj74IWnC96XyZ2vmf9fmzjnbOu/kVM5zJvB9Ul+j\nzr+xCd/N2nOOtM4r669FunW9cdd7jfhu1p2zL//2ugO4uLg0qwCLAIcAp3W9fxwwFXh33RkLy3kE\n8Luu95bPGf8BHFR3xlJyzqLO3wfcCny47oyF5Ryuztck9XX7O3BA3RlLyjmvpdMSNbMxTtKaEXHL\nCNYTsD7wUeBnpM7G15Ce/Ho+Ih7N6w3GjLcKW5WzkkExve/KDJ9F/oGVdDfw9oi4TNKLSE/Y7Qsc\nERFPtD3nPNT5c6SroU+R7rI+3q+MJeWsZJibOp8K7AGcGhGP9SNbiTlHRd0tQhcXl/4XYAfgG8A6\nlfdWms02x5EGWfw9sGzl/XFtzkm6zfQdYO1hPuu+ddW5jbo96WD8R+DjXesMtjxnL+u8LxlLydmD\nOj9klOq8iJyjXdyp32wMqzwGfitwN6l/FZI2Ad4jaYmZbPcZYGdgt4jYIiLu73wWaayfVubM+306\nZ/xYzjBO0tGS1ol8dKisO1nSksCewAPA8RHx1Up+RZ+ukDQ9Z5/qvB9XxYrImfc7r3V+dCV/k7+b\no5JztPmWpVlLSHo18FykcXwWjnzbpGsdRURIWjki7qy837dbKyXmlLQscAzwyYi4W9IrI+LKmay7\nGanD8bcr7w3EMLdoWpyz8XVeSs6C6ryInKPJDTKzMSSfSf4C2DEiHlGaVmR+4KKIuGs2275gvJ4+\n9sUpJecrgfER8du8vC5wT0Q8JmnBiHhqDvc3rk9XGBufs6A6LyVn4+u8pJxN4AaZ2RigNCGx8uX9\nn5IeE/8r8Dbgb6TR1j8aEf+qHiDyWeqewMmRRrh3Tma4ynEsqQ/Q+cDnSZ3cHwPeFBFPSpovIiZX\ntntJRFzf73wl5SyozkvJ2fg6Lylnk7gPmVnhJK1JGr9osiSRJijeCtgoIrYEPkIatuAQeEH/lcWB\n34/SgaTxOSUNSFqLNNwDwMnAUqQD7g8iYnPgUeBb+fOplW0FvFczjr7e9pyNr/NSchZU50XkbCJf\nITMrVOUMdALpFsu/ge2ATUmjhb8jIjbJZ/5rkfprHBMRvxnNy/6l5MxZFwHeBCwGLEG6QrIMsBdw\ndESck9f5N7B1RPy1jlsoTc9ZSp2XkjNnbXSdl5aziXyFzKwwSgY6fVUi4hnSLYDXAm+OiPsi4jhg\nnKQ98g/dbcAlwCfzNn3/8Sso54Dyk3SRxjNaljQQ5TbARaRpWW4AlpK0ZF7naNJo6y/ImM/yW5mz\noDovJWfj67yknE3nBplZQXLflYiI5yVtJOn9klYEPkiaJmj1fPYJ8AXgo5ImRMSzwBnAh51zhpwD\nEfF8ztm5xXIRcBZwLml8o+eAC4HNSfNkEhFfAu6QtEz3waNzkG9bzoLqvJScja/zknKWwLcszQoj\naSLweuBg4A5gPuBU4F7gKGC/iLgtr3sFcGFEHOmcM805nnS2viNppP9TgVtIo6xfFRGn5/W+DjwB\nnBQRdzvnsBlLqfNScja+zkvK2XTj6g5gZjOn4cfaOQPYhDRx7l2StgZOBCaRfgT3krQoaRDFnYHR\n6BRdZE5JOwJvAR4B1gHeChxPmhz6RmA9SeuRDthDpBHG76ls368hDRqfs9Q6LyVnE+u8pJwl8i1L\nswbr/PBJ2kLStvntj5L6aIzP6wwBVwFvJj1W/iTwIuC8iHg4Ip7t9O9oc87qgUTSMvntR0kHjoci\nYmpEnAlcDxwE/CBn/AVwOHBJRPywejul3we8Jucsoc5LyVlKnZeSs1S+ZWnWILkvy3IRcXVeXpLU\nIXYCcDPpx+1TwLeBJSPiTXm9/wUujogLVRnXR3rhQJUty7kwaWT1Z/PyWqR+QeNIB+CTSEMaLEaa\nQPsRSa8Avkiau1DAWhHxz8o+e561hJwF1XkpORtf5yXlHBOiARNquri4pAK8B/gXebJc4A3Agfn1\n90idjlchHVyeAr4OfDxvs1nXvgbanBPYmNSXZYu8vBBwds66Hmmwz4NIV0KGgM8CKwCnAF8cZn/9\nmmi5lJyNr/NSchZU50XkHCvFtyzNGkDTHxn/HnA/8N780ZKk/ixXkAZQ3Cki7oj0mP5HgHeTHsff\nJPIVgY7owzxvJeSsZLyWdMDdWNJSwMrAQ/m9U4DLgdMi4j7gh6Sxkw7L+Y/t3m/0aXLtgnI2ts5L\nyVlgnTc655hTd4vQxcUlFdKYPacCFwAPkwZTfDXwW2CPynq7kCbahdRX49359XjnnCHnS4EfARcD\nrwIWJB14rwJWr6y3Kak/7fnAPpX3+3YVp7ScBdV5KTkbX+cl5RwrxVfIzBog99P4AunAsTvpVsBn\n85+/AfaWtKWkH5A6HQ/mTQ8DPgQQuY+Hc4KkA0gH5rNIV0jeQbqtchxwf0TcJmklST8Bdot0JeQM\n4D2SVp7Jk3mtzFlQnZeSs/F1XlLOMaXuFqGLi0sArE16ImmJvDwReBB4WV7+MHAC8LnKNnLOmeY8\nEXhvfr0ecBrpcfwJpD4wZwB/Bz7btd0bnbPYOi8lZ+PrvKScY6l4HDKzUTKbJ4seIY1gvaykRyLi\nCUlXkybgfXlEHN/1yHnfxu4pIeeszr6VBql8CJhP0sSI+Kek+4EDgT9ExBvzk3eKiAfzNuMiYkpE\nnN3SnI2v81JyFlTnReRsE9+yNBslERGSVgVmmKst/zA+QBqr5wvAJEkbAncC90taLG//vLJ+HfBK\nyTmzA0n+7FngP6Qn6V6e374WWJrUOXkwIh6KiAeV5uBT9GlewoJyNr7OS8lZUJ0XkbNNPA6Z2SiR\ntDSpf8ur88Gj877ygUbAIaRHzTcCDo2I85xz2t8/kA+o8wOTga8CP4yIvwyTcTHS8Ac7kQauXBr4\nUkRc7JzDZm5knZeSs5Q6LyVnW/mWpVmPdd9WkbQDcCtpZPA/R8QD1XXyj1/n9sFXJS0OPNY5g+/j\nrZUicnZUzug7B4yVgSW61omc+VFJx5PGRtoAOKNzBt/9725TzlLqvJScHU2u8xJztpUbZGY9ks/O\nVb0VkG+b7AKsQeoAO1WVUcA78lnrQHoZj+RtO30yej3GUJE5Jc0HfBlYS9IlQAB/zZ9N6w9TOVBE\npHGUrs3rDEaa2qXnjZym5yy1zkvJ2cQ6LymnZdGAJwtcXMZSIZ1NfhbYsPLe+sC5wDOkOd2Wrnwm\nKiNYA8u0PSeV8YuAlZj+5NyipCe9vg08D3yQrtG/u3N276+NOUuo81JyllLnpeR0mV7ch8xsHlQv\n3UsaBI4CtgL+H6kz7DURcXz+fC/SOD6rkx7RvyQijqjsayHS/G9LAvtHxFNty9mVeSHgS6SBPa8H\nLo2IU/JnqwKXAdeQRg2/kjyRceXf2dn+mxFxcz8yNjlnKXVeSs6uzI2s81JzWuKnLM3mQeeHK1sM\n+EtEvJw0dcvLgAMlrZQ/34r0Y7cvsAfwtc6GSoMwXgJcQRo1vKcHkqbnzLeaun0CeDQiNgTuA75Q\nybgc8GNgr/znROCpyoFkv5zzD8AtvchYUk5ofp2XkrOUOi8lp81C3ZfoXFxKKqRL+aosv4E06vcm\neXlcXr4UeDHwHeDU/Nn3gO3y64H850TgU6RH9Xs2bUtJObuW1wdWzq8XydnOJ525n0eaNw9gR+DX\nw+xvLdIZ/+Fty1lSnZeSs+l1XlJOlxHUZd0BXFxKKczYJ2Mh0tn7ZaRbLL8iTVoM8E1g0/z6aFI/\njTU6P5LD7HeCczIfaXLivwFXV3LtVTkYb5AzbglMqqwzmP8UsAKVvkVtyVlonZeSs5F1XlJOl5EV\nP2VpNgKdp4tyH5cjSeMc3Q0cHBF/lrQ3afyjC0h9XHaQ9F7Sj9/OEXFrZV8zPDIeEc+0LWfe3/OS\nxgEfIZ2RXxMRe0r6KOlW1FakaVo6g4FuT7odNX9EXFHZz9T8Z+R/a081PWcpdV5Kzry/Rtd5aTlt\nZNyHzGwmJC0r6feQfrAkvZo00e5jpHneNgLWzI+S/wB4QtI7SIMpPgHMDxwWERdU91s9kLQs5ysl\nrV9Z3pJ0O+q1pI7YW+W/91hgQUlvA34H3EsaDHQl4HUR8Zte5ioxZ0F1XkrOxtd5STltLtV9ic7F\npckFOAc4Mr9+G+lsffm8/GnSmD5r5OXXAPcAi3Tto++Pi5eQkzSx86HAwqQDx9PAAfmznUi3qV6T\nl18H3A4smpdXds7y6ryUnAXVeRE5Xeau+AqZ2TDybRWAA4ANJU2IiB+TbqcckD87nfSk0haSFoyI\nS4BdIuIxKc2zp1lM4NuWnJWnv74BbAisGRGXA78mDV8A8BfSwWN7SQtExIWkM/rVACLiTiXOWUCd\nl5KzoDovIqfNo7pbhC4uTSh0PamU33vBWSSpE/LfgdXz8kHAt4DlasxeRM5Krk4H4s1I0+G8OC+/\nijT21Dsa8H1oTE5/N9tX52Mhp8ucFw8Ma60mabuI+NUI1ps2F56kI0kjiO+U+74sGBH/7XPOEc3F\n14CcC0TE0yPNKek7pC5B75e0MOnJr2s6Oes+m68zp7+bPc/p76Y1mm9ZWitJeqmknwEXS3rr7Nbv\nOuB8E3hU0mLA1Ij4b+f2Sp9yXgwcI2m3/N7gzNavMedGkq4C9p1dxk7U/OcngD0kbRARj0fEpdWc\nfWjkbCxpH0nrjnCTUc/p72bPc/q7aUXwFTJrFUkTSANIbgt8nzTFys0R8cPKOjM8Ul8XSbsCRwDH\nAg8CpwArRX7Evwk5JS1KGidqTdL0K+Mj4jUj3LZzhr9hRFzX55zzk/5bvp407tW2wMsj4tGm5PR3\ns7f83bTS+AqZtc2SwM3A5hFxAqlD8YYw/cy5eiDR8NORzPazHlke+E5EnAr8EvhR9e+uO2e+PfJB\n4BFgO1JH7dslLTeS7WP62EejcSBZkTTEwmYR8QHgLlJfHGZ3ZWYUc/q72SP+blqJ3CCzMU/SDpIO\nkLRaRNwTEd+NiOfyxz8DtpM0Xz7THJC0lqQvzm6/fbhlMS1nfmsA2FjSUcA/gVcC50vaLNKAkOvW\nlPMl+aD7OPCViDg0IiaTGhAbA4/n9TpPya2Wb02Nqk7OvDgReAZ4o6Q3kKaT2UzSqp3Gg6Q1Rjun\nv5v+btJpQe48AAAgAElEQVTQ76bVIBrwZIGLSz8KaQqQs4HfAycB55IGRayuswnwXWCVynvzAw+Q\nOh3XkfMCYGvSQW950hWId+d1DwEuz6/Hj3LOdUhjSv0p59wnvz/I9HkFrwT2rGyzGHAh8Mq8/IIn\nBvuc81vA2/P725EGH/0Hacymr5PmRtyINK7TqOX0d9PfzaZ+N13qK75CZmPZS4E7ImKLiNgfuJ40\ncW71VskDpKeSns/vzx/pCsXXgFVqyvk34KUx/erBY8AvACLiaGCZ3IH32dHKKWlt4HjSAXhLcl8X\nSctHum0SSlO4XEw60HVuXT1KmtLlHTl/X/sVzSTnTpJWiPTE4l+AMyKN/H4IaYqe1SNdUbl2tHLi\n72bP+LtpY4UbZDaW/ZI0CnjHraRxe4h0W2UgIu4CbgLent/v3C66gHRWWlfOSTnPPaSrFLtKWlzS\n+3Pe20Y5563ACRFxUv5vdANpMuOncs6IiCnA4sCqeZvO02zHA38YhYzD5fwnMA54Kh+UJ+bXC+fP\nH8qfQ2pAjFZOfzd7x99NGxPcILMxKyImR8QDlbfWJp2FTlsl/xDeDNyb++h0HhW/PiKmzK5TbR9z\nXltZPpzU/+VCYAfg8IjoHGxGJWe+0nBx5a0HSH1dBmGGoQQuIh2gx0Xqu0NEPBwRp/cz3whyzpcP\nytcALwG+LWmIdJD+bQ05/d3sXUZ/N21MGDf7VcyaK19JeD7/6D4/3OX8yq2eF5H6kXQeiX8mIp6V\n9M18tv8Cvbo9MI85L42ISyStFRE35/dnGFZgNG5jxIzjSG0C/F9EPNj12Y2kfjpTujOOlpnkvD9/\ndpGkPwJvAe6OiIv6lcPfzd7mnBV/N20s8BUyK5akbUm3HIiIqdUf2Eo/HCLiuXyWPh64RdJhpIEp\nX5Q/vydv06+BKecl50nAyvnzzgFvsJcHE023fV4eyYnasuSzfUmflPT6nPHfEXF1ft3TA14Pcn5K\n0k4R8UhEfKdzwNPsBwqdm6z+bvYmn7+b1hpukFnJbgXWl7Q5gKRNJR0Dwz5Ovz6wPWm8pFWBD0fE\nHdUV+njGPC85PzRMztlOUzMn8r97KeBnSo/aT8k5t5jFZpsCb5d0BbAScEUvM/Up54rA7zofVG4B\n9vS/Z+bvZg/4u9mX76Y1VTTgUU8Xl5EUQORJdSvvvR/4eH69GPCSmWz7ImAI2Ljy3gsmPm5hzgGm\nDwvwBeBn+fWCpCEXFhlmuwmkx/PPBzYZpXpvdM7C6ryUnI2u85JyupRRPHWSFUfSUqTL/f+MufgC\n57NPRZ/neGtyzk6/ofx6oYh4Mr/eJiJ+O7vtVJmqxTln+HsbW+ddf09jc5ZS56XktHK4QWZFkfQx\n0hn9EPAEcEREPFz9cczrTYqIP0kzdt7NfVz6fhugqTmH+Xs+Q3pK7izgT5H7As3B/lqds+vvaGSd\nl5KzlDovJaeVx33IrJEkTex0kM3L4yRtCKwWEWuROsPuDWwAM/Z3kbQZ8NbuH868Xk9/+ArKuULe\nb2cqlkFJXyfdVvkM8G5gH6WJjqvbLSvpDEmLDLffNuYsqM5Lydn4Oi8pp5XLw15Yk71C0k6k/hYX\nAU8CA5J+Rhqj500RcVn3QSPSk1RXO+cM9pZ0P2l+xJ2B00kHklOBfUkDaZ4R0wcf7WS8X9KREfGY\nc86ghDovJWcpdV5KTiuUr5BZY2jGR7yfBdYjTRfy74j4eX5/EvC7iHh1RPxW0qakp5WG219fvt8F\n5Zw2mCjwa9LceZcA/wUWAJYhzaF4bUT8T0RcpzR59fi8fedJr3/0I19JOQuq81JyNr7OS8ppY4Mb\nZNYYnUv3SnO7TQa+TZpgd0Je5bek6UaWkrSVpP1IZ6kbzGR/fekg2/ScnYNoRFQH+VwGuAr4Vz4w\nTwH+SOr3cmbe7nDgk6Szfirb9kUpOfPf0eg6LyVnKXVeSk4bW9yp3xpD0huAzwJ/JT2av7ek9YGj\ngBMj4peSVge2Bl5LeuT8MxFxk3MO29n4jcCOwGURcUZ+72Fgl4j4Xc74PmBz0gH7TuCwmMNOyWM1\nZ1fmRtZ5KTlLqfNSctoYFQ0Ye8OlXYV0ZXYSsHpeXoR0hn4RsCFp9O+ngA/mzz9KOgPt/PgNAgt1\n7U8tzjkRWLqyPD/wA+AXpOlZrmf6OFMfA/6SX69Iuu2yPLBpZfueZywlZ0F1XkrOxtd5STldxnbx\nLUurwwCpQ/FXJB0FHEyaaPeNpIPMz4CvA8cqPZl0EjCZ9FRYRJripTPmz0DMeFuhjTk3AE7If88B\nwPPAacBbgc1IB+udJW0aEccA4ySdSzrYrBMR90TENZWc/bpsXkLOUuq8lJwl1HlJOW0sq7tF6NKO\nQtcZOPB20hhIQ5X3Fgd+DqyXl29i+qjXE4EFnXOmOf8PuBf4IumMXcDhwGn589OAs/LrFYHdgEWd\ns+g6LyVn4+q8pJwu7Sm+QmajIvIZuKTtJO0PXA4cBjwoacG82rrAM8AESVuROiAvKGkC8HREPKU+\nT7ZbQs7qFQ1J6+an5M4FJkTE4RHxdEQE8GJSp2OA24ANlQb7vCsifhIR/3XOMuq8lJyl1HkpOa1d\n3KnfRoWkBYBTSGeWp0TE9yUtRpr77fGI+HRe7zOkDrKrA3tFxFXOOWzOFYCjSX2F9omIGyRdANwR\nEQdIGkca9PONQJAO0sdHxAwTGEeffwBKyFlQnZeSs/F1XlJOaw83yGxUSFoH2B/4REQ8LWlh4DlS\nf5dPAUcASwP/Bh6MiH9Xth21qUUKynkGcGNEfLHy3oqkCYtXiIjHJG0ALAVsFBHHj0auEnMWVOel\n5Gx8nZeU09rDI/XbaFkQWBv4haS/A68h3SL4Hmkcn9OBXwEfi4gnIE31EhFTRutAUkrO3En7X6Sx\npPYnTRL9DPC1XH6Zb2F9NyJOIN22mnabZjQylpSTAuq8lJyl1HkpOa1dfIXM5pkkAQOz+9FXmkdv\nYeAGYDXgEODDEXGf0kCWdzvnDKN7z/R/TkmvAN4F/A1YEliJ9Cj+tyTtCtwVEX+u7rPXt1ZKyFlY\nnZeSs9F1XlJOsypfIbN5ln+kpkpaFJhaOTvvnh/vuvz+i4C3AYvl1RQRd+cfUfXrDLSEnNUsszob\nj4g/kjsb5zP500jDHhAR53T21fl39+OAV0LOEuq8lJyl1HkpOc26+SlLmyvqmuNO0j7ArcDXJL23\n8/Yw270c+D5pnJ/XRcS9nR+6/JvXl6laCsg52Nl3Xv4c8GVJ21Y/79pmGUkHAn8BbgTOqX7ep4Nd\n43MWVOel5Gx8nZeU02xmfIXM5lj1rDP/2C0ALEQa0Xp10mCUl0bE7cOcoV4D7BERD+ft+9bZuISc\nmv74fWcOwsWAbYG1SP1WfixprU6OLv8lHcBfG7kDd79uq5SWM79uZJ2XkrO0Om96TrPZcR8ymyuS\nVgP2A15NatjfDuweEc9K+iqwXES8s2ubcRExpbLc9ye/Csq5AnAq6aD8ALB3RDwi6TTS+FH7da0/\nQD6B72QE+jXaelE5C6rzUnI2vs5Lymk2M75labPVfalf0tKkOfMWjohNSSNbP0yaQw/gOGADSa/L\n688HEBFTJE2UdJCk5Xp9ICktp6RBJV8EPk6aO+8wYDypfxCkefO2UeqATF5/MKYPajlR0kKRpsLp\nyy2gJucsrc5LydnkOi8pp9mccIPMZqtyK2BjSctExP8BZ5MGn4TU7+IpYOv8+X2kR/FXy9tPztt/\ngDS6+L8i4j9tyylN6yDcOYh2bo0sBGwPXJD/3h8CL5O0dv43/Az4ct42Kv/O/YFrgTV6lbGknNWM\nTa3zUnKWUuel5DSbK9GA+ZtcmlWALYElK8ubAdeTJtL9PbAVsAwwRLrFAvA/pB+9Nwyzv01ITzN9\nGhjfwpxLdC2/gXQA/jqwBenE6O/ADvnzdYEvAV+obLNK5fVOwCVtzFlQnZeSs/F1XlJOF5d5KbUH\ncGleAX4KnJ5fjwP+lzQFC6Rxj44GtgZ2Bn5f2e5DwNpd+xoHbAcs3cacpAmgjwdWy8uTgKuBHYBD\ngZ/kA/GuwFWV7XbPB5QlSONTkQ8665Eez1+upTkbX+el5CyozovI6eIyr6X2AC7NKOQHPPLrF5Ee\nAX9ZXv4Z8P78ekngKOCdwCBwGWkqF+ecMee4/OfSpH4tb8kHg48Dx+TPJpLO1H+Uly8HPtn5bHb/\n/rbkLKjOS8nZ+DovKaeLS6+K+5AZMMPYPfuT5sR7jnRWD3AlsHDu2/IQMAVYL1I/jPcA3+jsp9PH\nwzmnPQn3KtJceG8BFgf+BEySND7S4J8PkSYuhjRJ9EJ5+87AoANd+w16qIScBdV5KTkbX+cl5TTr\nFTfIWqrzpFHXe5OA95NuD+wGrKE0hcgQqXPxkZI2IfXZuAYgIm6LiMc7P3q9/rErKWdX5gUlfRd4\nN2kQzy1It1DuJk3VckRefRlgfqXhDC6JiMOq+43eD/LZ+Jwl1XkpObsyN67OS8pp1jd1X6JzGf1C\n7k+RXy8ODObXuwDHVz57I3ATMAFYETgG+DnwbuccPmflvQmkTsOr5OXdgO8ALwdWJY0Mfhapo/cO\n1f0Mt7+25CyxzkvJ2dQ6Lymni0s/S+0BXGqs/NS5+G7SmEdbAZsCvyP33cjrPAR8Ob8e17X9qPTF\nKCjnETnrxqSrz6cC21U+HwJOBObPB5sNaqr3xucsqM5Lydn4Oi8pp4tLP4pvWbZAdx8KSVtI+gFp\n8MTXkB4X/zTwV+Ae4AhJa0h6PXAFMJj3MbW6v4jo9a2VUnKqa/mlkr4FrAzMB5wd6TbJvcDGkjpj\nHN0ErA2sHxHPRMTf8/YvmGOvLTkLqvNScja+zkvKaTaaPHXSGKcZ58ybEBHPSFqX1Mn4UxFxkqQl\nSGemt5EGo/wQ6THyQWC/iPinc07LOcNUNZLWJp2xPxERb8zv/Qb4DXACcDjw0rz6Q8DnIuIG5yyq\nzkvJ2fg6Lymn2air+xKdS/8L6Qmlb5M6Gk/K7x0B/CS/FulWyxXAxvm9lbv20fc+GQXlXALYF1gm\nL+8DnAm8JC+vTjpwLJeXdwJ2raHeG5+zoDovJWfj67yknC4uo1l8y3KMGebWynakEa2vBq4Dvi5p\nY+DzwIaSXhsRAfwT+CWwFkBE3Jm3H8zLvX6iqtScB5D6CG0AfDo/QfcT0hAHG0taOCJuAy4mjZ1E\nRFwQEecMt7825Sy4zkvJ2bg6LymnWd18y3KMkvQG0lnmhcDzpLF5jgGWJf3QfYI0MOXHIuIleZsZ\nbiW0PWfXrarNSINQrg98lzSVzamkg/CBpFsqOwGnRcSVuY/MihHxb+d8Qd7G1nkpOUup81JymjVC\n3ZfoXOa9kPpZdKZlWYr0aPi0p5OAVUgHlVeQDjA3MH3+vNOBFbr215cnv0rICSwHbAsslJdXBQ7K\nuVYgPdn1EeAqYA/ga8Cn87r/j3SAno/+DxNQSs7G13kpOQuq8yJyurg0rdQewGUeKm/6Fc6dgL/n\n1xOBu4DjKuu9DLg5vx5Huu1yZOcH0zlnyPo64Ifkx+lJ099cCCyVlxcBflRZPoM0wOemnfecs5w6\nLyVnCXVeWk4Xl6YV34svWERE/vMC4DpJr440XcihwPaVVW/Jn59P6gdzBulJpSeh/30ySsmZM15I\netR+m/zW50iP4k/Jt6MeI91aOUjSPqSDyxHAdRHxYM7Z16lvSshZSp2XkjNnbHSdl5bTrGnch6wQ\nkhTDVFanj4bStCFTOu8BPyb9wH0xv7c4adqRyyPiH9Vt25hzJtk7GVcEJkfEf/L7lwNnRcT/5uX1\nSNPjrEGaFPr6fmdrcs5S6ryUnDPJ3qg6Lz2nWRO5QVYASS8GloyIv8+q03DnrDIiInegPYPUB+aO\nrvUG8mo9rfyCcs4fEc+NYL35ImKypFeS+hRtExEPdO+j+u9pW86C6ryUnI2v85JympXEtyzLsC1p\nzjZmdiDJn007QETE1cC5wLrVdfJVguf79MPX+JydA4Ok8bNbNx9IFBFXAreTRmLvfNY5kAxW/z1t\ny0kBdV5KzlLqvJScZqXxFbKG6j4Dl/Q70uPg3+18HrO5VTKzWzQtzTkhIp7Jry8GTomIn45gu84t\nmEVIT3s92vacBdV5KTkbX+cl5TQrla+QNVDnBywionIW+h7g8s461QOJZtKhOG/ft86xJeRUmnPw\nUuB4SXvlt08E1u7kmdXfXcn/eEQ8OrN/Q4tyNr7OS8lZUJ0XkdOsdL5C1lCSlgc+CTwM/Jz0SP7U\nytnmAOmx+y9HxH9Hcrbfppz5AHEg8D7SiOpPAV8Hto+Ifw2z/sKkPkZ39DtbiTm7MjSyzkvJWUqd\nl5LTbKzwmUoDSFq1a3k10hNed+e3PgPsAulss9OHBViJ9Lg4o3QgKSWnAAHXA2+MiLMiPYp/DbBn\nZb3q9/9DwP6V7fuuhJwF1XkpORtf5yXlNBtL3CCrmaT5gaMlrSZpG0lbkqZmeToijoqIzwAXAJtI\nelFns/zn8cAjkhZwzhmf1MoH1z8Cd0gal1e5G/hbZ/3OgTkv/hjYTmkevX73GSolZ+PrvJScBdV5\nETnNxiI3yGrSObOM9KTRvaT53A4GHgeeBf6jNFYPwF+BrYHJeZvOmfz/Ad+IiKedUwOdg4Ck1SUd\nFxFPRRpXqpPjJcAT1e1yH6GBiLiFdDtmoX6e3ZeQs6Q6LyVn0+u8pJxmY5UbZDXJZ5YT862V/5AO\nDKdHxN+AKaSpW96YV78eeBQY37WP2yPifueclnNRSceTOm9/SNKb8seStAywdET8VtIikrbunPV3\nDs4R8f2I+E8/z+5LyFlYnZeSs9F1XlJOs7HKDbJ6fR44ICKOBN5Bun1CRFwHnAdsLumnpLngfhcR\n9zhnouGf1PoysDhpougjKzmnAqsC/5B0CKkfzCb5zN85h9e4Oi8lZyl1XkpOs9aIBkyoOZYL+UnW\nyvJLgB3z6zVJB43X5eVLgS/m1+sC85MGtFzVOWfINlB53ZmgeAFSH5Y1Kp9dDhyRX+9Nuu1yCrC6\nc5ZT56XkLKHOS8vp4tKm4itkfaQ8AnVleSFgB2A3SUtH6nNxGfAWSROADwDvlnQhcDQwPiIujYjb\nJQ30sc9Q43NKWlbSUjDt1spaks4CTpf0JdLVXgFvq2z2c+ADkhYDriJN27JPRNzmnM2v81JyFlTn\nReQ0a626W4RjrTB9bLfOnwuQHrnfClgUWIV0G+ADlc9vAD6UlycBuzjnDFkXIXUWPiwvD5KmtDkk\n5/wW8CNgOeABYGNgYeBQ4E/A0V37G2hjzlLqvJScJdR5aTldXNpcfIWsh5Se6PoSTHvy6G2ks8qF\ngFcBJ0QaNPFPwEslrR7p6a5bga0lLRcRf4qIc/P+Bluec31JX4yIx0hXQVaUtAmwGOnWyQkRcUdE\n7AdsCSwBfAr4IPAPUsfurwGP5f11Hunv6XhTJeQsqM5Lydn4Oi8pp5m5U3+vjQOWlbRzXn4O2BH4\nPqkfy2vzAeZM0hNh35c0BNwGfDgi/lPdWcxiEuSW5BwEVpC0HXAxcB/whoh4iNQ/aFJl3TOALSPN\nU7g/sAFwdn59b87Zrye/SshZSp2XkrOEOi8pp5nVfYluLBVgArAH8GlSX4z5ge2AvwCvBd4KXAsM\n5vX3BnaqbC/nHDbnp/Lya0kdijcEdiMNZbAKabT1XwOb5/XmJw13cB3wVucsss5LydnYOi8pp4uL\nS3guy16TNDEinqgsHwRMiYhvSnoz6cfwhIg4vGu7UZ3vr8SckhYH3gWsFhEHSfoisDzpLP/0iPhK\nZbulIuJB5xw+Y15ufJ2XkrOpdV5STrO2c4NsDuU+Kv8ZwXqdCYyPIHWUvRPYFDgd+E1EPOycc5Rz\nMNLE0JsB7wSujYhTJS1C6mD8aF6vLwflEnKOwTovJae/m2Y2z9yHbA5I2hPYT9JKlfeWns1mRwO/\nAtYnPal0VkQ8rOEHZXTOmeSM6X2BriN1Nl5W0njgiYh4VNKgNG3C6NblHIt1XkpOfzfNrBd8hWwE\nJI2LiCn56aTdgGsi4qeStiadsZ8SEY+McF/TJu91zrnLqTR58eO9zlVizrbUeSk5/d00s7nlBtks\nDHfpXtJbgP9ExBWSloz0tNJc7885nXOsZnTO9uY0sznnBtkI5NsCuwPHRsRldeeZGefsrRJylpAR\nnLPXSslpZiPnPmRdOv1SlKYFmU/SYcC7gaOqP3yqDDipNCXJnpIWdk7nbHNG52xvTjObN26QZZ0f\nvfxU14SIeD4iJgPrAacCd0vaUtIueb3qgJOrAXeOUt8R52xZzhIyOmd7c5pZb7T+lqUkVTsHS/oY\nsBNpUt2rgAWB04Bf5FV2Bb4QESeNZv8L52xfzhIyOmd7c5pZb42rO0BdpBc+qSXpYNKZ5T7AJ4Dt\ngV2AjTpPLEm6nvzfbbQOys7ZrpwlZHTO9uY0s/5o7S3LyCStLOkbkpYB1gZ+AuxLGvPoqxExBXhU\n0haSfkz6Yfy1czpnmzM6Z3tzmll/tKpB1jkDrSwfDHwFuD0iHgCmkCbTvSMiXhkRl0laB1iCNLbP\nNRGxcUTc6JzO2baMztnenGY2CqIBE2qOdgFWzn+eAtxaeX9X4Exg/bx8IHA+ad636vaDzumcbc3o\nnO3N6eLi0r9Se4C+/wPzgwuV5Z2A64GJwArAv4Ct8meLAR8Afg9cDpwHbDyr/Tmnc47ljM7Z3pwu\nLi6jW2oP0Jd/FMwHLFJZXgR4dWX5QuDg/PpDwK+7tl+c1Gm2s9yvH2bnbFnOEjI6Z3tzuri41Fdq\nD9DzfxCMB7YBds/Lk4CDgZOASfm9VwJ/BVYFJgC/AQ7Mn3WfvfblVoBzti9nCRmds705XVxc6i1j\nplN/5ZHxZ4HngQ9Luh3YGLgI+C/wSqVJda8E7gS+FBHPACcCy+bto7rfmHGwRed0zjGZ0Tnbm9PM\nmqH4BpmSwa4frUFgJeAPEfGtiLgB+AuwIrBlXufXwC6S1o+In0fEYc7pnG3L6JztzWlmzTJmRuqX\ntDTpCaQLgWtJgym+H/hLRJyuNKfb+4CtgQAeBc6MiIsq++j7KNfO2b6cJWR0zvbmNLNmKLJBJr1g\napH9gf2An5GeVJovIj6S398IOCginpO0PLAOsBlwdL8v/Ttn+3KWkNE525vTzJqrqAaZKpPtVt5b\nANgZ+CXph+040q2BPYAbgU8Cq5CecjomIoaq++vH2adzti9nCRmds705zawA0YAnC2ZXgJWZ8ZHx\njUijWW9feW8v4BpgE9Jj45fk95cGjgW269pnP4Y0cM6W5Swho3O2N6eLi0s5pfYAswyXHjo4HPgt\n00eqPgS4AngzcDFwQn7/M0x/rHw/4Bngfc7pnG3N6Jztzeni4lJeGUezLQq8jjSS9URJE4GngLeQ\nzkhXBK7M674YWFzShsDqpNsDo9U51jnbl7OEjM7Z3pxmVpq6W4QzK8BA/vP3wEPAMcAawB+Aq4Ef\nAy+rrL8E8E7gR+Qz1/x+X28DOGf7cpaQ0Tnbm9PFxaXMUnuAWYZL04WcCzwOrJHf+yppLJ/OOmsD\n3wWW79p21H70nLN9OUvI6Jztzeni4lJeacRTlsNduq8+Ri7pI8DbI2JTSfOTbgn8AViANOXISRFx\nwqz255zOOVYzOmd7c5rZ2NGIBtnMVH/ElKYc+UykARXXBJYnjd1zSkQ87JzO6YzO6ZxmVqy6Ls0x\nvT/GfKSOsN8AJgyz3mD+c1fgsZnsa5A+3Q5wzvblLCGjc7Y3p4uLy9gstc1lGRHPS1o8IiYDk0nT\niowfZr2p+c9zgEskbVD9PN9GmBoRfbnU55zty1lCRudsb04zG5tG7Zal0mS7UyvLy5I6x15EepR8\nSkR8XNK4iJgyq22d0znbltE525vTzNphVK6Qdc4Y8+u1JS0UEfeTBku8FdgSeFc+04yubQeA7s61\nfRk/zTnbl7OEjM7Z3pxm1iL9vB9aLcCmwKXA2aQ53parfPYR4E7g28BPST+Gg+Q+HXmdzYBvOKdz\ntjGjc7Y3p4uLSztKf3aaO71WlhcFzgFel5dvB04HxuXlQ4E9gXHA54FtK9suDpwIXAis45zOOdYz\nOmd7c7q4uLS39HZnXU8VkaYSWSK/XhR4DXAtcCRwE/DW/Nm3gcOG2d/bgRuoTNjrnM45VjM6Z3tz\nuri4uPRuRzNeyl8DuBH4NXBZ5f1vAe/Ir78B3AYsRZobbpn8fudBgwFgXWC+nv6DnbN1OUvI6Jzt\nzeni4uIS0YNhLyQJpj0yvr6kg4ENgIMj4rXAoKRP546wU4FlJa2bN78h/2heGBEP5I620dlfRNwQ\n6RH0eeac7ctZQkbnbG9OM7MZzE0rjtSvYgdyJ1jSdCFvAC4ndZC9Fdgvf7YpcAvworzOGaT+Gu/o\nd2vTOduXs4SMztnenC4uLi4zK3O3Ufohq3aI/SZwM7BRXv4IcBywdF7+LnBmfr0EsGBlX4Nzk8E5\nnbPUjM7Z3pwuLi4uMysjvmUp6WWSfidpo4i4Dzgf2E7SfMCXAQGL5dWvII3d8+a8fAjwnKQFgUcj\n4ilJgzB91Otecc725Swho3O2N6eZ2YiMtOVG+gE7u7I8Dli3snwU8PP8en7gXcBPgDVHs4XpnO3L\nWUJG52xvThcXF5eRlNmvMP0Jo61IgyiumJcHuv4UcB2wc15eC3hV174G5jWwczpnSRmds705XVxc\nXOakjHguS0lLAR8FiIhPdn02GBFTJe0HHBgR649op33gnL1VQs4SMuYsztlDpeQ0MxuJORn24hHS\nyNRrSlqh+kHkPhcR8S3gTb2LN1ecs7dKyFlCRnDOXislp5nZbI24QZZ/4MYDC85mvZs64wDVwTl7\nq4ScJWTMf79z9lApOc3MRmJOnrIcBHYnTbB7tKQ98/sv2EeM9D5oHzhnb5WQs4SM4Jy9VkpOM7OR\nGHEfshk2kvYCdo+I10taIiIe7n20eeecvVVCzhIygnP2Wik5zcxmZm6nTvo1cJekW4EjhjsjbQjn\n7FBYq98AAACGSURBVK0ScpaQEZyz10rJaWY2rLm6QgYgaTywcUT8obeRess5e6uEnCVkBOfstVJy\nmpkNZ25vWaqEPhnO2Vsl5CwhIzhnr5WS08xsZub6CpmZmZmZ9Yb7WZiZmZnVzA0yMzMzs5q5QWZm\nZmZWMzfIzMzMzGrmBpmZmZlZzdwgMzMzM6vZ/wfNMgQ6LfP57QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x12e8c8d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#dec_tree_parameters = {'max_depth':np.arange(10,99),'max_features':['auto', 'sqrt', 'log2',None, np.arange(0.01,0.99)],'class_weight':['balanced',{1:0.5,-1:0.3, 0:0.2 }, None]}\n",
    "#tuning process for decision tree\n",
    "dec_tree_parameters = {'max_depth':np.arange(12,14),'max_features':['auto', 'log2'],'random_state':np.arange(27,29)}\n",
    "pre_processing.tuning(dt_classifier, dec_tree_parameters, dev_sparse, dev_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>Comment:</b>\n",
    "Overfitting occurs when a classifier runs better on some data, but runs bad on others. That is because the classifier overlearns from its training data. In the process of tuning we are running the model with different parameter values on the same data set in order to find best parameter suite. However, since we are running on the same data set this parameter suite, this parameter suite might be good for this data set, but it might be very bad for different data set( the data set that the model have never seen)(i.e the parameter suite might overfit to the development set). In order to determine this situation we can perform cross-validation on the data set in order to evaluate model's performance with the best parameter values on unseen data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.481683980317\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "         -1       0.34      0.34      0.34       360\n",
      "          0       0.44      0.64      0.52       700\n",
      "          1       0.71      0.40      0.52       769\n",
      "\n",
      "avg / total       0.53      0.48      0.48      1829\n",
      "\n",
      "--- LR Classification time : 97.9670000076 seconds ---\n"
     ]
    }
   ],
   "source": [
    "#logistic regression classifier\n",
    "time1 = time.time()\n",
    "log_reg_classifier = LogisticRegression(solver='lbfgs', multi_class='multinomial')\n",
    "log_reg_classifier.fit(train_sparse, train_labels)\n",
    "pre_processing.classifying(log_reg_classifier, dev_sparse, dev_labels)\n",
    "print(\"--- LR Classification time : %s seconds ---\" % (time.time() - time1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score: 0.457627118644\n",
      "Parameters : \n",
      "  C: 0.01\n",
      "  random_state: 0\n",
      "  solver: 'newton-cg'\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjIAAAFpCAYAAACGbcLwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmYJFWd7vHv290sCggoKgIiKouCuwOiMA7KKLjidQUd\nFRfgysVlXEBUhFFnFMFBR9xAdHTUK44rooJruSKiiLiAIN5BFodxRQEVbN77xznZHZ1UdVU1VZVx\nMt/P8+RTlZmRmRH9dp38RcSJc2SbiIiIiBYtG/UKRERERKyrFDIRERHRrBQyERER0awUMhEREdGs\nFDIRERHRrBQyERER0awUMhFzIGlfSRdKukjSEdM8/3eSfi/p3Hp71SjWMyJi0qwY9QpE9J2kZcCJ\nwN7AlcA5kj5l+8KhRb9m+7FLvoIRERMsR2QiZrcbcLHtS23fAHwY2G+a5bS0qxURESlkIma3NXBZ\n5/7l9bFhD5R0nqTPSNp5aVYtImKy5dRSxML4HrCt7eskPQL4JLDjdAtKyrwgERHzZHvao945IhMx\nuyuAbTv3t6mPrWL7GtvX1d8/B6wn6dYzvaHtOd+OPvroeS3f2i3b1/ZtnLdvnLette1bmxQyEbM7\nB9he0p0krQ/sD5zWXUDS7Tu/7wbI9m+XdjUjIiZPTi1FzML2SkmHAZ+nFP+n2L5A0iHlaZ8EPFHS\n84AbgD8BTxndGkdETI4UMhFzYPsMYKehx97V+f1twNsW47P32muvxXjb3sj2tW2ct2+ctw3GZ/s0\n27mniFhYkpy/u4iIuZOE09k3IiIixk0KmYiIiGhWCpmIiIhoVgqZiIiIaFYKmYiIiGhWCpmIiIho\nVgqZiIiIaFYKmYiIiGhWCpmIiIhoVgqZiIiIaFYKmYiIiGhWCpmIiIhoVgqZiBGQtKi3LbfcbiTb\nteWW2y36to379o1q27J92b6+b99MMvt1xBKTZFjsvzsxir9tSSz+tsF4b99otg2yfQv0Kdm+xfhU\nZfbriIiIGEMpZCIiIqJZKWQiIiKiWSlkIiIiolkpZCIiIqJZKWQiIiKiWSlkIiIiolkpZCIiIqJZ\nKWQiIiKiWSlkIiIiolkpZCIiIqJZKWQiIiKiWSlkIiIiolkpZCIiIqJZKWQiIiKiWSlkIiIiolkp\nZCIiIqJZKWQi5kDSvpIulHSRpCPWstyukm6Q9PilXL+IiEmVQiZiFpKWAScC+wC7AAdIutsMy70B\nOHNp1zAiYnKlkImY3W7AxbYvtX0D8GFgv2mWez7wUeB/lnLlIiImWQqZiNltDVzWuX95fWwVSVsB\nj7P9DkBLuG4RERNtxahXIGJMvBno9p2ZpZg5pvP7XvUWEREAU1NTTE1NzWlZ2V7ctYlonKTdgWNs\n71vvvxyw7WM7y/x88CuwBXAtcLDt06Z5P8Ni/92JUfxtS2Lxtw3Ge/tGs22Q7VugT8n2LcanStie\ndgcxR2QiZncOsL2kOwG/BPYHDuguYPsug98lvRf49HRFTERELKwUMhGzsL1S0mHA5yn9yk6xfYGk\nQ8rTPmn4JUu+khEREyqnliKWWE4tLcgnjfH25dTEon1ytm8hPqV3p5Zy1VJEREQ0K4VMRERENCuF\nTERERDQrhUxEREQ0K4VMRERENCuFTERERDQrhUxEREQ0K4VMRERENCuFTERERDQrhUxEREQ0K4VM\nRERENCuFTERERDQrhUxEREQ0K4VMRERENCuFTERERDQrhUxEREQ0K4VMRERENCuFTERERDQrhUxE\nREQ0K4VMRERENCuFTERERDQrhUxEREQ0K4VMRERENCuFTERERDQrhUxEREQ0K4VMRERENCuFTERE\nRDQrhUxEREQ0K4VMRERENCuFTERERDQrhUxEREQ0K4VMxBxI2lfShZIuknTENM8/VtIPJH1f0ncl\nPXQU6xkRMWlke9TrENFrkpYBFwF7A1cC5wD7276ws8wtbV9Xf78n8Anb28/wfobF/rsTo/jblsTi\nbxuM9/aNZtsg27dAn5LtW4xPlbCt6Z7LEZmI2e0GXGz7Uts3AB8G9usuMChiqo2BXy/h+kVETKwU\nMhGz2xq4rHP/8vrYGiQ9TtIFwGeBFyzRukVETLQVo16BiHFh+5PAJyXtCfwHsNPMSx/T+X2veouI\nCICpqSmmpqbmtGz6yETMQtLuwDG29633Xw7Y9rFrec0lwG62fzPNc+kjc/M/aYy3L30sFu2Ts30L\n8SnpIxPRoHOA7SXdSdL6wP7Aad0FJN218/v9AKYrYiIiYmHl1FLELGyvlHQY8HlK8X+K7QskHVKe\n9knAEyQ9A7geuBZ4yujWOCJicuTUUsQSy6mlBfmkMd6+nJpYtE/O9i3Ep+TUUkRERMRCSSETERER\nzUohExEREc1KIRMRERHNSiETERERzUohExEREc1KIRMRERHNSiETERERzUohExEREc1KIRMRERHN\nSiETERERzUohExEREc1KIRMRERHNSiETERERzUohExEREc1KIRMRERHNSiETERERzUohExEREc1K\nIRMRERHNSiETERERzUohExEREc1KIRMRERHNSiETERERzUohExEREc1KIRMRERHNSiETERERzUoh\nExEREc1KIRMRERHNSiETERERzUohExEREc1KIRMRERHNSiETMQeS9pV0oaSLJB0xzfNPlfSDevuG\npHuOYj0jIiZNCpmIWUhaBpwI7APsAhwg6W5Di/0ceLDtewOvA05e2rWMiJhMKWQiZrcbcLHtS23f\nAHwY2K+7gO1v27663v02sPUSr2NExERKIRMxu62Byzr3L2fthcpzgc8t6hpFRAQAK0a9AhHjRNJD\ngGcBe659yWM6v+9VbxERATA1NcXU1NSclpXtxV2biMZJ2h04xva+9f7LAds+dmi5ewEfA/a1fcla\n3s+w2H93YhR/25JY/G2D8d6+0WwbZPsW6FOyfYvxqRK2Nd1zObUUMbtzgO0l3UnS+sD+wGndBSRt\nSylinr62IiYiIhZWTi1FzML2SkmHAZ+nFP+n2L5A0iHlaZ8EHAXcGni7ym7RDbZ3G91aR0RMhpxa\nilhiObW0IJ80xtuXUxOL9snZvoX4lJxaioiIiFgoKWQiIiKiWSlkIiIiolkpZCIiIqJZKWQiIiKi\nWSlkIiIiolkpZCIiIqJZKWQiIiKiWSlkIiIiolkpZCIiIqJZKWQiIiKiWSlkIiIiolkpZCIiIqJZ\nKWQiIiKiWSlkIiIiolkpZCIiIqJZKWQiIiKiWSlkIiIiolkpZCIiIqJZKWQiIiKiWSlkIiIiolkp\nZCIiIqJZKWQiIiKiWSlkIiIiolkpZCIiIqJZKWQiIiKiWSlkIiIiolkpZCIiIqJZKWQiIiKiWSlk\nIiIiolkpZCIiIqJZKWQi5kDSvpIulHSRpCOmeX4nSd+S9GdJLx7FOkZETKIVo16BiL6TtAw4Edgb\nuBI4R9KnbF/YWew3wPOBx41gFSMiJlaOyETMbjfgYtuX2r4B+DCwX3cB27+2/T3gr6NYwYiISZVC\nJmJ2WwOXde5fXh+LiIgRSyETERERzUofmYjZXQFs27m/TX3sZjim8/te9RYREQBTU1NMTU3NaVnZ\nXty1iWicpOXATymdfX8JfAc4wPYF0yx7NHCN7Tet5f0Mi/13J0bxty2Jxd82GO/tG822QbZvgT4l\n27cYnyphW9M9lyMyEbOwvVLSYcDnKadjT7F9gaRDytM+SdLtge8CmwA3SnohsLPta0a35hER4y9H\nZCKWWI7ILMgnjfH2ZY9+0T4527cQn9K7IzLp7BsRERHNSiETERERzUohExEREc1KIRMRERHNSiET\nERERzUohExEREc1KIRMRERHNSiETERERzUohExEREc1KIRMRERHNSiETERERzUohExEREc1KIRMR\nERHNSiETERERzUohExEREc1KIRMRERHNSiETERERzUohExEREc1KIRMRERHNSiETERERzUohExER\nEc1KIRMRERHNSiETERERzUohExEREc1KIRMRERHNSiETERERzUohExEREc1KIRMRERHNSiETERER\nzUohExEREc1KIRMRERHNSiETERERzUohEzEHkvaVdKGkiyQdMcMy/ybpYknnSbrPwn361MK9VS9N\njXoFFtnUqFdgkU2NegUW0dSoV2CRTY16BRZECpmIWUhaBpwI7APsAhwg6W5DyzwCuKvtHYBDgHcu\n3BpMLdxb9dLUqFdgkU2NegUW2dSoV2ARTY16BRbZ1KhXYEGkkImY3W7AxbYvtX0D8GFgv6Fl9gPe\nD2D7bGBTSbdf2tWMiJg8KWQiZrc1cFnn/uX1sbUtc8U0y0RExAJbMeoViJhMmufy/zT/T9B8P2Oh\nrMvnZvvW+ISRbRtk+4a19H8Txn/7biqFTMTsrgC27dzfpj42vMwdZ1kGANv9agUiIhqWU0sRszsH\n2F7SnSStD+wPnDa0zGnAMwAk7Q783vZVS7uaERGTJ0dkImZhe6Wkw4DPU4r/U2xfIOmQ8rRPsv1Z\nSY+U9DPgWuBZo1zniIhJIdujXoeIiIiIdZJTSxEREdGsFDIRERHRrBQyESMm6ZajXofFomKDwe+j\nXp+FluzalvzGQwqZiBGRtL2kLwPHS9qzPrZ8xKu1YCTtC5wF/JOkLWy7TvfQvGTXtuQ3XsZ2wyL6\nql7CDbAx8BbK5d3HwaorpJree5K0Xv31r8ArgPWBY0e3Rgsn2bUt+Y2nXLUUsYTqrNgvBw60/ef6\n2HqU+Zu+Y/tYSXKjf5iSngnczfaRncfuBHwUeIHtsyQtt71yZCu5jpJdu9lB8ms9v7XJEZmIJTA4\nrGv7PGAj4MmD5+pElMcDL5f0GuCFg3Pbregclv8asHv90gDA9qWUL4t3SzoOeERLe77Jrt3sIPnR\neH5zkUImYhHVDnfLbN/YefhYYEdJKwbL2D4LeCrwSGDK9l9GsLrzJmlZXf+VALb/H/BxYOfucrbf\nRDmU/1DgjBb2epNd0WJ2kPwGWs1vPjKyb8Qiqo2GJW0KPAX4k+3/AL4xzeJfpczRdAOUPa2+HwYe\nfElI2hV4HnCS7bd2l+l8mZwP3MH2X+vjvT6Mn+zazQ6SX32u2fzmI0dkIhbY8KFbSS+lzMV0CHCo\npLvVx5dDaXBrw3Id8CY6nQ+XdMXnaLDeg+2UdARwIvBlSufJNZYbNLi2vw18RdI/L/U6z1WyW3O5\nlrKD5De8XGv5rasUMhELpHMo253H7gc8Dng08BDg18BDJd2q21h2XnMKcGPtpNcrtWNkt5G/bf25\nGfB2yrn4HSTdU9J6M1wF8m5gG0kb9GmPMNm1mx0kPxrP7+ZKIROxQFzcWBuT4yRtTjl9+ytghe0/\nUBqTRwN37762NsTrA68CrgF+s8SrPyNJ94dVHSOR9EBJ/wl8rDaWPwSOoXwRvIGyZ/vi+pruF8sd\ngJcCF1IP4fdFsms3O0h+NJ7fzZU+MhELqB7qfTLwQdu/U+lUeCVwN+As25+SdDywr6Sf2v49rGp0\nrpf0H7a/P7IN6FAZ9fTtwG0kPRv4I3AycAfgE5TD8AfY/pCk79q+qH4h7A/cYpq3vA443PaPl2YL\n5ifZtZsdJL/W87s5UshELBCVyzzvTBmn4ocAtr8l6RHAgZI2AzYHfgH8HfDW4ffoS0Na3QG4NeXL\n4d7AJfXxZ9i+UtKvgddK+hjwc0lbUvYGHw0cOvxmtq8Grl6SNZ+nZNdudpD8Ws/v5sqppYh5mOa8\nc9dtgH2AP9RlN6qPn0y5KuIFwL7AfsC1QB/Pxa8apt32JZRG9L+AXYHtgNsBN9Tz7KdSDlMfVa+G\neCZlxNQH2J5a2jWfXbJrNztIfjSe32LKyL4Rc6SbjkkxeHzVpZqS3gZsbPuZ9f5OwIa2f1Ab182A\nI4CdgKcMDm+PmqS7ApfZvr7eXw5sSxks7P62t6uPnwGcAbylXvHxFkqHynvY/mPn/Xp1+Wqyazc7\nSH718WbzW2w5IhMxi8GeYO1MuELS6yQdoHp1Q71CYH1J/wAcCTxQ0qskvRn4LHDX+lYrKZ3yrrC9\nTx8aUpVBtbYDXgM8UtLmkt5PGThsQ9tPAL5cvyQAXknZ832TpNMpc7r8jDKOxar37EtDmuzazQ6S\nH43nt1RyRCZiBtKag0ZJ2g04GLgVcCnwN7YfImkv4I3AD2wfpHLZ512B+wHH2f5t5z02cE9GDpX0\nGOCxdZ0PBe5FWe9PAXcEdgD+jXKlw1nAfrbPl7QD8CDgz8BHgBOAU11GSO2FZNdudpD8aDy/JWc7\nt9xy69wAAcuGHtsDuAg4vvPYl4FnA1sC91vL+y2n7jT04QbsU39uC/yIcg5+a+D9wJc7yx0MvL7+\nfgxl+PbBc8uAv6cMwvU+YKNRb1eyazu75Nd+fqO65dRSxBAXN0raUtJBkrax/U3KZY+bSdq4Lvoa\nytgM19k+tx76XuNvqu5ZrnRtgXrizZIOsf0L4D3Aa21fQRkBdaVWTzr3I2APSRvaPgb4pcpw7wAG\nNgFeZPuZtq9d4m2YVrJrNztIfq3nNzKjrqRyy60PN+CWQ/cPAn4CfIgyauaLKaNpfhXYndWnZT8F\nvH3U6z/HbVxef+5BObcuYEPK3u3TgA2AfwZOrMvdBvgScMdRr3uyG8/skl/7+fXhNvIVyC23Ud4o\nh573Bnat9+9SG5m3Ug9ZA7sA51HOW78AeBdwm/rcFsDtRr0d89jewZfAozuPPRr4LrA+8MDa0H4K\nOJcyiFb39cuWal2T3fhml/zaz69Pt3T2jYk1uExR0hOAZ1EaxosoVwGcBzzH9tfqskdRGt43At8E\n/tGd8Ro0w+WhozZNp8lpZ72tV0v83PYxko6mDJ51qu1fLuHqzlmyW2O5prKD5De0XHP59U0KmZg4\n3QZFZVjvnYBPU64OeLbtP0k6HHiY7YfV5V4J/Mr2SZJ2sv3TUa3/2ki6BWXP9cu2z5ll2e6/w30o\nY1TcF7jaZTbgwYipnq4BHoVkt2rZ5rKD5NdZtsn8+iqdfWPiDDUMHwUeAzyesie0q6QVtt8I3FbS\nv0g6EngicEV9/U9h1pFGR2X9+nP/wQOS/o+kBwwv2P13sH0e8Bjbv+w0pLJ9Y58a0mRXtJgdJL+B\nVvPrqxQyMRG6VzRI2lPS2+vd44H72D4X+C3wUMpQ3wBPpVw9cCfgybY/033PvjQy3UbdZU6VrwF/\nkrSDpA2AH1DOua/V8F5kj7Yv2c2ir9lB8qPx/FqQU0sxMSTdzvb/SHoS8F7gtZSrI+5NuTpifeDl\nwAXAnpQBtaY6r+/1oV5JDwJeTRko60O2/zziVVowya5tyS8WU47IxFhSZwK2en9T4ExJD6aMlvk+\n4HLgRZSZZje2/RPKMOabA1+brkNhXxrS7vZJWk/Sc4GjgXfafs90DamkrbV6Mr3eSnbtZgfJr/X8\nWpRCJsaSV08k9yxJ+1Jmin018EJK43l34HTgW8CBwHPr674KHGH72KH368VVEYND2V49x8xmtm+g\nzPp7O8rVHdN9mWxMGQn1Xku8yvOW7NrNDpJf6/m1aMWoVyBiMajMKPs+4L+A7wAvAR5JGQ58R2BT\n4AmUSzq3BDaQtJ7tG2y702j1ZS9wjb1SSc+hbNPnJd1o+8WS7g/cWdKtbP+hdhg0gO1rJP2LG5hQ\nLtm1mx0kv9bza1H6yETzNM209ZIeRWkLPyvpA8B2lMbzT5QJ5f4TOBN4JmVk0T8u7VqvO0l7AM8B\nXgbcA/gKZcCwHYGHA18c7hzZee20Y1mMSrJrNztIfjSe37jIqaVolqQVsOpQ73JJT5Z0h/r0rsAJ\nks4CLrK9p+2rgJX1/PsbKBOxrRw0pMOHhEetdnBExfqSTqj9De5GuRriaOD1wAG2LwG+CPwOeHjn\n32ENfWlIk1272UHyo/H8xk2OyERzJO1j+8zO/YcDRwLrAX+kXP1wI+VqiENsf6MudzBwme3PLf1a\nz52k3Wx/Z5rHvwKcAFxD2aM91PbJ9bl7Uc7Vbwjcqfvv0yfJrt3sIPnReH7jKkdkokUnSDoEQNLT\ngHcCR9reE/gxsB+lMX0v8CZJ/yDpTOAplMs7Vxmcj+8LSfsAb5G0jaQHSnpJZx0/Bmxj+8uU7biF\npK20+pLWv7N9Yc8b0mTXbnaQ/FrPbyylkIlmdA4/H0Q5Rw3wfcpVEXev9z8AbAPsYvt44DhgZ+B9\ntve2/V/d9+zD4d56+HrQYJ4HfB04gHKJ6sOAV0naAljJ6u18NmWwsJMoV368wPb7lnK95yPZtZsd\nJD8az2/suQczV+aW21xvrD4d+pj6cwNK57uPABvUx54PvAe49zSvXz7qbZhtfYAHAZ+jnI/fAjgW\n+CClc+G5wCadZbfq/tsM/n36eEt27WaX/NrPb5xvOSITvTbT4Wfbn64//0LpaPd7ysy5UA4Dfwv4\n2fD7uGeXQHr1mBtHSHqupL+x/S3gq8DLbP/a9hGUPggvAjaijMUxeP2V9fXLXS39Vkwv2bWbHSS/\n1vObJClkolck3aI2LLvCTQ8/d+93GtrLKQNsPUXSVravtP1u29dO97pRkrRssN71qPbWkj5NGap9\nE+Djku5B2Z6NJA0moHs+5QviIuoEel19+JJIdu1mB8mPxvObZClkom/mPYNsbUi+CbxwsJdUX9e3\nzoTLXQfWkrRlXf/NKJelPtX2CZSOg8+mDCb2OeAZkm5p+yqX4c8f0/2S6Jlk1252kPxaz29ipZCJ\nkes2el7HGWRt/8ZDl032YU9QZQyKbev6rJR0K0lvAT4laXPgjsBvJG1VX/Jm4EnA7SmXeV4CbD/0\nnr0ZcyPZtZsdJD8azy+KFDIxcoNGT9KDJJ1BuTrgdbYvtv0X299wmdOkKZJuCRwCHCxphaT1gfcD\nvwYeaft3wDmUYdofUPcafwd8g3IJ61WUPd3zu+/bp0PZya7d7CD50Xh+UWSupRgJdYY2l7QeZbjy\nJwHvsP3JGV6zNfD7Vg7v2r5O0o8pl6Q+BDifMs/M6cCekm4D/AR4F/C/gMdL2o7S7+DK+iVjqV9D\nmye7drOD5Efj+cVNpZCJJTVoGOqh3vUpc638XtJNZpDt7v1o9QyyXwTOGsW6r6NzKB0Jf277Kknf\nBY6iNKK3B46gNKQvpQwm9gvbX+y+QV8a0mTXbnaQ/FrPL2aWKQpiSajOINu5v2oGWWAwg+yxwK+A\nkzw0g2x9zU0mqGuNpBW2/9q5/0ngBNtfHVquN9ua7IoWs4PkN9BqfjG79JGJJTHUkO4B7AH8LfAJ\n4EWS7gpMAVvXx6e7/HNlt3NiH9RD1HNm+6+10+Hhks6lHMr+duf9ejfmxrhmB/O7uqbF7GB885O0\n2XyWbzW/mF0KmVg0GuMZZCU9WNLZwLslPacefp/rF+O1lPP1z7V9mMvAYkCvtm+cs9tL0umSdrZt\nze9KlN5nB2Of34MlTQHvlHRQPU02V03kF/OTU0ux4DTmM8iqXJb6Xsoe7c+BFwM/Bf7V9jXDh7CH\nXjt8mH8wtPmN0y2/1CYgu32BNwC/Aa62/fih52fs3Nn37GAi8jsYOAx4BXA18E/AMba/1jmi0mx+\nsW5yRCYWlCZjBtmNgd2AM2x/D3gHcGvKDL8MihiVK0KGx+pYoyGtnS970ZBOSHZnU74IHwFsKenx\nsHp8kMGXYGvZwcTk9z3g6bZPt/114GJKltQ8ms0v1l0KmbjZ6uHriZhBtu7V/Qb4EvCs+vB36+2+\n9UtkY0kvBvaFmfcQ+3Aoe5KyA3AZK+Sbtq8H3k6dybn2AVkmaQtJ/0gD2cHk5Ue54uh8SYMrbi+l\nDFwHlD5rLeUXCyOFTNwsGpowzfZVwCeBh1ImWfsHyhGMt1Aa2T0kbWL7u7ZfAhxs+1G2vznUKPeS\n7Rvr3vsXgXtJupPtPwMXsnqI95XAbSnb3bvh2gcmLbuBzpfYqcBVko6sj98I/JGS3SbQ3+xgMvOz\n/ae6vYMjKbtS+vYMXEMj+cXCSSETN4vHcAZZSbeUtGP9fcXQc6rbfB5l9M+DAWx/lzL53O1s/4ky\nX8su9bmRb9N0Ji27YS4j1r4eeJykXSQ9EzBl5uad6zIj36aZTHJ+dYdiU8p8SZ+UtJmkJ7h03r2E\nBvKLhZNCJuZFYz6DbG08n0wZOKvb32XHwSL18Yspe/QPr30RDgf+TOmAiO13Aa9e2rVfu0nPbrq9\nc9tnU7b9fGBH29fbfg89yw6S3zT53Y5yNOYYymXUd6mvO4Ue5heLJ4VMzJnGeAbZQSNZG89vAVcP\nvggkPYYyAugaHQZd5mF5LmWE7L+hXNJ5yeD96l5jLw5tJ7ub7p1L2kTSxyhXnm1v+5X18WV9yg6S\nX31++OjKTsDjKKeSHmH7uMH79S2/WFy5/DrWSmWMhi1t/6LevxXwWmB3Soe6B1DmaTnK9pUqs8qe\nD+xFGbPhlcDJ7ky+pp6NnDm8PnXPcHfgKtsXS9rQpR/MXN5rjUs8RynZzZ6dpLvY/vngvSgj3fai\nUUx+a89P0jbAHW2fVe8vo168tBTrHv2RIzIxI03IDLIuV6xsJOl4lXEq7uwy6+/F9fmbNKTT7e31\nrIhJdsyeXbeIsb2yL1+CyW+t+Q0ulb+8W8QMjlgt6QZEL6SQiRnZvg74MaWz4EOAzVlzBtlnAztQ\nZpB9LPDvkr5eX36li94d4pV066H7uwGfBa6kXHl0hqQtp3ndg2pfmGk7EfaliIFkN83rZsuub1/w\nyW/N5br53SSrPv3txdLLqaVYK0mbUC7n/JHtSyQdB9yV1TPIPphyDvsqZphBtk9U5pX5O+DTlC+H\nqylXONyG8sXxL5QOvQfa/v3Qa3cArrd96ZKu9DpKdmu8tqnsIPkNvba5/GLppJCJeVGjM8jWDoCW\ndFvgnykT511FGRDsQEqnwT8A77B96mD57muH32upt+HmSnbtZgfJb7r7EZBTSxNPYz57s4aGnrf9\nK8rVDn8GXls7Up4J3BJ4lu1T60tPkfSU7msH+tSQzufUQbLrXXZjPXvzuOcX/bHWAaNifEl6MHAc\ncKWk04FTXSY8nMseT3cG2XO7T/StofHqQcMOpBy2PhN4OGVP8FGSzrN9tqTTgNdLug64L/Ad4IyR\nrPQsJO0LBrhNAAASaElEQVQFvBQ43PZP5rkHnuxGrP7tvQb4b0lfAt7nMmXCXCS/iCE5tTSBNN6z\nN29MOZd+fb1/F+B9lM6En6OM+/IEytUehwJfsv0RSXcElgN/D3zP9vfr63t1KFtjPHvzuGcHoDGe\nvXkS8ot+SiEzgerppLOB+9r+o6Q9gScCP3QZFXOw3Hq2b5ipQelbQ1MP1T+EMtrnWZQRT2+kzDfz\nDcqcM4+jNKDPkHQQsA+wIXAu8E+dvchefUkMqIwVsgtlr3UKON72x4ePyiS7/mUHIOn+wF9t/6De\nfxfwW9tHDi2X/HqYX/RT+shMGI3Z7M2w+lw85bD7ZpRG8yfAFvXnhZTt/S1wf2B3SXvYPply6efZ\ntl/dbUhd9K4h9fjN3jwx2VVjNXvzBOYXPZRCZsJ4jGZvhlUN3+BIxArKuBu3Bj5k+7Mu43HcGTjf\n9lGUBvdqShGAy9Dtr63vtaw+1osviZl01q/12ZsnMbuxmb15EvOLfkohM4Y0IbM3Q1k3SbtKOhN4\nI/Bu4DnA5pIeVBe7BXB3SYcBHwbeQz0aNfiiqP8uI98LXFt2w9z+7M1jlR3MPT+PwezN45hftClX\nLY0ZrZ5Bdm/g6e7MIGv7IsrVA3aZx+RUyqWOv6V0tltj9ubBXlKfSdoZOBE4FvhEbVx/DDwIeDTw\nLdtfrn1L9gS+YPsd9bWr+hn04Qtjtuym6xfhctXHYPbmN9RTTu9JdktvHfLrzt78JOAUKLM3J7+I\nuev9H0vMzWDvxmM6e/Na3Bv4IfALyiWdL6T8v/4KsIWkT0n6IHCe7X+0/WboV2fJeWTX7OzNM2g+\nO1j3/Gh/9uaxyC/al6uWxoDGdPZmmL3RU5lcb3Cl1fmUcSp+DRwEbEc5jH2i6yR0fWtEb2526vfs\nzWOdHdy8/NTz2ZsnIb8YDylkxoSkjShjUlwEfGXQeKxl+Zs0Kj0sYta6Pp0jDxvUvgVIujPlyoln\nulzhM6f3GqUFyq5vw9JPRHawTvndJKu+beMk5Rfty6mlBmnMZ2/W6isYbpR0C0lH13XfbPj5+vMv\nknaUdCLwGeDjfW1IFzG7XhQx45wdLFh+vZ29edzzi/GUzr6NUZ1BVlJ3BtmNgLeyegbZH1I67g77\nFeWS3V4aNHqDhk/SHShDuW9Fme33Gsqw/NM1jOtTDmvvbvsP3Sf60pAmu3azg+RH4/nF+MqppUYM\nTidoAmaQlfRQ4HmUL4of2n6LpPsArwT+3fZnunt6w3t9WssUC6OQ7NrNDpJf6/nF+MuppZ7TmM8g\nK62+MkPSRpKOAf438H+BuwL3kLQeZS6oLwBPq30MblQZyXZ5p1G9Fay6emTkkl272UHyo/H8YnKk\nkOk5d2aQlfQsSVtRrg44iXLJ4+a2zwYGM8i+R9L3gRvo8QyynXPt3YZ9G8qlqpfZ/jhwJLAtcE+X\nQfo+T/nSOKi+9kavHpr/dcDxKmOq9EKyazc7SH40nl9MjhQyPaMyz9H6nft3kfR14BGUUVs/QhkN\n9NuUYcwfVhd9K/ByyjgWz7Z9iO2ru3tdfVEPrw/25A6S9FqVUU1/CpwA3FvSpra/Telz8AyVK0N+\nQTlv/6HOez2DMindz4H/bfuPS709nXVJdo1mV9cn+TWcX0ww27n15EZpJP8XsBewAbAbZaC6vSgd\ns98GXAG8vy5/EPBR4HRKI7O8814Clo16m4bW5wzg4fX+RpRhzb8APIUyLPszKINsnQC8si63NfAD\n4IFD73dLyoiobwY27sH2JbtGs0t+7eeX22TfRr4CuZlBI0iZdO1ZtfG4BHhkbTS2BL4KvLb+fhGw\nR33Ns4Gjht5Po96mofXZoP48jNKBEMqUCJ8Bdqn3HwX8W/0C2ZvS9+Be9bm7zfC+m/Rg25Jdo9kl\nv/bzyy032zm1NGr1UO9YzyDrOmAWcA5wB0kH1m3+b2C7+m/wGUpHyrtTDld/l3KOHtsXwpqdE+vj\noz4VkewazQ6SH43nFzGQQmbE7PGbQXa40ZN0T0k/BZ4IfAl4XV3mUuA+lAYU4EfAdbXxPcb26d33\n6eGXRLJrNDtIfjSeX8RAxpEZMZUZZN/LmjPIbk05B7++7VfU5Z5AmUH2Uvd48jVNM5KnpP2APW2/\nrN7/LOUQ/uuAo4D7AddTJs97gu3LO6/t3TYOJLt2s4PkR+P5RQykkBkxSQdQzku/k3IO/q7Ax4G7\nAE9j9aiar3admbq+rleNjNYcJGtT4BDgNNsXSnoVZXK8Q+rz9wC+TjkPf5mkhwEb2v70qNZ/XSS7\ndrOD5Nd6fhEDKWQW2WyNnsZsBllJe1MG1doU+CXwFUrHwp9ROhderjK53NeAn9jeZ+j1vZn8MNm1\nmx0kPxrPL2KuUsgsoukO9U73vBqcQbaeZ1+jb4Ckg4B3AQ+1PSXpccCTgJcCzwQeDJxFGeL9Q8C3\nbP9syVd+DpJdu9lB8ms9v4j5SGffRaAxn0F2sGdat29bSTsB2D6ZMgvwznXRcymT6T3f9hsoY0/c\nCnin7ffb/tng36Ivkl272UHyo/H8ItZFjsgsoOFGT2vOIHspcI3tw2d47T0oVxb8q4dmkO2L7qFn\nlRFQX0cZg+I84Me2/0XSkynbsE1dbk/gJZQJ5z419H69OVSf7NrNDpJf6/lF3BypyBdQp8PdQyX9\nJ2UQrR/ZfhRlfpY7S3pUXWbVv31thH9k+xjbf5C0YhTrP5POXuygIb0VpeG37V0oV0G8UNLDbX8E\n+Jmk19SXXwD8K2WulsH7qb5fbxrSZNdudpD8Ws8v4ubIEZmbqbtnozInycsoh3c/DDyf0tHuUMqA\nW0+nDHn+dNcJ1ygZrGqk+rRHKGlXSqfAa+v9e1EaxnMpc8vcmjJ0+3qUzoW3p5yX34XSeXJTdwbO\n6tteYLJrNztIfjSeX8RCyRGZddTZUxrnGWT/D6UDISoDhJ0KnG778LoHfG9gPduPp3QqfAxwqO0f\nAbva/uNgDxD6sxeY7NrNDpJf6/lFLLReHUZtRd27WTWDLKXBPM/2xySdAPyD6gyykgYzyP6U1TPI\n/qzzXs+gXDL5bsp4FSPvWNjpb3AocIGkHSmHqX9BHQm0HoK/AbiNpPsBf0uZk+YSANvfqz971YAm\nu3azg+RXl2k2v4hF4R5M+NTCDSZrBllWT6a3F7Ax5ejd3pQrO3auz90ReCFlePP3ArcZ9Xonu/HK\nLvm1n19uuS32LX1k5kB1rAmV+VYOsX1PScuB04DDbf+4diTcB/gAsAlwOPAy2+dLupvr5GtD77uJ\nG5p8TdLmlL4HW9h+QefxrW1fUX/v2+WqyY42s4PkN9BqfhFLIX1k5sBjOIOspA3m+xqX8TVOB+4t\n6SGdx69Q0buGNNkVLWYHyW+g1fwilkIKmWkMN3oasxlka+fGb0vafh1efiFlj/er3QddjLwhTXZr\n1evsIPnNovf5RYxCTi0NmW7PRmMyg2z9AngsqyfE+yTw9s5e77zWt0/bBsmu5ewg+bWeX8SopJCp\nuo2oxmwGWZXh2a+1fYOk7YCrgB0o88q8xPa5Q8tvafu/l3xF11GyW2P5prKD5De0fHP5RYxaCpkh\nGqMZZCVtSJnd9/aU9T/a9lWd598IbAAc5ToYmMqMwJ8GXm/7iy3t+SW7drOD5Nd6fhGjMrF9ZAYd\n5IYeO4hySefbbD8c+ATwMMp4O68HTpJ0FPAO4JWUQavW0JeGtHoi8FfKnCzLgJdJemjn+eOBe1HG\noQDA9nWUxnTXer93DWmyAxrNDpJf1Wx+EX0zkYXMYE/H4z+D7N6UPde/UOaeuQx4XN3zw/b/UMag\neLykAyWdXM/l/w+lQ6JmeuNRSXbtZgfJr/X8InrJPRjMZqlu1IGm6u/rUwbV+jHwQeAV9fEnA5d3\nltuTsne43zTvp1Fv0wzbuaz+fAzwEUofAoAHUAYBe2Jn2R2AG4GfAI+vj/V2kLBk1152ya/9/HLL\nrc+3Pu/RLBiN8QyyklaoTCi3ildf+XEJZQ/vSfX+hcBvKSObImkr4BXAa23v7DJHDbavWYp1n4tk\n1252kPxoPL+IFox1Z1+N+Qyykg6gXIJ6BWUQsA/a/pGk9VyukrgF8ARgX8pcMj+X9ApgM9uHq4yQ\nutz29fX9Vtj+64g2Zw3Jrt3sIPm1nl9ES8b9iMy4zyC7H/Cy+vNq4GSA2pDKq2f9vRR4j6R9KJ0P\nf1SXW2n7+tr5Uj1rSJNdu9lB8ms9v4hmjGUh0+kEeCjwt5rbDLIH09AMsvXQ9A2Uvdc/uXSIvIWk\n59dF1oPSqdD2K4GPAk8F3mf7/d33crV0az+zZAc0mh0kv7pIs/lFNMk96KizGDcmYAZZYAp4duf+\nnpQ9wG7Hyt1neO2yUa9/shu/7JJf+/nllltrt7HuIzNMYzKDrOrAX/Vw9Xttb9V57lTgVNsfl7QH\n8GjbR3ae7/32TSfZtbF9M0l+bWxfRIuaPbWkMZ9BVtIuku5Qf1+v+1xtSGX7TOD7ko7rPP0Hykii\n2P5mtyGtj418+5Jdu9lB8ms9v4hx02QhozGeQbbzJfEk4AOwqgPhppJ27iw6yO5gYCdJJ0j6APA3\nwLVD79mbnJMd0Gh2kPyqZvOLGEdN/ZHVPbf9gB0pez+PGd477F7pMB3b19k+22Vk0V6NninpxcD/\nlfQo28cAW0naqz79aOBBg2XrnuGyelj+eZQvhwuB3Wxf0n3fnnxJJLuqtewg+dF4fhHjrIk+Mhrz\nGWQl3RP4d8oIn2+kdAb8gaQHAJfOd1vUr8nzkt383q832UHyaz2/iEnQ60JGEzKDrKQXAFvafsUc\nl5+xsezL9ia7GZfvfXaQ/NayfBP5RUySvp9aGssZZFWGNt9H0h1UxtS4P/D1+tx69ee0h94lHQi8\nYLrnoFfbm+xu+toDaSM7SH7TvfZA2skvYmL0vZAZ1xlkb0s5t34/lxE9b0Nt/IGVsLpRHGxrp9Pg\n6dQRU3su2bWbHSS/1vOLmBi9LGQ6DcfHgftL2tD2ZcC3KbPFPrKz+FnAsyhXQ3yuNkKn2f5Kn/aQ\nJN1X0tcl3dv2Lyl7rg+TtBFlVtyHS7pt7Qi5fn3N1pQvlG6nwd/Yvq6vXxTJrt3sIPm1nl/EJBp5\nIaPJmUH274Ff2/5Bvf8+4CSXSfW+Q9m2twK4zMGyHXACZRj3VTkNviD68EWR7NrNDpJf6/lFRDHS\nQkZlBtnzgTdJer2ke9THB4NQ/T/KnuA+ku5i+2rKeft71OevAg6xfXR93Yol3YA56Oy9nQ3cStId\n6/0bbf8EoHaWfCFwa0mfkPQfwBeA79n+d/fwEs5k1252kPxazy8iVhv1EZmxn0G2s/f2E8re36H1\n8VUNpMqVENcCTwGOBM4EHmj72Pp8Hw9lJzuazQ6SH9B0fhFRjayQ0eTNIPs74LPADpK26T7hejmn\n7d/ZvtD2B2z/WtLy+iXRq21Ldqu1lh0kv+4TLeYXEWsaWSFj+0rKDLgP6zQWhwEvrXtJ1wNI2r0u\nf6LtZ9o+qT4+6qNJ81IbzA2ofQxmUxvRlX1sSJPd2vU5O0h+s+l7fhGxppE0SJKW119fD7xu8Ljt\nb1DOy+9Xl9tj8Hvntcvqsk2du67bvD+wHDhO0lPr49Nm0NdGNNm1mx0kPxrPLyJuatFG9pW0C/Bb\n27+UtJ7tG4ael21L+gxlvIqX1cdPBt5q+/xFWbEekPR0YH/bj5J0a9u/HfU6dSW7mfU9O0h+a9NC\nfhExPwt+REaZQXYuvgBcJulnwOv6sn3Jbk56mR0kvznqbX4RsW4W9IiMygyyewKn2P6MpAuA59me\nkvQ04Ba2391ZfpnLIFRbU0bXvAdw7PAe5DiqXzr3s33WqNcFkt189C07SH7z0cf8ImLdLUgho8wg\nOy99uhoi2c1Pn7KD5DdffcsvIm6+hSpkMoNso5Jd25JfREy6dTo/rMwg26xk17bkFxGxpnXt6JYZ\nZNuV7NqW/CIiOuZcyCgzyDYr2bUt+UVEzGw+R2Qyg2y7kl3bkl9ExAxmLWQ6e2+ZQbYxya5tyS8i\nYnZzvmpJ0hbASwBsHzn03HLbKyVtDtyeMrDWGbZ/XZ/P1RAjlOzalvwiImY2n0JmOfAgyt7fi2xf\nPsfX3JiGdLSSXduSX0TEzFbMdcG61zfvGWTXec1iwSS7tiW/iIiZzeeqpcwg26hk17bkFxExs3Ua\n2VeZQbZZya5tyS8iYk3rOiBeZpBtV7JrW/KLiOhY57mWlBlkm5Xs2pb8IiJWW9dTS7mks1HJrm3J\nLyJiTQsy+3VERETEKOT8ekRERDQrhUxEREQ0K4VMRERENCuFTERERDQrhUxEREQ06/8DwjVa9shM\nl0oAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xa402128>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#logistic regression tuning\n",
    "log_reg_parameters = { 'C':np.arange(0.01, 0.03), 'random_state':np.arange(0, 3), 'solver':['newton-cg', 'lbfgs']}\n",
    "pre_processing.tuning(LogisticRegression(multi_class='multinomial'), log_reg_parameters, dev_sparse, dev_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>Comment</b>\n",
    "Since we have to use multi_class:'multinominal', there are two options available for solver method, which are 'newton-cg' and 'lbfgs'. Although, both methods result into the same accuracy score, the execution time for 'newton-cg' method is reasonably less than 'lbfgs' method.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>Comment</b>\n",
    "Our accuracies for two classifiers are below 50% which means none of the classifiers could fit to the development set, despite the fact that we have a huge number of features(#15423). Also, DecisionTree classifier takes much longer time to train than LogisticRegression classifier. This is a challenging task in terms of performance, because the classifier each time tries a different parameter and value, the model has to be retrained each time and the training task is the most expensive task in terms of performance."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>MY LOGISTIC REGRESSION</b>\n",
    "\n",
    "Logistic Regression class from scratch, [only predict and predict proba methods]\n",
    "Initialization parameters are taken from the builtin LogisticRegression classifier which we trained above."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#definition of MyLogisticRegression class, it imitates the builtin LogisticRegression class\n",
    "class MyLogisticRegression:\n",
    "    \n",
    "    #initializes parameters\n",
    "    def __init__(self, weights, constants, labels):\n",
    "        self.weights = weights #coefs\n",
    "        self.constants = constants #intercepts\n",
    "        self.labels = labels #classes\n",
    "\n",
    "    #predicts the probability of each class according to the formula\n",
    "    def predict_proba(self, X):\n",
    "        \n",
    "        result = []\n",
    "        for row in X:\n",
    "\n",
    "            dotProdNeg = np.exp(np.add(np.dot(row.toarray(),self.weights[0]),self.constants[0]))[0]\n",
    "            dotProdNeut = np.exp(np.add(np.dot(row.toarray(),self.weights[1]),self.constants[1]))[0]\n",
    "            dotProdPos = np.exp(np.add(np.dot(row.toarray(),self.weights[2]),self.constants[2]))[0]\n",
    "            Z = dotProdNeg + dotProdNeut + dotProdPos\n",
    "            \n",
    "            dotProdNeg = dotProdNeg/Z;\n",
    "            dotProdNeut = dotProdNeut/Z;\n",
    "            dotProdPos = dotProdPos/Z;\n",
    "            \n",
    "            result.append([dotProdNeg,dotProdNeut,dotProdPos])\n",
    "            \n",
    "        return result;  \n",
    "        \n",
    "    #finds the label\n",
    "    def label(self,y):\n",
    "        return self.labels[y]\n",
    "    \n",
    "    #calculates the probabilities and returns the labels corresponding to the maximum probability\n",
    "    def predict(self, X):\n",
    "\n",
    "        Z = self.predict_proba(X)\n",
    "        sig = []\n",
    "        for row in Z:\n",
    "            dotProdNeg = 1 / (1 + np.exp(-1 * row[0]))\n",
    "            dotProdNeut = 1 / (1 + np.exp(-1 * row[1]))\n",
    "            dotProdPos = 1 / (1 + np.exp(-1 * row[2]))\n",
    "            sig.append([dotProdNeg, dotProdNeut, dotProdPos])    \n",
    "        #finds the labels of maxi-probabilities\n",
    "        labels=map(self.label, np.argmax(sig, axis=1))\n",
    "        \n",
    "        return labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#the definition of class is above\n",
    "my_clf = MyLogisticRegression(log_reg_classifier.coef_, log_reg_classifier.intercept_, log_reg_classifier.classes_)\n",
    "#creating 5 sample data from development set to test if the classifier creates same results\n",
    "sample_data = dev_sparse[0:5]\n",
    "my_predict = my_clf.predict(sample_data)\n",
    "my_predict_proba = my_clf.predict_proba(sample_data)\n",
    "\n",
    "log_reg_predict = log_reg_classifier.predict(sample_data)\n",
    "log_reg_predict_proba = log_reg_classifier.predict_proba(sample_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Classifier Comparison:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict proba : They are equal\n"
     ]
    }
   ],
   "source": [
    "#show that my classifier's predict_proba method creates the same probablities with built-in log. regression\n",
    "#(may have little calc. differences so I used np.allclose method)\n",
    "if np.allclose(np.asarray(my_predict_proba), np.asarray(log_reg_predict_proba)):\n",
    "    print 'Predict proba : They are equal'\n",
    "else:\n",
    "    print 'Predict proba : They are not equal'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict : They are equal\n"
     ]
    }
   ],
   "source": [
    "#show that my classifier's predict method creates the same labels with built-in log. regression\n",
    "if np.array_equal(np.asarray(my_predict), np.asarray(log_reg_predict)):\n",
    "    print 'Predict : They are equal'\n",
    "else:\n",
    "    print 'Predict : They are not equal'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>POLARITY LEXICON</b>\n",
    "\n",
    "There are four different lexicons we are going to use. Sentiword_net, Manual Lexicon, Word2vec, Brown corpus with PPMI."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>SEEDS</b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "positive_seeds = [\"good\",\"nice\",\"excellent\",\"positive\",\"fortunate\",\"correct\",\"superior\",\"great\"]\n",
    "negative_seeds = [\"bad\",\"nasty\",\"poor\",\"negative\",\"unfortunate\",\"wrong\",\"inferior\",\"awful\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>MANUAL LEXICON</b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#manually annotated set\n",
    "positive_words = opinion_lexicon.positive()\n",
    "negative_words = opinion_lexicon.negative()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>1. SENTIWORD _NET </b>\n",
    "\n",
    "Sentiword is a lexicon which has polarity scores for words. We are going to use it in this block.\n",
    "The definitions of the methods are on top of each method. Also, there are explanations for some important operations in the methods. The important method is 'sentiword_net_calc()'. This method returns two lists which are lists of positive and negative words in sysnset. We decide whether a word is positive or negative by looking all of the lemmas of that word and we get the most common polarity score as the result."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "class SentiWord_Class:\n",
    "\n",
    "    #this method compares pos_score and neg_score of synset_name, and with obj_score, then selects the bigger one as polarity type\n",
    "    def get_polarity_type(self, synset_name):\n",
    "        swn_synset =  swn.senti_synset(synset_name)#brings synset\n",
    "        if not swn_synset:\n",
    "            return None\n",
    "        elif swn_synset.pos_score() > swn_synset.neg_score() and swn_synset.pos_score() > swn_synset.obj_score():\n",
    "            return 1\n",
    "        elif swn_synset.neg_score() > swn_synset.pos_score() and swn_synset.neg_score() > swn_synset.obj_score():\n",
    "            return -1\n",
    "        else:\n",
    "            return 0\n",
    "\n",
    "    #returns the global resulting positive and negative lists \n",
    "    def sentiword_net_calc(self):\n",
    "        syn_pos_list = []\n",
    "        syn_neg_list = []\n",
    "        #move through all synsets\n",
    "        for synset in wn.all_synsets():\n",
    "            #select each lemmas in the synsets\n",
    "            for lemma in synset.lemmas():\n",
    "                #take lemma_list for each lemma\n",
    "                lemma_list = list(swn.senti_synsets(str(lemma.name())))\n",
    "                lemma_dict = {}\n",
    "                for lem in lemma_list:\n",
    "                    count = self.get_polarity_type(lem.synset.name())\n",
    "                    #dictionary collects positive, negative and neutral count of lemmas\n",
    "                    lemma_dict[count] = lemma_dict.get(count, 0) + 1\n",
    "\n",
    "                #filter the dictionary according to the number of positive, negative and neutral counts\n",
    "                key, _ = max(lemma_dict.iteritems(), key=lambda x:x[1])    \n",
    "                if key == 1:\n",
    "                    syn_pos_list.append(str(lemma.name()))\n",
    "                elif key == -1:\n",
    "                    syn_neg_list.append(str(lemma.name()))\n",
    "\n",
    "        return syn_pos_list, syn_neg_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sentiword_net = SentiWord_Class()#creates an instance from SentiWord_Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#import sentiword_net\n",
    "\n",
    "syn_pos_list = []#the list of positive words in synset\n",
    "syn_neg_list = []#the list of negative words in synset\n",
    "#syn_pos_list, syn_neg_list = sentiword_net.sentiword_net_calc()\n",
    "syn_pos_list, syn_neg_list = sentiword_net.sentiword_net_calc()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(\"'boffo'\", False)\n",
      "(\"'trustfulness'\", False)\n",
      "(\"'lasciviousness'\", False)\n",
      "(\"'free-spoken'\", False)\n",
      "(\"'reenlistment'\", False)\n"
     ]
    }
   ],
   "source": [
    "#this block takes 5 random value from syn_pos_list and compares the results with positive words list of manual annotated set\n",
    "i = 0\n",
    "while(i<5):\n",
    "    i += 1\n",
    "    random_index = randrange(0,len(syn_pos_list))\n",
    "    print ('\\''+syn_pos_list[random_index]+'\\'' , syn_pos_list[random_index] in set(positive_words))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(\"'flub'\", False)\n",
      "(\"'Pythium_debaryanum'\", False)\n",
      "(\"'scrofulous'\", False)\n",
      "(\"'thankless'\", True)\n",
      "(\"'laxity'\", False)\n"
     ]
    }
   ],
   "source": [
    "#this block takes 5 random value from syn_neg_list and compares the results with negative words list of manual annotated set\n",
    "i = 0\n",
    "while(i<5):\n",
    "    i += 1\n",
    "    random_index = randrange(0,len(syn_neg_list))\n",
    "    print ('\\''+syn_neg_list[random_index]+'\\'' , syn_neg_list[random_index] in set(negative_words))\n",
    "     "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>Comment:</b>\n",
    "Note that FALSE results only show that the word is not included in the manually annotated lexicon.\n",
    "By intuition the words \"confirmable, neat and hearty\" look positive words and the lexicon identified them as positive, thefore it is correct. However since they are not in the manually annotated lexicon the result was False for them. Because, the size of the manually annotated set is very small. Since the lexicon finds the most common polarity among the lemmas of the words, that may help to better identification."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>2. COSINE SIMILARITY</b>\n",
    "\n",
    "This block includes the Cosine similarity. We will calculate cosine similarity between our seeds above and the words in gensim vovabulary."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "word2vec_sample = str(find('models/word2vec_sample/pruned.word2vec.txt'))\n",
    "model = gensim.models.Word2Vec.load_word2vec_format(word2vec_sample, binary=False)\n",
    "\n",
    "class CosSimilarity_Class:\n",
    "\n",
    "    def cos_similarity(self, positive_seeds, negative_seeds):\n",
    "\n",
    "        #calculating cosine similarity between our seeds and the words in gensim vocabulary, creates matrixes\n",
    "        pos = cosine_similarity(model[positive_seeds], model[model.vocab.keys()])\n",
    "        neg = -1 * cosine_similarity(model[negative_seeds], model[model.vocab.keys()])\n",
    "\n",
    "        #creating matrixes' transposes, because we need a matrix showing cos distances of each word in the vocab to each seed words\n",
    "        #for example the transpose of pos matrix will create a 8 rows and len(vocab) columns matrix\n",
    "        pos = pos.T \n",
    "        neg = neg.T\n",
    "\n",
    "        total_cs = pos + neg\n",
    "\n",
    "        all_words = np.asarray(model.vocab.keys())\n",
    "        #finding the average of value in the matrix\n",
    "        total_cs_avg = np.average(total_cs, axis = 1)\n",
    "\n",
    "        #filtering the words according to the total average and creating pos and neg words lists\n",
    "        pos_words = all_words[np.where(total_cs_avg>0.03)[0]]\n",
    "        pos_words_score = total_cs_avg[np.where(total_cs_avg>0.03)[0]]\n",
    "\n",
    "        neg_words = all_words[np.where(total_cs_avg<-0.03)[0]]\n",
    "        neg_words_score = total_cs_avg[np.where(total_cs_avg<-0.03)[0]]\n",
    "\n",
    "        return pos_words, pos_words_score, neg_words, neg_words_score\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cosine_sim = CosSimilarity_Class()#creates an instance from CosSimilarity_Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cs_pos_list, cs_pos_words_score, cs_neg_list, cs_neg_words_score = cosine_sim.cos_similarity(positive_seeds, negative_seeds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(u\"'champion'\", True)\n",
      "(u\"'reconnaissance'\", False)\n",
      "(u\"'toasted'\", False)\n",
      "(u\"'undying'\", False)\n",
      "(u\"'strategic'\", False)\n"
     ]
    }
   ],
   "source": [
    "#this block takes 5 random value from cs_pos_list and compares the results with positive words list of manual annotated set\n",
    "\n",
    "i = 0\n",
    "while(i<5):\n",
    "    i += 1\n",
    "    random_index = randrange(0,len(cs_pos_list))\n",
    "    print ('\\''+cs_pos_list[random_index]+'\\'', cs_pos_list[random_index] in set(positive_words))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(u\"'unfertilized'\", False)\n",
      "(u\"'rubbed'\", False)\n",
      "(u\"'numbing'\", False)\n",
      "(u\"'beards'\", False)\n",
      "(u\"'Policies'\", False)\n"
     ]
    }
   ],
   "source": [
    "#this block takes 5 random value from cs_neg_list and compares the results with negative words list of manual annotated set\n",
    "i = 0\n",
    "while(i<5):\n",
    "    i += 1\n",
    "    random_index = randrange(0,len(cs_neg_list))\n",
    "    print ('\\''+cs_neg_list[random_index]+'\\'' , cs_neg_list[random_index] in set(negative_words))\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>Comment:</b>\n",
    "It identifies the positive words like 'complement', 'imaginative' as positive and negative words like 'scream' as negative. Some of the words such as 'deport' classified as negative, however they intuitively look like neutral. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#creates dictionaries of pos and neg words to increase performance\n",
    "cs_pos_dict = dict(zip(cs_pos_list, cs_pos_words_score))\n",
    "cs_neg_dict = dict(zip(cs_neg_list, cs_neg_words_score))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>3. PPMI</b>\n",
    "\n",
    "This block contains calculating PPMI with the help of Brown corpus. The blok consists of:\n",
    "preprocessing step, \n",
    "creating pmi matrix step, \n",
    "calculating total and \n",
    "finding average(filtering) step. THere are definitions on top of each methods and explanations for important code blocks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "class PPMI_Class:\n",
    "    \n",
    "    def init():\n",
    "        pass\n",
    "    \n",
    "    #preprocessing of PPMI\n",
    "    def prep_ppmi(self, pos_set, neg_set):\n",
    "        #finding the frequency of each word in brown corpus, but counting only once in each file\n",
    "        word_freq = {}\n",
    "        files = brown.fileids()\n",
    "        length = len(files)\n",
    "        for i in range(length):\n",
    "            cur_file_words = brown.words(files[i])\n",
    "            for j in range(len(cur_file_words)):\n",
    "                    word_freq[str(files[i]) + '_' + cur_file_words[j]] = 1  #count only once since it is binary\n",
    "\n",
    "        #calculating frequency\n",
    "        word_freq_n = {}\n",
    "        for word in word_freq:\n",
    "        #     pdb.set_trace()\n",
    "            word = re.sub(r'.+_', '', word, flags=re.IGNORECASE)\n",
    "            word_freq_n[word] = word_freq_n.get(word, 0) + 1\n",
    "\n",
    "        word_freq = word_freq_n #we put every single word counts into dictionary word_freq\n",
    "\n",
    "        #co-ocurrence\n",
    "        #finds each word combinations appearing together in a file\n",
    "        #this process doesn't get in sentence level(like 'good idea' appearing together), it worksin file level\n",
    "        #namely it is enough for us to consider the two words have cooccurence if they appear in the same text, (no location concern) \n",
    "        word_freq_co = {}\n",
    "        files = brown.fileids()\n",
    "        length = len(files)\n",
    "        for i in range(length):\n",
    "            cur_file_words = brown.words(files[i])\n",
    "            #pdb.set_trace()\n",
    "            for j in range(len(cur_file_words)):\n",
    "                if (cur_file_words[j] in pos_set or cur_file_words[j] in neg_set):\n",
    "                    for doc_word in cur_file_words:\n",
    "                        #we are adding the file_id to the front in order to escape from duplicates\n",
    "                        word_freq_co[str(files[i]) + '_' + cur_file_words[j] + '-' + doc_word ] = 1\n",
    "\n",
    "        #calculating frequencies of cooccurences\n",
    "        word_freq_con = {}\n",
    "        for collocation in word_freq_co:\n",
    "            collocation = re.sub(r'.+_', '', collocation, flags=re.IGNORECASE)\n",
    "            word_freq_con[collocation] = word_freq_con.get(collocation, 0) + 1\n",
    "\n",
    "        return word_freq, word_freq_con\n",
    "\n",
    "\n",
    "    #creating the matrixes for representing ppmi results of positive seeds and words in brown corpus \n",
    "    def create_pmi_matrix(self, word_freq, word_freq_con, positive_seeds, negative_seeds):\n",
    "        pmi_matrix_pos = []\n",
    "        total_count = len(brown.words());\n",
    "        for seed in positive_seeds:\n",
    "            row = []\n",
    "            for word in word_freq:\n",
    "                both_count = word_freq_con.get(seed + '-' + word, 0)\n",
    "                word1_count = word_freq.get(seed, 0);\n",
    "                word2_count = word_freq.get(word, 0);\n",
    "\n",
    "                ppmi = math.log((both_count/total_count)/((word1_count/total_count)*(word2_count/total_count)), 2) if both_count!=0 else 0\n",
    "                #ppmi doesn't get negative results\n",
    "                row.append( ppmi if  ppmi > 0 else 0)\n",
    "            pmi_matrix_pos.append(row);\n",
    "\n",
    "        #creating the matrixes for representing ppmi results of negative seeds and words in brown corpus    \n",
    "        pmi_matrix_neg = []\n",
    "        for seed in negative_seeds:\n",
    "            row = []\n",
    "            for word in word_freq:\n",
    "                both_count = word_freq_con.get(seed + '-' + word, 0)\n",
    "                word1_count = word_freq.get(seed, 0);\n",
    "                word2_count = word_freq.get(word, 0);\n",
    "\n",
    "                ppmi = math.log((both_count/total_count)/((word1_count/total_count)*(word2_count/total_count)), 2)  if both_count!=0 else 0\n",
    "                #ppmi doesn't get negative results\n",
    "                row.append( ppmi if  ppmi > 0 else 0)\n",
    "                #row.append( math.log((both_count/total_count)/((word1_count/total_count)*(word2_count/total_count)), 2) if both_count!=0 else 0)\n",
    "            pmi_matrix_neg.append(row);\n",
    "\n",
    "        return pmi_matrix_pos, pmi_matrix_neg\n",
    "\n",
    "\n",
    "    #creating matrixes' transposes, because we need a matrix showing ppmi of each word in the vocab to each seed words\n",
    "    #for example the transpose of pmi_pos matrix will create a 8 rows and len(vocab) columns matrix\n",
    "    def create_total_avg_matrix(self, pmi_matrix_pos, pmi_matrix_neg):\n",
    "        pmi_matrix_pos = np.asarray(pmi_matrix_pos).T\n",
    "        pmi_matrix_neg = (-1*np.asarray( pmi_matrix_neg)).T\n",
    "        total = pmi_matrix_pos + pmi_matrix_neg\n",
    "        #then we get the total and average\n",
    "        total_avg = np.average(total,axis = 1)\n",
    "        # pmi_matrix_avg_pos = np.average(pmi_matrix_pos,axis=1)\n",
    "        # pmi_matrix_avg_neg = np.average(pmi_matrix_neg,axis=1)\n",
    "        return total_avg\n",
    "\n",
    "    #filtering the words according to the total average and creating pos and neg words lists    \n",
    "    def filter(self, word_freq, total_avg):\n",
    "\n",
    "        words =  np.asarray(word_freq.keys())\n",
    "        pmi_positive_words = words[np.where(total_avg>0.03)[0]]\n",
    "        pos_words_score = total_avg[np.where(total_avg>0.03)[0]]\n",
    "\n",
    "        #the formula below could be used to get most importants, @@TODO\n",
    "        # pos_dict = sorted(pos_dict.items(), key=operator.itemgetter(1),reverse=True)\n",
    "\n",
    "        pmi_negative_words = words[np.where(total_avg<-0.03)[0]]\n",
    "        neg_words_score = total_avg[np.where(total_avg<-0.03)[0]]\n",
    "\n",
    "        # neg_dict = sorted(neg_dict.items(), key=operator.itemgetter(1),reverse=True)\n",
    "\n",
    "        return pmi_positive_words, pos_words_score, pmi_negative_words, neg_words_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#put seeds into set for better performance\n",
    "pos_set = set(positive_seeds)\n",
    "neg_set = set(negative_seeds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#import ppmi\n",
    "ppmi = PPMI_Class()#creates a new instance of PPMI_CLass\n",
    "word_freq, word_freq_con = ppmi.prep_ppmi(pos_set, neg_set)\n",
    "pmi_matrix_pos, pmi_matrix_neg = ppmi.create_pmi_matrix(word_freq, word_freq_con, positive_seeds, negative_seeds)\n",
    "total_avg = ppmi.create_total_avg_matrix(pmi_matrix_pos, pmi_matrix_neg)\n",
    "pmi_positive_list, pmi_pos_words_score, pmi_negative_list, pmi_neg_words_score = ppmi.filter(word_freq, total_avg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(u\"'complicate'\", False)\n",
      "(u\"'staging'\", False)\n",
      "(u\"'ferret'\", False)\n",
      "(u\"'Lavaughn'\", False)\n",
      "(u\"'Corp.'\", False)\n"
     ]
    }
   ],
   "source": [
    "#this block takes 5 random value from pmi_positive_list and compares the results with positive words list of manual annotated set\n",
    "i = 0\n",
    "while(i<5):\n",
    "    i += 1\n",
    "    random_index = randrange(0,len(pmi_positive_list))\n",
    "    print ('\\''+pmi_positive_list[random_index]+'\\'' , pmi_positive_list[random_index] in set(positive_words))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(u\"'favors'\", False)\n",
      "(u\"'defiantly'\", True)\n",
      "(u\"'barley'\", False)\n",
      "(u\"'platitudinous'\", False)\n",
      "(u\"'hotel's'\", False)\n"
     ]
    }
   ],
   "source": [
    "#this block takes 5 random value from pmi_negative_list and compares the results with negative words list of manual annotated set\n",
    "i = 0\n",
    "while(i<5):\n",
    "    i += 1\n",
    "    random_index = randrange(0,len(pmi_negative_list))\n",
    "    print ('\\''+pmi_negative_list[random_index]+'\\'' , pmi_negative_list[random_index] in set(negative_words))\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>Comment:</b>\n",
    "It was really hard to find a word that looks intuitively positive and identified correct in this lexicon. This might be because of the size of documents and words in the brown corpus. A positive word 'gentle' is recognised as positive which is true, however a negative word like 'inaccuracies' identified as positive, which is wrong. Also, many of the neutral words identified as positive or negative. Since we look words appearing in the same text level(not in sentence level), that may not give us valid information about the word's polarity, as polarity can vary in a text."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pmi_pos_dict = dict(zip(pmi_positive_list, pmi_pos_words_score))\n",
    "pmi_neg_dict = dict(zip(pmi_negative_list, pmi_neg_words_score))\n",
    "\n",
    "#print len(np.asarray(word_freq.keys())[total_avg<-0.03])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>ACCURACY RESULTS OF POLARITY LEXICONS</b>\n",
    "\n",
    "We are going to use a generic class which is Acc_Calculator as defined below for evaluating the accuracies of the lexicons."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "class Acc_Calculator:\n",
    "    #accuracy calculator method\n",
    "    def acc_calc(self, pos_list, neg_list, positive_words, negative_words ):\n",
    "\n",
    "        #filtering automatically produced words \n",
    "        syn_neg_words = [s.lower() for s in neg_list]\n",
    "        #taking automatically produced negative words overlapping with manually annotated set\n",
    "        syn_neg_words = set(syn_neg_words) & (set(positive_words) | set(negative_words)) #correct neg words\n",
    "\n",
    "        syn_pos_words = [s.lower() for s in pos_list]\n",
    "        #taking automatically produced positive words overlapping with manually annotated set\n",
    "        syn_pos_words = set(syn_pos_words) & (set(positive_words) | set(negative_words)) #correct pos words\n",
    "\n",
    "        #accuracy result will be the division of correct_results by the all words\n",
    "        acc = (len(set(syn_neg_words) & set(negative_words) ) + len(set(syn_pos_words) & set(positive_words)))/len(set(syn_neg_words) | set(syn_pos_words))\n",
    "        return acc, syn_pos_words, syn_neg_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lex_acc_calc = Acc_Calculator()#creates an instance of Acc_Calculator class"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>ACCURACY SENTIWORD_NET</b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.845197740113\n"
     ]
    }
   ],
   "source": [
    "acc_senti_word, syn_pos_words, syn_neg_words = lex_acc_calc.acc_calc(syn_pos_list, syn_neg_list, positive_words,negative_words)\n",
    "print acc_senti_word"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>ACCURACY PPMI</b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.479035166817\n"
     ]
    }
   ],
   "source": [
    "acc_pmi, pmi_pos_words, pmi_neg_words = lex_acc_calc.acc_calc(pmi_pos_dict.keys(), pmi_neg_dict.keys(), positive_words, negative_words )\n",
    "print acc_pmi"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>ACCURACY COSINE SIMILARITY</b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.967262723521\n"
     ]
    }
   ],
   "source": [
    "acc_cos_sim, cs_pos_words, cs_neg_words = lex_acc_calc.acc_calc(cs_pos_dict.keys(), cs_neg_dict.keys(), positive_words, negative_words )\n",
    "print acc_cos_sim"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>TESTING AUTOMATICALLY-PRODUCED LEXICONS AGAINST A MANUALLY-ANNOTATED SET</b>\n",
    "\n",
    "For this task we are going to use the class Lex_Polarity as defined below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "class Lex_Polarity:\n",
    "\n",
    "    #this method calculates the polar score of a sentence according to one of the three lexicons sent as a parameter\n",
    "    def calculate_polar_score(self, sentence, lexicon):\n",
    "        lex_pos_set = lexicon[0]\n",
    "        lex_neg_set = lexicon[1]\n",
    "\n",
    "        pos_count = 0\n",
    "        neg_count = 0\n",
    "        #moves throgh the words in sentence and finds out if they are in pos_list or neg_list\n",
    "        for word in sentence:\n",
    "            if word in lex_pos_set:\n",
    "                pos_count += 1\n",
    "            elif word in lex_neg_set:\n",
    "                neg_count += 1\n",
    "\n",
    "        #then returns the polar score\n",
    "        if pos_count > neg_count:\n",
    "            return 1\n",
    "        elif pos_count < neg_count:\n",
    "            return -1\n",
    "        else:\n",
    "            return 0\n",
    "\n",
    "    #this method calculates accuracy of lexicons\n",
    "    #most common usage is lex_a is one of automatically created lexicons {cs_lex, ppmi_lex, syn_lex} and\n",
    "    #lex_m is manual lexicon. Since we will test these automatically-produced lexicons against the manually-annotated lexicon\n",
    "    #also this method typically works on development set\n",
    "    def get_acc(self, tweets, lex_a, lex_m):\n",
    "        correct_pred = 0\n",
    "        for tweet in tweets:\n",
    "            #itertools.chain(): flattens the tweet, as we have seperated sentences in the tweet\n",
    "            tweet = itertools.chain(*tweet)\n",
    "            pred = self.calculate_polar_score(tweet, lex_a)\n",
    "            act = self.calculate_polar_score(tweet, lex_m)\n",
    "            if(pred == act):\n",
    "                correct_pred += 1\n",
    "        return correct_pred/len(tweets)\n",
    "\n",
    "    #calculates polarity score in given tweet_set with respect to the given lexicon\n",
    "    def get_polarity_score(self, lexicon, tweet_set):\n",
    "        score = []\n",
    "        for tweet in tweet_set:\n",
    "            tweet = itertools.chain(*tweet)\n",
    "            score.append(self.calculate_polar_score(tweet, lexicon))\n",
    "        return score\n",
    "\n",
    "    #We will create polarity vectors and will add them into the sparse matrix as new features\n",
    "    def get_polarity_vectors(self, dev_sparse, tweet_set, cs_lex, ppmi_lex, syn_lex, man_lex):\n",
    "        #we need to get transposes of vectors, as we need each polarity score for each tweet\n",
    "        cs_pol_vector = np.asarray(self.get_polarity_score(cs_lex, tweet_set))[np.newaxis].T\n",
    "        ppmi_pol_vector = np.asarray(self.get_polarity_score(ppmi_lex, tweet_set))[np.newaxis].T\n",
    "        syn_pol_vector = np.asarray(self.get_polarity_score(syn_lex, tweet_set))[np.newaxis].T\n",
    "        man_pol_vector = np.asarray(self.get_polarity_score(man_lex, tweet_set))[np.newaxis].T\n",
    "        dev_sparse2 = dev_sparse.toarray()\n",
    "\n",
    "        #adding the vectors to sparse matrix\n",
    "        dev_sparse2 = np.append(dev_sparse2, cs_pol_vector, axis=1)\n",
    "        dev_sparse2 = np.append(dev_sparse2, ppmi_pol_vector, axis=1)\n",
    "        dev_sparse2 = np.append(dev_sparse2, syn_pol_vector, axis=1)\n",
    "        dev_sparse2 = np.append(dev_sparse2, man_pol_vector, axis=1)\n",
    "\n",
    "        #from scipy import sparse\n",
    "        dev_sparse = sparse.csr_matrix(dev_sparse2)\n",
    "        return dev_sparse\n",
    "\n",
    "    def get_polarity_vectors_imp(self, dev_sparse, tweet_set, cs_lex):\n",
    "        #we need to get transposes of vectors, as we need each polarity score for each tweet\n",
    "        cs_pol_vector = np.asarray(self.get_polarity_score(cs_lex, tweet_set))[np.newaxis].T\n",
    "        dev_sparse2 = dev_sparse.toarray()\n",
    "        dev_sparse2 = np.append(dev_sparse2, cs_pol_vector, axis=1)\n",
    "        #from scipy import sparse\n",
    "        dev_sparse = sparse.csr_matrix(dev_sparse2)\n",
    "        return dev_sparse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lex_polarity = Lex_Polarity()# creates an instance of Lex_Polarity class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#import lex_polarity\n",
    "\n",
    "#all lexicons we created so far, represented here as tuples\n",
    "cs_lex = (set(cs_pos_words), set(cs_neg_words))#cosine sim lexicon\n",
    "ppmi_lex = (set(pmi_pos_words),set(pmi_neg_words))#ppmi lexicon\n",
    "syn_lex = (set(syn_pos_words),set(syn_neg_words))#sentiword lexicon\n",
    "man_lex = (set(positive_words),set(negative_words))#manual lexicon"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>A small test</b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n"
     ]
    }
   ],
   "source": [
    "#a small test of a polar score method via ppmi lexicon\n",
    "sentence = \"good habit\"\n",
    "sentence = sentence.split()\n",
    "print lex_polarity.calculate_polar_score(sentence, ppmi_lex)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>Accuracies</b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#We are gettting the accuracies against the manually annotated lexicon(man_lex)\n",
    "cs_lex_accuracy = lex_polarity.get_acc(dev_tweets, cs_lex, man_lex)\n",
    "ppmi_lex_accuracy = lex_polarity.get_acc(dev_tweets, ppmi_lex, man_lex)\n",
    "syn_lex_accuracy = lex_polarity.get_acc(dev_tweets, syn_lex, man_lex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.476216511755\n"
     ]
    }
   ],
   "source": [
    "print cs_lex_accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.358119190815\n"
     ]
    }
   ],
   "source": [
    "print ppmi_lex_accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.835975943138\n"
     ]
    }
   ],
   "source": [
    "print syn_lex_accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>Comment:</b>\n",
    "The results show that Sentiwordnet lexicon is the most successful one. Intuitevely speaking, the word without context can be determined as positive, negative or neutral, by looking at its most common polarity among its lemmas. That makes this lexicon stronger than others. Therefore, sentiword_net lexicon resulted in high accuracy."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>ADDING POLARITIES TO THE DATA</b> Includes new \"convert_to_feature_dict function\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#creating and adding polarity vectors[all of them] for each training and development datas\n",
    "new_train_sparse = lex_polarity.get_polarity_vectors(sparse.csr_matrix(train_sparse), train_tweets, cs_lex, ppmi_lex, syn_lex, man_lex)\n",
    "new_dev_sparse = lex_polarity.get_polarity_vectors(sparse.csr_matrix(dev_sparse), dev_tweets, cs_lex, ppmi_lex, syn_lex, man_lex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.513395297977\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "         -1       0.42      0.28      0.33       360\n",
      "          0       0.46      0.68      0.55       700\n",
      "          1       0.66      0.47      0.55       769\n",
      "\n",
      "avg / total       0.53      0.51      0.51      1829\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#classifier with the best parameters\n",
    "log_reg_classifier = LogisticRegression(solver='lbfgs', multi_class='multinomial', C=0.01, random_state=0)\n",
    "#we will train the classifier with the new sparse document, which has the new features we defined and added as vectors above\n",
    "log_reg_classifier.fit(new_train_sparse, train_labels)\n",
    "pre_processing.classifying(log_reg_classifier, new_dev_sparse, dev_labels)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>Comment:</b>\n",
    "I added the polarity scores to the training data and development data as new features. However, the accuracy of the classifier did not improve, since, the accuracies of our polarity lexicons were not very high. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>ERROR ANALYSIS AND IMPROVEMENT</b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[([u'2nd', u'computer', u'with', u'same', u'error', u'!!'],\n",
       "  0.50648623157565975,\n",
       "  0.32756308921587241,\n",
       "  -1,\n",
       "  0),\n",
       " ([u'hopefully',\n",
       "   u'it',\n",
       "   u'works',\n",
       "   u'out',\n",
       "   u'for',\n",
       "   u'dev',\n",
       "   u'to',\n",
       "   u'replace',\n",
       "   u'my',\n",
       "   u'laptop',\n",
       "   u'=)'],\n",
       "  0.45835811663184661,\n",
       "  0.32819928431754086,\n",
       "  1,\n",
       "  0),\n",
       " ([u'spent',\n",
       "   u'over',\n",
       "   u'40',\n",
       "   u'mins',\n",
       "   u'&',\n",
       "   u'gave',\n",
       "   u'up',\n",
       "   u'coz',\n",
       "   u'it',\n",
       "   u'changes',\n",
       "   u'just',\n",
       "   u'before',\n",
       "   u'you',\n",
       "   u'print',\n",
       "   u'every',\n",
       "   u'time',\n",
       "   u'.'],\n",
       "  0.48199944315755483,\n",
       "  0.41640652713230353,\n",
       "  1,\n",
       "  0),\n",
       " ([u'i', u'will', u'be', u'at', u'the', u'race', u'on', u'sun'],\n",
       "  0.45241036233034393,\n",
       "  0.38078293676807834,\n",
       "  1,\n",
       "  0),\n",
       " ([u'you', u\"'\", u've', u'got', u'a', u'month', u'.'],\n",
       "  0.3943898562585505,\n",
       "  0.24524702181480107,\n",
       "  -1,\n",
       "  0),\n",
       " ([u'2nd', u'try', u'!'], 0.40757539904793399, 0.34411394431622883, -1, 1),\n",
       " ([u'on',\n",
       "   u'hold',\n",
       "   u'with',\n",
       "   u'support',\n",
       "   u'for',\n",
       "   u'52',\n",
       "   u'minutes',\n",
       "   u'now',\n",
       "   u'.'],\n",
       "  0.39226126490558011,\n",
       "  0.24074105642177893,\n",
       "  -1,\n",
       "  0),\n",
       " ([u'gotta',\n",
       "   u'find',\n",
       "   u'someone',\n",
       "   u'else',\n",
       "   u'to',\n",
       "   u'fix',\n",
       "   u'the',\n",
       "   u'drive',\n",
       "   u'.'],\n",
       "  0.3911498938196421,\n",
       "  0.36193024000599688,\n",
       "  0,\n",
       "  -1),\n",
       " ([u'this', u'is', u'the', u'3rd', u'picture', u'they', u'show', u'me', u'.'],\n",
       "  0.42416721919958278,\n",
       "  0.23507739428675478,\n",
       "  1,\n",
       "  0),\n",
       " ([u'here',\n",
       "   u'today',\n",
       "   u',',\n",
       "   u'gone',\n",
       "   u'tomorrow',\n",
       "   u'-',\n",
       "   u'but',\n",
       "   u'still',\n",
       "   u'here',\n",
       "   u'!'],\n",
       "  0.54142311259185216,\n",
       "  0.39061619437452455,\n",
       "  1,\n",
       "  0),\n",
       " ([u'this', u'city', u'is', u'changing', u'.'],\n",
       "  0.47097212801252497,\n",
       "  0.3647989383204866,\n",
       "  1,\n",
       "  0),\n",
       " ([u'20',\n",
       "   u'years',\n",
       "   u'since',\n",
       "   u'we',\n",
       "   u\"'\",\n",
       "   u've',\n",
       "   u'come',\n",
       "   u'to',\n",
       "   u'love',\n",
       "   u'you',\n",
       "   u'(&',\n",
       "   u'backward',\n",
       "   u'compatibility',\n",
       "   u')'],\n",
       "  0.49612789841940397,\n",
       "  0.32483991063066747,\n",
       "  -1,\n",
       "  0),\n",
       " ([u'for',\n",
       "   u'the',\n",
       "   u'2nd',\n",
       "   u'time',\n",
       "   u'i',\n",
       "   u'have',\n",
       "   u'been',\n",
       "   u'charged',\n",
       "   u'for',\n",
       "   u'something',\n",
       "   u'i',\n",
       "   u'wasnt',\n",
       "   u'made',\n",
       "   u'aware',\n",
       "   u'of',\n",
       "   u'.'],\n",
       "  0.49827372954589194,\n",
       "  0.44707102980498908,\n",
       "  1,\n",
       "  0),\n",
       " ([u'plz',\n",
       "   u'add',\n",
       "   u'l1520',\n",
       "   u'in',\n",
       "   u'the',\n",
       "   u'1st',\n",
       "   u'wave',\n",
       "   u'of',\n",
       "   u'windows10',\n",
       "   u'phones',\n",
       "   u'release',\n",
       "   u'.',\n",
       "   u'plz',\n",
       "   u'dont',\n",
       "   u'hurt',\n",
       "   u'ur',\n",
       "   u'diehardfans'],\n",
       "  0.37030989360193745,\n",
       "  0.29867113995241107,\n",
       "  -1,\n",
       "  0),\n",
       " ([u'history', u'always', u'repeats', u'!'],\n",
       "  0.44173233846094984,\n",
       "  0.3087132114573331,\n",
       "  1,\n",
       "  0),\n",
       " ([u':)'], 0.49573789394726009, 0.4043464374486091, 1, 0),\n",
       " ([u'startup', u'your', u'monday', u'in', u'the', u'right', u'way', u'.'],\n",
       "  0.48501678857767705,\n",
       "  0.43728271668335339,\n",
       "  1,\n",
       "  0),\n",
       " ([u'predictive',\n",
       "   u'analytics',\n",
       "   u'withazuremachine',\n",
       "   u'learning',\n",
       "   u'2nd',\n",
       "   u'ed',\n",
       "   u'.'],\n",
       "  0.47423470111829913,\n",
       "  0.45930999216159113,\n",
       "  1,\n",
       "  0),\n",
       " ([u':)'], 0.44924756494780549, 0.41257325702406522, 1, 0),\n",
       " ([u'i',\n",
       "   u'don',\n",
       "   u\"'\",\n",
       "   u't',\n",
       "   u'have',\n",
       "   u'internet',\n",
       "   u'until',\n",
       "   u'thursday',\n",
       "   u'and',\n",
       "   u'my',\n",
       "   u'xbox',\n",
       "   u'one',\n",
       "   u'won',\n",
       "   u\"'\",\n",
       "   u't',\n",
       "   u'even',\n",
       "   u'let',\n",
       "   u'me',\n",
       "   u'sign',\n",
       "   u'onto',\n",
       "   u'my',\n",
       "   u'account',\n",
       "   u'offline',\n",
       "   u'.'],\n",
       "  0.45635416281238544,\n",
       "  0.45183503342005932,\n",
       "  1,\n",
       "  0),\n",
       " ([u'lunch',\n",
       "   u'-',\n",
       "   u'and',\n",
       "   u'-',\n",
       "   u'learn',\n",
       "   u',',\n",
       "   u'inpalm',\n",
       "   u'coast',\n",
       "   u',',\n",
       "   u'this',\n",
       "   u'wednesday',\n",
       "   u',',\n",
       "   u'sept',\n",
       "   u'9th',\n",
       "   u':'],\n",
       "  0.40012812895289346,\n",
       "  0.24506542274970428,\n",
       "  -1,\n",
       "  1),\n",
       " ([u'you', u'coming', u'sat', u'?'],\n",
       "  0.45482807152845417,\n",
       "  0.44294223002083166,\n",
       "  0,\n",
       "  1),\n",
       " ([u'6',\n",
       "   u'years',\n",
       "   u'ago',\n",
       "   u'this',\n",
       "   u'weekend',\n",
       "   u',',\n",
       "   u'i',\n",
       "   u'saw',\n",
       "   u'ac',\n",
       "   u'/',\n",
       "   u'dc',\n",
       "   u'at',\n",
       "   u'united',\n",
       "   u'center',\n",
       "   u'on',\n",
       "   u'a',\n",
       "   u'friday',\n",
       "   u'and',\n",
       "   u'blink',\n",
       "   u'-',\n",
       "   u'182',\n",
       "   u'in',\n",
       "   u'tinley',\n",
       "   u'the',\n",
       "   u'next',\n",
       "   u'day',\n",
       "   u'.'],\n",
       "  0.47073723120208522,\n",
       "  0.3920936097084316,\n",
       "  1,\n",
       "  0),\n",
       " ([u'no',\n",
       "   u'power',\n",
       "   u'at',\n",
       "   u'home',\n",
       "   u',',\n",
       "   u'sat',\n",
       "   u'in',\n",
       "   u'the',\n",
       "   u'dark',\n",
       "   u'listening',\n",
       "   u'to',\n",
       "   u'ac',\n",
       "   u'/',\n",
       "   u'dc',\n",
       "   u'in',\n",
       "   u'the',\n",
       "   u'hope',\n",
       "   u'that',\n",
       "   u'it',\n",
       "   u\"'\",\n",
       "   u'll',\n",
       "   u'make',\n",
       "   u'the',\n",
       "   u'electricity',\n",
       "   u'come',\n",
       "   u'back'],\n",
       "  0.42733512639307208,\n",
       "  0.40655285165404503,\n",
       "  0,\n",
       "  1),\n",
       " ([u'also',\n",
       "   u'have',\n",
       "   u'ac',\n",
       "   u'/',\n",
       "   u'dc',\n",
       "   u'on',\n",
       "   u'the',\n",
       "   u'26th',\n",
       "   u'and',\n",
       "   u'buckcherry',\n",
       "   u'at',\n",
       "   u'starland',\n",
       "   u'on',\n",
       "   u'the',\n",
       "   u'28th',\n",
       "   u'before',\n",
       "   u'the',\n",
       "   u'carney',\n",
       "   u'!'],\n",
       "  0.50565874472703942,\n",
       "  0.37081922215417468,\n",
       "  1,\n",
       "  0),\n",
       " ([u'ac',\n",
       "   u'/',\n",
       "   u'dc',\n",
       "   u'with',\n",
       "   u'vintage',\n",
       "   u'trouble',\n",
       "   u'at',\n",
       "   u'gillette',\n",
       "   u'stadium',\n",
       "   u'on',\n",
       "   u'sat',\n",
       "   u',',\n",
       "   u'8',\n",
       "   u'/',\n",
       "   u'22',\n",
       "   u'!'],\n",
       "  0.46063213528336422,\n",
       "  0.4388376779804441,\n",
       "  1,\n",
       "  0),\n",
       " ([u'i',\n",
       "   u'have',\n",
       "   u'2',\n",
       "   u'extra',\n",
       "   u'tickets',\n",
       "   u'(',\n",
       "   u'on',\n",
       "   u'the',\n",
       "   u'field',\n",
       "   u'!)'],\n",
       "  0.51230547975862284,\n",
       "  0.32091944120854199,\n",
       "  -1,\n",
       "  0),\n",
       " ([u'for',\n",
       "   u'the',\n",
       "   u'ac',\n",
       "   u'/',\n",
       "   u'dc',\n",
       "   u'concert',\n",
       "   u'on',\n",
       "   u'saturday',\n",
       "   u'night',\n",
       "   u'at',\n",
       "   u'gillette',\n",
       "   u'stadium',\n",
       "   u'.'],\n",
       "  0.44518065464039053,\n",
       "  0.40011559573718952,\n",
       "  -1,\n",
       "  0),\n",
       " ([u'i', u\"'\", u'm', u'looking', u'...'],\n",
       "  0.53808234582500647,\n",
       "  0.41883342644346766,\n",
       "  1,\n",
       "  0),\n",
       " ([u'ac',\n",
       "   u'/',\n",
       "   u'dc',\n",
       "   u'will',\n",
       "   u'winrock',\n",
       "   u'1000',\n",
       "   u'with',\n",
       "   u'thunderstruck',\n",
       "   u'!'],\n",
       "  0.50730094389480851,\n",
       "  0.33246634491302396,\n",
       "  0,\n",
       "  1)]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#sampling data 80/20\n",
    "\n",
    "#splitting the training set\n",
    "#we are finding indices to able to know which data are selected, \n",
    "#so we can get the corresponding data in all matrixes such as from train_tweets, train_sparse etc.\n",
    "train_indices = np.random.rand(new_train_sparse.shape[0]) < 0.8\n",
    "# test_indices =  np.random.rand(new_train_sparse.shape[0]) > 0.8\n",
    "test_indices = np.asarray([not x for x in train_indices])\n",
    "\n",
    "#X_train: training data and y_train:label\n",
    "X_train = new_train_sparse[train_indices]\n",
    "y_train = np.asarray(train_labels)[train_indices]\n",
    "\n",
    "#X_test: test data and y_test:label\n",
    "X_test = new_train_sparse[test_indices]\n",
    "y_test = np.asarray(train_labels)[test_indices]\n",
    "\n",
    "# X_test, y_train, y_test = train_test_split(new_train_sparse, train_labels, test_size=0.20, random_state=42)\n",
    "log_reg_classifier.fit(X_train, y_train)\n",
    "\n",
    "#finding predicted probabilities with predict_proba method of the classifier\n",
    "pred_probas = log_reg_classifier.predict_proba(X_test)\n",
    "#finding predicted classes with predict method of the classifier\n",
    "pred_labels = log_reg_classifier.predict(X_test)\n",
    "#actual classes are the labels in our training set\n",
    "act_labels = y_test\n",
    "act_proba = []\n",
    "\n",
    "#finding actual probalities from act_labels list, with the help of the indices defined above\n",
    "for i in range(len(act_labels)):\n",
    "    act_proba.append(pred_probas[i][act_labels[i]+1])\n",
    "\n",
    "pred_probas  = np.max(pred_probas,axis=1)\n",
    "\n",
    "#finding the error between pred_probas and act_probas\n",
    "error = np.asarray(pred_probas) - np.asarray(act_proba)\n",
    "\n",
    "train_tweets_flatten = [val for tweetList in train_tweets for val in tweetList]\n",
    "\n",
    "#printing first 30 results, where the probability of the predicted class and the actual class are fairly close (less than 0.2)\n",
    "error_ind = np.where((error<0.2) & ( error>0))\n",
    "zip(np.asarray(train_tweets_flatten)[error_ind],pred_probas[error_ind],np.asarray(act_proba)[error_ind],act_labels[error_ind],pred_labels[error_ind])[0:30]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#err_analyser = Error_Analyser()#creates an instance from Error_Analyser\n",
    "#print err_analyser.analyse()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# 1.0 * len(error[np.where(error > 0)])/len(error)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>IMPROVEMENT</b>|"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#taking the data from files(train and dev) again in order to apply the improved preprocess \n",
    "impr_train_tweets, impr_train_labels, impr_train_word_dict = pre_processing.preprocess_file_improvement(TRAIN_DATA_FILE_PATH)\n",
    "impr_train_feature_dicts = pre_processing.convert_to_feature_dicts(impr_train_tweets, True, 2, impr_train_word_dict) \n",
    "impr_train_sparse = vectorizer.fit_transform(impr_train_feature_dicts).toarray()\n",
    "\n",
    "impr_dev_tweets, impr_dev_labels, impr_dev_word_dict = pre_processing.preprocess_file_improvement(DEV_DATA_FILE_PATH)\n",
    "impr_dev_feature_dicts = pre_processing.convert_to_feature_dicts(impr_dev_tweets, True, 2, impr_dev_word_dict)\n",
    "impr_dev_sparse = vectorizer.transform(impr_dev_feature_dicts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#in this case we are adding only sentiword_net lexicon to data, because it has the best accuracy\n",
    "impr_new_train_sparse = lex_polarity.get_polarity_vectors_imp(sparse.csr_matrix(impr_train_sparse), impr_train_tweets, cs_lex)\n",
    "impr_new_dev_sparse = lex_polarity.get_polarity_vectors_imp(sparse.csr_matrix(impr_dev_sparse), impr_dev_tweets, cs_lex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.515582285402\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "         -1       0.46      0.14      0.22       360\n",
      "          0       0.45      0.74      0.56       700\n",
      "          1       0.66      0.48      0.56       769\n",
      "\n",
      "avg / total       0.54      0.52      0.49      1829\n",
      "\n"
     ]
    }
   ],
   "source": [
    "log_reg_classifier = LogisticRegression(solver='newton-cg', multi_class='multinomial', C=0.01, random_state=0)\n",
    "#we will retrain the classifier with the new sparse document, which has the new features we defined and added as vectors above\n",
    "log_reg_classifier.fit(impr_new_train_sparse, impr_train_labels)\n",
    "pre_processing.classifying(log_reg_classifier, impr_new_dev_sparse, impr_dev_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>TESTING</b>\n",
    "\n",
    "Includes preprocessing, classifying, one of the lexicons(result of Cosine Similarity) and my improvement (which is on preprocessing)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "test_tweets, test_labels, test_word_dict = pre_processing.preprocess_file_improvement(TEST_DATA_FILE_PATH)\n",
    "test_feature_dicts = pre_processing.convert_to_feature_dicts(test_tweets, True, 2, test_word_dict)\n",
    "test_sparse = vectorizer.transform(test_feature_dicts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "test_sparse = lex_polarity.get_polarity_vectors_imp(sparse.csr_matrix(test_sparse), test_tweets, cs_lex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.520199225235\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "         -1       0.41      0.19      0.26       290\n",
      "          0       0.43      0.73      0.54       625\n",
      "          1       0.71      0.48      0.57       892\n",
      "\n",
      "avg / total       0.56      0.52      0.51      1807\n",
      "\n"
     ]
    }
   ],
   "source": [
    "pre_processing.classifying(log_reg_classifier, test_sparse, test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- Total Exec. time : 440.164999962 seconds ---\n"
     ]
    }
   ],
   "source": [
    "print(\"--- Total Exec. time : %s seconds ---\" % (time.time() - start_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2><b>FINAL DISCUSSION</b></h2>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>I grouped the major steps that I followed during this project:</b>\n",
    "   \n",
    "<b>1.</b> Cleaning data and making it in a processable format for my future classifier\n",
    "   \n",
    "        The preprocessing step.\n",
    "    \n",
    "<b>2.</b> Finding the best Logistic Regression Classifier by tuning the parameters\n",
    "   \n",
    "        This section was challenging, because we have multiple labels(not binary) in our data, therefore for the Logistic Regresssion classifier, we have to run it in multi_class:'multinominal mode'. However, the other options are very limited for multinominal mode. I was only able to determine a parameter which inreases the performance of the classifier, but I was not able to find out parameters for better accuracy. \n",
    "        Also, the other classifier which is DecisionTree, takes longer than Logistic regression. The accuracy, other scores and time comparison of these classifiers is as in the tables below. \n",
    "        According to the precision, recall and f1-score values: Relatively, Decision Tree classifier seems to be having higher recall but lower precision(average precision and recall values are too close) by comparing ratios(precision/recall). Hence, we can say that Decision Tree classifier seems to be creating more incorrect results. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>Training and Classification - {Without improvements}</b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Log regression \n",
    "# Accuracy = 0.481683980317\n",
    "#              precision    recall  f1-score   support\n",
    "\n",
    "#          -1       0.34      0.34      0.34       360\n",
    "#           0       0.44      0.64      0.52       700\n",
    "#           1       0.71      0.40      0.52       769\n",
    "\n",
    "# avg / total       0.53      0.48      0.48      1829\n",
    "\n",
    "# --- LR Classification time : 97.9670000076 seconds ---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Decision tree\n",
    "# Accuracy = 0.428649535265\n",
    "#              precision    recall  f1-score   support\n",
    "\n",
    "#          -1       0.29      0.25      0.27       360\n",
    "#           0       0.42      0.56      0.48       700\n",
    "#           1       0.52      0.40      0.45       769\n",
    "\n",
    "# avg / total       0.44      0.43      0.42      1829\n",
    "\n",
    "# --- DT Classification time : 137.391000032 seconds ---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>Tuning</b>\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Logistic regression tuning results show that the chosen parameters are not effective. \n",
    "#Unfortunately If ipython kernel restarts, the graph will be lost."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score: 0.457627118644\n",
      "Parameters : \n",
      "  C: 0.01\n",
      "  random_state: 0\n",
      "  solver: 'newton-cg'\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjIAAAFpCAYAAACGbcLwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmYJFWd7vHv290sCggoKgIiKouCuwOiMA7KKLjidQUd\nFRfgysVlXEBUhFFnFMFBR9xAdHTUK44rooJruSKiiLiAIN5BFodxRQEVbN77xznZHZ1UdVU1VZVx\nMt/P8+RTlZmRmRH9dp38RcSJc2SbiIiIiBYtG/UKRERERKyrFDIRERHRrBQyERER0awUMhEREdGs\nFDIRERHRrBQyERER0awUMhFzIGlfSRdKukjSEdM8/3eSfi/p3Hp71SjWMyJi0qwY9QpE9J2kZcCJ\nwN7AlcA5kj5l+8KhRb9m+7FLvoIRERMsR2QiZrcbcLHtS23fAHwY2G+a5bS0qxURESlkIma3NXBZ\n5/7l9bFhD5R0nqTPSNp5aVYtImKy5dRSxML4HrCt7eskPQL4JLDjdAtKyrwgERHzZHvao945IhMx\nuyuAbTv3t6mPrWL7GtvX1d8/B6wn6dYzvaHtOd+OPvroeS3f2i3b1/ZtnLdvnLette1bmxQyEbM7\nB9he0p0krQ/sD5zWXUDS7Tu/7wbI9m+XdjUjIiZPTi1FzML2SkmHAZ+nFP+n2L5A0iHlaZ8EPFHS\n84AbgD8BTxndGkdETI4UMhFzYPsMYKehx97V+f1twNsW47P32muvxXjb3sj2tW2ct2+ctw3GZ/s0\n27mniFhYkpy/u4iIuZOE09k3IiIixk0KmYiIiGhWCpmIiIhoVgqZiIiIaFYKmYiIiGhWCpmIiIho\nVgqZiIiIaFYKmYiIiGhWCpmIiIhoVgqZiIiIaFYKmYiIiGhWCpmIiIhoVgqZiBGQtKi3LbfcbiTb\nteWW2y36to379o1q27J92b6+b99MMvt1xBKTZFjsvzsxir9tSSz+tsF4b99otg2yfQv0Kdm+xfhU\nZfbriIiIGEMpZCIiIqJZKWQiIiKiWSlkIiIiolkpZCIiIqJZKWQiIiKiWSlkIiIiolkpZCIiIqJZ\nKWQiIiKiWSlkIiIiolkpZCIiIqJZKWQiIiKiWSlkIiIiolkpZCIiIqJZKWQiIiKiWSlkIiIiolkp\nZCIiIqJZKWQi5kDSvpIulHSRpCPWstyukm6Q9PilXL+IiEmVQiZiFpKWAScC+wC7AAdIutsMy70B\nOHNp1zAiYnKlkImY3W7AxbYvtX0D8GFgv2mWez7wUeB/lnLlIiImWQqZiNltDVzWuX95fWwVSVsB\nj7P9DkBLuG4RERNtxahXIGJMvBno9p2ZpZg5pvP7XvUWEREAU1NTTE1NzWlZ2V7ctYlonKTdgWNs\n71vvvxyw7WM7y/x88CuwBXAtcLDt06Z5P8Ni/92JUfxtS2Lxtw3Ge/tGs22Q7VugT8n2LcanStie\ndgcxR2QiZncOsL2kOwG/BPYHDuguYPsug98lvRf49HRFTERELKwUMhGzsL1S0mHA5yn9yk6xfYGk\nQ8rTPmn4JUu+khEREyqnliKWWE4tLcgnjfH25dTEon1ytm8hPqV3p5Zy1VJEREQ0K4VMRERENCuF\nTERERDQrhUxEREQ0K4VMRERENCuFTERERDQrhUxEREQ0K4VMRERENCuFTERERDQrhUxEREQ0K4VM\nRERENCuFTERERDQrhUxEREQ0K4VMRERENCuFTERERDQrhUxEREQ0K4VMRERENCuFTERERDQrhUxE\nREQ0K4VMRERENCuFTERERDQrhUxEREQ0K4VMRERENCuFTERERDQrhUxEREQ0K4VMRERENCuFTERE\nRDQrhUxEREQ0K4VMRERENCuFTERERDQrhUxEREQ0K4VMxBxI2lfShZIuknTENM8/VtIPJH1f0ncl\nPXQU6xkRMWlke9TrENFrkpYBFwF7A1cC5wD7276ws8wtbV9Xf78n8Anb28/wfobF/rsTo/jblsTi\nbxuM9/aNZtsg27dAn5LtW4xPlbCt6Z7LEZmI2e0GXGz7Uts3AB8G9usuMChiqo2BXy/h+kVETKwU\nMhGz2xq4rHP/8vrYGiQ9TtIFwGeBFyzRukVETLQVo16BiHFh+5PAJyXtCfwHsNPMSx/T+X2veouI\nCICpqSmmpqbmtGz6yETMQtLuwDG29633Xw7Y9rFrec0lwG62fzPNc+kjc/M/aYy3L30sFu2Ts30L\n8SnpIxPRoHOA7SXdSdL6wP7Aad0FJN218/v9AKYrYiIiYmHl1FLELGyvlHQY8HlK8X+K7QskHVKe\n9knAEyQ9A7geuBZ4yujWOCJicuTUUsQSy6mlBfmkMd6+nJpYtE/O9i3Ep+TUUkRERMRCSSETERER\nzUohExEREc1KIRMRERHNSiETERERzUohExEREc1KIRMRERHNSiETERERzUohExEREc1KIRMRERHN\nSiETERERzUohExEREc1KIRMRERHNSiETERERzUohExEREc1KIRMRERHNSiETERERzUohExEREc1K\nIRMRERHNSiETERERzUohExEREc1KIRMRERHNSiETERERzUohExEREc1KIRMRERHNSiETERERzUoh\nExEREc1KIRMRERHNSiETERERzUohExEREc1KIRMRERHNSiETMQeS9pV0oaSLJB0xzfNPlfSDevuG\npHuOYj0jIiZNCpmIWUhaBpwI7APsAhwg6W5Di/0ceLDtewOvA05e2rWMiJhMKWQiZrcbcLHtS23f\nAHwY2K+7gO1v27663v02sPUSr2NExERKIRMxu62Byzr3L2fthcpzgc8t6hpFRAQAK0a9AhHjRNJD\ngGcBe659yWM6v+9VbxERATA1NcXU1NSclpXtxV2biMZJ2h04xva+9f7LAds+dmi5ewEfA/a1fcla\n3s+w2H93YhR/25JY/G2D8d6+0WwbZPsW6FOyfYvxqRK2Nd1zObUUMbtzgO0l3UnS+sD+wGndBSRt\nSylinr62IiYiIhZWTi1FzML2SkmHAZ+nFP+n2L5A0iHlaZ8EHAXcGni7ym7RDbZ3G91aR0RMhpxa\nilhiObW0IJ80xtuXUxOL9snZvoX4lJxaioiIiFgoKWQiIiKiWSlkIiIiolkpZCIiIqJZKWQiIiKi\nWSlkIiIiolkpZCIiIqJZKWQiIiKiWSlkIiIiolkpZCIiIqJZKWQiIiKiWSlkIiIiolkpZCIiIqJZ\nKWQiIiKiWSlkIiIiolkpZCIiIqJZKWQiIiKiWSlkIiIiolkpZCIiIqJZKWQiIiKiWSlkIiIiolkp\nZCIiIqJZKWQiIiKiWSlkIiIiolkpZCIiIqJZKWQiIiKiWSlkIiIiolkpZCIiIqJZKWQiIiKiWSlk\nIiIiolkpZCIiIqJZKWQi5kDSvpIulHSRpCOmeX4nSd+S9GdJLx7FOkZETKIVo16BiL6TtAw4Edgb\nuBI4R9KnbF/YWew3wPOBx41gFSMiJlaOyETMbjfgYtuX2r4B+DCwX3cB27+2/T3gr6NYwYiISZVC\nJmJ2WwOXde5fXh+LiIgRSyETERERzUofmYjZXQFs27m/TX3sZjim8/te9RYREQBTU1NMTU3NaVnZ\nXty1iWicpOXATymdfX8JfAc4wPYF0yx7NHCN7Tet5f0Mi/13J0bxty2Jxd82GO/tG822QbZvgT4l\n27cYnyphW9M9lyMyEbOwvVLSYcDnKadjT7F9gaRDytM+SdLtge8CmwA3SnohsLPta0a35hER4y9H\nZCKWWI7ILMgnjfH2ZY9+0T4527cQn9K7IzLp7BsRERHNSiETERERzUohExEREc1KIRMRERHNSiET\nERERzUohExEREc1KIRMRERHNSiETERERzUohExEREc1KIRMRERHNSiETERERzUohExEREc1KIRMR\nERHNSiETERERzUohExEREc1KIRMRERHNSiETERERzUohExEREc1KIRMRERHNSiETERERzUohExER\nEc1KIRMRERHNSiETERERzUohExEREc1KIRMRERHNSiETERERzUohExEREc1KIRMRERHNSiETERER\nzUohExEREc1KIRMRERHNSiETERERzUohEzEHkvaVdKGkiyQdMcMy/ybpYknnSbrPwn361MK9VS9N\njXoFFtnUqFdgkU2NegUW0dSoV2CRTY16BRZECpmIWUhaBpwI7APsAhwg6W5DyzwCuKvtHYBDgHcu\n3BpMLdxb9dLUqFdgkU2NegUW2dSoV2ARTY16BRbZ1KhXYEGkkImY3W7AxbYvtX0D8GFgv6Fl9gPe\nD2D7bGBTSbdf2tWMiJg8KWQiZrc1cFnn/uX1sbUtc8U0y0RExAJbMeoViJhMmufy/zT/T9B8P2Oh\nrMvnZvvW+ISRbRtk+4a19H8Txn/7biqFTMTsrgC27dzfpj42vMwdZ1kGANv9agUiIhqWU0sRszsH\n2F7SnSStD+wPnDa0zGnAMwAk7Q783vZVS7uaERGTJ0dkImZhe6Wkw4DPU4r/U2xfIOmQ8rRPsv1Z\nSY+U9DPgWuBZo1zniIhJIdujXoeIiIiIdZJTSxEREdGsFDIRERHRrBQyESMm6ZajXofFomKDwe+j\nXp+FluzalvzGQwqZiBGRtL2kLwPHS9qzPrZ8xKu1YCTtC5wF/JOkLWy7TvfQvGTXtuQ3XsZ2wyL6\nql7CDbAx8BbK5d3HwaorpJree5K0Xv31r8ArgPWBY0e3Rgsn2bUt+Y2nXLUUsYTqrNgvBw60/ef6\n2HqU+Zu+Y/tYSXKjf5iSngnczfaRncfuBHwUeIHtsyQtt71yZCu5jpJdu9lB8ms9v7XJEZmIJTA4\nrGv7PGAj4MmD5+pElMcDL5f0GuCFg3Pbregclv8asHv90gDA9qWUL4t3SzoOeERLe77Jrt3sIPnR\neH5zkUImYhHVDnfLbN/YefhYYEdJKwbL2D4LeCrwSGDK9l9GsLrzJmlZXf+VALb/H/BxYOfucrbf\nRDmU/1DgjBb2epNd0WJ2kPwGWs1vPjKyb8Qiqo2GJW0KPAX4k+3/AL4xzeJfpczRdAOUPa2+HwYe\nfElI2hV4HnCS7bd2l+l8mZwP3MH2X+vjvT6Mn+zazQ6SX32u2fzmI0dkIhbY8KFbSS+lzMV0CHCo\npLvVx5dDaXBrw3Id8CY6nQ+XdMXnaLDeg+2UdARwIvBlSufJNZYbNLi2vw18RdI/L/U6z1WyW3O5\nlrKD5De8XGv5rasUMhELpHMo253H7gc8Dng08BDg18BDJd2q21h2XnMKcGPtpNcrtWNkt5G/bf25\nGfB2yrn4HSTdU9J6M1wF8m5gG0kb9GmPMNm1mx0kPxrP7+ZKIROxQFzcWBuT4yRtTjl9+ytghe0/\nUBqTRwN37762NsTrA68CrgF+s8SrPyNJ94dVHSOR9EBJ/wl8rDaWPwSOoXwRvIGyZ/vi+pruF8sd\ngJcCF1IP4fdFsms3O0h+NJ7fzZU+MhELqB7qfTLwQdu/U+lUeCVwN+As25+SdDywr6Sf2v49rGp0\nrpf0H7a/P7IN6FAZ9fTtwG0kPRv4I3AycAfgE5TD8AfY/pCk79q+qH4h7A/cYpq3vA443PaPl2YL\n5ifZtZsdJL/W87s5UshELBCVyzzvTBmn4ocAtr8l6RHAgZI2AzYHfgH8HfDW4ffoS0Na3QG4NeXL\n4d7AJfXxZ9i+UtKvgddK+hjwc0lbUvYGHw0cOvxmtq8Grl6SNZ+nZNdudpD8Ws/v5sqppYh5mOa8\nc9dtgH2AP9RlN6qPn0y5KuIFwL7AfsC1QB/Pxa8apt32JZRG9L+AXYHtgNsBN9Tz7KdSDlMfVa+G\neCZlxNQH2J5a2jWfXbJrNztIfjSe32LKyL4Rc6SbjkkxeHzVpZqS3gZsbPuZ9f5OwIa2f1Ab182A\nI4CdgKcMDm+PmqS7ApfZvr7eXw5sSxks7P62t6uPnwGcAbylXvHxFkqHynvY/mPn/Xp1+Wqyazc7\nSH718WbzW2w5IhMxi8GeYO1MuELS6yQdoHp1Q71CYH1J/wAcCTxQ0qskvRn4LHDX+lYrKZ3yrrC9\nTx8aUpVBtbYDXgM8UtLmkt5PGThsQ9tPAL5cvyQAXknZ832TpNMpc7r8jDKOxar37EtDmuzazQ6S\nH43nt1RyRCZiBtKag0ZJ2g04GLgVcCnwN7YfImkv4I3AD2wfpHLZ512B+wHH2f5t5z02cE9GDpX0\nGOCxdZ0PBe5FWe9PAXcEdgD+jXKlw1nAfrbPl7QD8CDgz8BHgBOAU11GSO2FZNdudpD8aDy/JWc7\nt9xy69wAAcuGHtsDuAg4vvPYl4FnA1sC91vL+y2n7jT04QbsU39uC/yIcg5+a+D9wJc7yx0MvL7+\nfgxl+PbBc8uAv6cMwvU+YKNRb1eyazu75Nd+fqO65dRSxBAXN0raUtJBkrax/U3KZY+bSdq4Lvoa\nytgM19k+tx76XuNvqu5ZrnRtgXrizZIOsf0L4D3Aa21fQRkBdaVWTzr3I2APSRvaPgb4pcpw7wAG\nNgFeZPuZtq9d4m2YVrJrNztIfq3nNzKjrqRyy60PN+CWQ/cPAn4CfIgyauaLKaNpfhXYndWnZT8F\nvH3U6z/HbVxef+5BObcuYEPK3u3TgA2AfwZOrMvdBvgScMdRr3uyG8/skl/7+fXhNvIVyC23Ud4o\nh573Bnat9+9SG5m3Ug9ZA7sA51HOW78AeBdwm/rcFsDtRr0d89jewZfAozuPPRr4LrA+8MDa0H4K\nOJcyiFb39cuWal2T3fhml/zaz69Pt3T2jYk1uExR0hOAZ1EaxosoVwGcBzzH9tfqskdRGt43At8E\n/tGd8Ro0w+WhozZNp8lpZ72tV0v83PYxko6mDJ51qu1fLuHqzlmyW2O5prKD5De0XHP59U0KmZg4\n3QZFZVjvnYBPU64OeLbtP0k6HHiY7YfV5V4J/Mr2SZJ2sv3TUa3/2ki6BWXP9cu2z5ll2e6/w30o\nY1TcF7jaZTbgwYipnq4BHoVkt2rZ5rKD5NdZtsn8+iqdfWPiDDUMHwUeAzyesie0q6QVtt8I3FbS\nv0g6EngicEV9/U9h1pFGR2X9+nP/wQOS/o+kBwwv2P13sH0e8Bjbv+w0pLJ9Y58a0mRXtJgdJL+B\nVvPrqxQyMRG6VzRI2lPS2+vd44H72D4X+C3wUMpQ3wBPpVw9cCfgybY/033PvjQy3UbdZU6VrwF/\nkrSDpA2AH1DOua/V8F5kj7Yv2c2ir9lB8qPx/FqQU0sxMSTdzvb/SHoS8F7gtZSrI+5NuTpifeDl\nwAXAnpQBtaY6r+/1oV5JDwJeTRko60O2/zziVVowya5tyS8WU47IxFhSZwK2en9T4ExJD6aMlvk+\n4HLgRZSZZje2/RPKMOabA1+brkNhXxrS7vZJWk/Sc4GjgXfafs90DamkrbV6Mr3eSnbtZgfJr/X8\nWpRCJsaSV08k9yxJ+1Jmin018EJK43l34HTgW8CBwHPr674KHGH72KH368VVEYND2V49x8xmtm+g\nzPp7O8rVHdN9mWxMGQn1Xku8yvOW7NrNDpJf6/m1aMWoVyBiMajMKPs+4L+A7wAvAR5JGQ58R2BT\n4AmUSzq3BDaQtJ7tG2y702j1ZS9wjb1SSc+hbNPnJd1o+8WS7g/cWdKtbP+hdhg0gO1rJP2LG5hQ\nLtm1mx0kv9bza1H6yETzNM209ZIeRWkLPyvpA8B2lMbzT5QJ5f4TOBN4JmVk0T8u7VqvO0l7AM8B\nXgbcA/gKZcCwHYGHA18c7hzZee20Y1mMSrJrNztIfjSe37jIqaVolqQVsOpQ73JJT5Z0h/r0rsAJ\nks4CLrK9p+2rgJX1/PsbKBOxrRw0pMOHhEetdnBExfqSTqj9De5GuRriaOD1wAG2LwG+CPwOeHjn\n32ENfWlIk1272UHyo/H8xk2OyERzJO1j+8zO/YcDRwLrAX+kXP1wI+VqiENsf6MudzBwme3PLf1a\nz52k3Wx/Z5rHvwKcAFxD2aM91PbJ9bl7Uc7Vbwjcqfvv0yfJrt3sIPnReH7jKkdkokUnSDoEQNLT\ngHcCR9reE/gxsB+lMX0v8CZJ/yDpTOAplMs7Vxmcj+8LSfsAb5G0jaQHSnpJZx0/Bmxj+8uU7biF\npK20+pLWv7N9Yc8b0mTXbnaQ/FrPbyylkIlmdA4/H0Q5Rw3wfcpVEXev9z8AbAPsYvt44DhgZ+B9\ntve2/V/d9+zD4d56+HrQYJ4HfB04gHKJ6sOAV0naAljJ6u18NmWwsJMoV368wPb7lnK95yPZtZsd\nJD8az2/suQczV+aW21xvrD4d+pj6cwNK57uPABvUx54PvAe49zSvXz7qbZhtfYAHAZ+jnI/fAjgW\n+CClc+G5wCadZbfq/tsM/n36eEt27WaX/NrPb5xvOSITvTbT4Wfbn64//0LpaPd7ysy5UA4Dfwv4\n2fD7uGeXQHr1mBtHSHqupL+x/S3gq8DLbP/a9hGUPggvAjaijMUxeP2V9fXLXS39Vkwv2bWbHSS/\n1vObJClkolck3aI2LLvCTQ8/d+93GtrLKQNsPUXSVravtP1u29dO97pRkrRssN71qPbWkj5NGap9\nE+Djku5B2Z6NJA0moHs+5QviIuoEel19+JJIdu1mB8mPxvObZClkom/mPYNsbUi+CbxwsJdUX9e3\nzoTLXQfWkrRlXf/NKJelPtX2CZSOg8+mDCb2OeAZkm5p+yqX4c8f0/2S6Jlk1252kPxaz29ipZCJ\nkes2el7HGWRt/8ZDl032YU9QZQyKbev6rJR0K0lvAT4laXPgjsBvJG1VX/Jm4EnA7SmXeV4CbD/0\nnr0ZcyPZtZsdJD8azy+KFDIxcoNGT9KDJJ1BuTrgdbYvtv0X299wmdOkKZJuCRwCHCxphaT1gfcD\nvwYeaft3wDmUYdofUPcafwd8g3IJ61WUPd3zu+/bp0PZya7d7CD50Xh+UWSupRgJdYY2l7QeZbjy\nJwHvsP3JGV6zNfD7Vg7v2r5O0o8pl6Q+BDifMs/M6cCekm4D/AR4F/C/gMdL2o7S7+DK+iVjqV9D\nmye7drOD5Efj+cVNpZCJJTVoGOqh3vUpc638XtJNZpDt7v1o9QyyXwTOGsW6r6NzKB0Jf277Kknf\nBY6iNKK3B46gNKQvpQwm9gvbX+y+QV8a0mTXbnaQ/FrPL2aWKQpiSajOINu5v2oGWWAwg+yxwK+A\nkzw0g2x9zU0mqGuNpBW2/9q5/0ngBNtfHVquN9ua7IoWs4PkN9BqfjG79JGJJTHUkO4B7AH8LfAJ\n4EWS7gpMAVvXx6e7/HNlt3NiH9RD1HNm+6+10+Hhks6lHMr+duf9ejfmxrhmB/O7uqbF7GB885O0\n2XyWbzW/mF0KmVg0GuMZZCU9WNLZwLslPacefp/rF+O1lPP1z7V9mMvAYkCvtm+cs9tL0umSdrZt\nze9KlN5nB2Of34MlTQHvlHRQPU02V03kF/OTU0ux4DTmM8iqXJb6Xsoe7c+BFwM/Bf7V9jXDh7CH\nXjt8mH8wtPmN0y2/1CYgu32BNwC/Aa62/fih52fs3Nn37GAi8jsYOAx4BXA18E/AMba/1jmi0mx+\nsW5yRCYWlCZjBtmNgd2AM2x/D3gHcGvKDL8MihiVK0KGx+pYoyGtnS970ZBOSHZnU74IHwFsKenx\nsHp8kMGXYGvZwcTk9z3g6bZPt/114GJKltQ8ms0v1l0KmbjZ6uHriZhBtu7V/Qb4EvCs+vB36+2+\n9UtkY0kvBvaFmfcQ+3Aoe5KyA3AZK+Sbtq8H3k6dybn2AVkmaQtJ/0gD2cHk5Ue54uh8SYMrbi+l\nDFwHlD5rLeUXCyOFTNwsGpowzfZVwCeBh1ImWfsHyhGMt1Aa2T0kbWL7u7ZfAhxs+1G2vznUKPeS\n7Rvr3vsXgXtJupPtPwMXsnqI95XAbSnb3bvh2gcmLbuBzpfYqcBVko6sj98I/JGS3SbQ3+xgMvOz\n/ae6vYMjKbtS+vYMXEMj+cXCSSETN4vHcAZZSbeUtGP9fcXQc6rbfB5l9M+DAWx/lzL53O1s/4ky\nX8su9bmRb9N0Ji27YS4j1r4eeJykXSQ9EzBl5uad6zIj36aZTHJ+dYdiU8p8SZ+UtJmkJ7h03r2E\nBvKLhZNCJuZFYz6DbG08n0wZOKvb32XHwSL18Yspe/QPr30RDgf+TOmAiO13Aa9e2rVfu0nPbrq9\nc9tnU7b9fGBH29fbfg89yw6S3zT53Y5yNOYYymXUd6mvO4Ue5heLJ4VMzJnGeAbZQSNZG89vAVcP\nvggkPYYyAugaHQZd5mF5LmWE7L+hXNJ5yeD96l5jLw5tJ7ub7p1L2kTSxyhXnm1v+5X18WV9yg6S\nX31++OjKTsDjKKeSHmH7uMH79S2/WFy5/DrWSmWMhi1t/6LevxXwWmB3Soe6B1DmaTnK9pUqs8qe\nD+xFGbPhlcDJ7ky+pp6NnDm8PnXPcHfgKtsXS9rQpR/MXN5rjUs8RynZzZ6dpLvY/vngvSgj3fai\nUUx+a89P0jbAHW2fVe8vo168tBTrHv2RIzIxI03IDLIuV6xsJOl4lXEq7uwy6+/F9fmbNKTT7e31\nrIhJdsyeXbeIsb2yL1+CyW+t+Q0ulb+8W8QMjlgt6QZEL6SQiRnZvg74MaWz4EOAzVlzBtlnAztQ\nZpB9LPDvkr5eX36li94d4pV066H7uwGfBa6kXHl0hqQtp3ndg2pfmGk7EfaliIFkN83rZsuub1/w\nyW/N5br53SSrPv3txdLLqaVYK0mbUC7n/JHtSyQdB9yV1TPIPphyDvsqZphBtk9U5pX5O+DTlC+H\nqylXONyG8sXxL5QOvQfa/v3Qa3cArrd96ZKu9DpKdmu8tqnsIPkNvba5/GLppJCJeVGjM8jWDoCW\ndFvgnykT511FGRDsQEqnwT8A77B96mD57muH32upt+HmSnbtZgfJb7r7EZBTSxNPYz57s4aGnrf9\nK8rVDn8GXls7Up4J3BJ4lu1T60tPkfSU7msH+tSQzufUQbLrXXZjPXvzuOcX/bHWAaNifEl6MHAc\ncKWk04FTXSY8nMseT3cG2XO7T/StofHqQcMOpBy2PhN4OGVP8FGSzrN9tqTTgNdLug64L/Ad4IyR\nrPQsJO0LBrhNAAASaElEQVQFvBQ43PZP5rkHnuxGrP7tvQb4b0lfAt7nMmXCXCS/iCE5tTSBNN6z\nN29MOZd+fb1/F+B9lM6En6OM+/IEytUehwJfsv0RSXcElgN/D3zP9vfr63t1KFtjPHvzuGcHoDGe\nvXkS8ot+SiEzgerppLOB+9r+o6Q9gScCP3QZFXOw3Hq2b5ipQelbQ1MP1T+EMtrnWZQRT2+kzDfz\nDcqcM4+jNKDPkHQQsA+wIXAu8E+dvchefUkMqIwVsgtlr3UKON72x4ePyiS7/mUHIOn+wF9t/6De\nfxfwW9tHDi2X/HqYX/RT+shMGI3Z7M2w+lw85bD7ZpRG8yfAFvXnhZTt/S1wf2B3SXvYPply6efZ\ntl/dbUhd9K4h9fjN3jwx2VVjNXvzBOYXPZRCZsJ4jGZvhlUN3+BIxArKuBu3Bj5k+7Mu43HcGTjf\n9lGUBvdqShGAy9Dtr63vtaw+1osviZl01q/12ZsnMbuxmb15EvOLfkohM4Y0IbM3Q1k3SbtKOhN4\nI/Bu4DnA5pIeVBe7BXB3SYcBHwbeQz0aNfiiqP8uI98LXFt2w9z+7M1jlR3MPT+PwezN45hftClX\nLY0ZrZ5Bdm/g6e7MIGv7IsrVA3aZx+RUyqWOv6V0tltj9ubBXlKfSdoZOBE4FvhEbVx/DDwIeDTw\nLdtfrn1L9gS+YPsd9bWr+hn04Qtjtuym6xfhctXHYPbmN9RTTu9JdktvHfLrzt78JOAUKLM3J7+I\nuev9H0vMzWDvxmM6e/Na3Bv4IfALyiWdL6T8v/4KsIWkT0n6IHCe7X+0/WboV2fJeWTX7OzNM2g+\nO1j3/Gh/9uaxyC/al6uWxoDGdPZmmL3RU5lcb3Cl1fmUcSp+DRwEbEc5jH2i6yR0fWtEb2526vfs\nzWOdHdy8/NTz2ZsnIb8YDylkxoSkjShjUlwEfGXQeKxl+Zs0Kj0sYta6Pp0jDxvUvgVIujPlyoln\nulzhM6f3GqUFyq5vw9JPRHawTvndJKu+beMk5Rfty6mlBmnMZ2/W6isYbpR0C0lH13XfbPj5+vMv\nknaUdCLwGeDjfW1IFzG7XhQx45wdLFh+vZ29edzzi/GUzr6NUZ1BVlJ3BtmNgLeyegbZH1I67g77\nFeWS3V4aNHqDhk/SHShDuW9Fme33Gsqw/NM1jOtTDmvvbvsP3Sf60pAmu3azg+RH4/nF+MqppUYM\nTidoAmaQlfRQ4HmUL4of2n6LpPsArwT+3fZnunt6w3t9WssUC6OQ7NrNDpJf6/nF+MuppZ7TmM8g\nK62+MkPSRpKOAf438H+BuwL3kLQeZS6oLwBPq30MblQZyXZ5p1G9Fay6emTkkl272UHyo/H8YnKk\nkOk5d2aQlfQsSVtRrg44iXLJ4+a2zwYGM8i+R9L3gRvo8QyynXPt3YZ9G8qlqpfZ/jhwJLAtcE+X\nQfo+T/nSOKi+9kavHpr/dcDxKmOq9EKyazc7SH40nl9MjhQyPaMyz9H6nft3kfR14BGUUVs/QhkN\n9NuUYcwfVhd9K/ByyjgWz7Z9iO2ru3tdfVEPrw/25A6S9FqVUU1/CpwA3FvSpra/Telz8AyVK0N+\nQTlv/6HOez2DMindz4H/bfuPS709nXVJdo1mV9cn+TWcX0ww27n15EZpJP8XsBewAbAbZaC6vSgd\ns98GXAG8vy5/EPBR4HRKI7O8814Clo16m4bW5wzg4fX+RpRhzb8APIUyLPszKINsnQC8si63NfAD\n4IFD73dLyoiobwY27sH2JbtGs0t+7eeX22TfRr4CuZlBI0iZdO1ZtfG4BHhkbTS2BL4KvLb+fhGw\nR33Ns4Gjht5Po96mofXZoP48jNKBEMqUCJ8Bdqn3HwX8W/0C2ZvS9+Be9bm7zfC+m/Rg25Jdo9kl\nv/bzyy032zm1NGr1UO9YzyDrOmAWcA5wB0kH1m3+b2C7+m/wGUpHyrtTDld/l3KOHtsXwpqdE+vj\noz4VkewazQ6SH43nFzGQQmbE7PGbQXa40ZN0T0k/BZ4IfAl4XV3mUuA+lAYU4EfAdbXxPcb26d33\n6eGXRLJrNDtIfjSeX8RAxpEZMZUZZN/LmjPIbk05B7++7VfU5Z5AmUH2Uvd48jVNM5KnpP2APW2/\nrN7/LOUQ/uuAo4D7AddTJs97gu3LO6/t3TYOJLt2s4PkR+P5RQykkBkxSQdQzku/k3IO/q7Ax4G7\nAE9j9aiar3admbq+rleNjNYcJGtT4BDgNNsXSnoVZXK8Q+rz9wC+TjkPf5mkhwEb2v70qNZ/XSS7\ndrOD5Nd6fhEDKWQW2WyNnsZsBllJe1MG1doU+CXwFUrHwp9ROhderjK53NeAn9jeZ+j1vZn8MNm1\nmx0kPxrPL2KuUsgsoukO9U73vBqcQbaeZ1+jb4Ckg4B3AQ+1PSXpccCTgJcCzwQeDJxFGeL9Q8C3\nbP9syVd+DpJdu9lB8ms9v4j5SGffRaAxn0F2sGdat29bSTsB2D6ZMgvwznXRcymT6T3f9hsoY0/c\nCnin7ffb/tng36Ivkl272UHyo/H8ItZFjsgsoOFGT2vOIHspcI3tw2d47T0oVxb8q4dmkO2L7qFn\nlRFQX0cZg+I84Me2/0XSkynbsE1dbk/gJZQJ5z419H69OVSf7NrNDpJf6/lF3BypyBdQp8PdQyX9\nJ2UQrR/ZfhRlfpY7S3pUXWbVv31thH9k+xjbf5C0YhTrP5POXuygIb0VpeG37V0oV0G8UNLDbX8E\n+Jmk19SXXwD8K2WulsH7qb5fbxrSZNdudpD8Ws8v4ubIEZmbqbtnozInycsoh3c/DDyf0tHuUMqA\nW0+nDHn+dNcJ1ygZrGqk+rRHKGlXSqfAa+v9e1EaxnMpc8vcmjJ0+3qUzoW3p5yX34XSeXJTdwbO\n6tteYLJrNztIfjSeX8RCyRGZddTZUxrnGWT/D6UDISoDhJ0KnG778LoHfG9gPduPp3QqfAxwqO0f\nAbva/uNgDxD6sxeY7NrNDpJf6/lFLLReHUZtRd27WTWDLKXBPM/2xySdAPyD6gyykgYzyP6U1TPI\n/qzzXs+gXDL5bsp4FSPvWNjpb3AocIGkHSmHqX9BHQm0HoK/AbiNpPsBf0uZk+YSANvfqz971YAm\nu3azg+RXl2k2v4hF4R5M+NTCDSZrBllWT6a3F7Ax5ejd3pQrO3auz90ReCFlePP3ArcZ9Xonu/HK\nLvm1n19uuS32LX1k5kB1rAmV+VYOsX1PScuB04DDbf+4diTcB/gAsAlwOPAy2+dLupvr5GtD77uJ\nG5p8TdLmlL4HW9h+QefxrW1fUX/v2+WqyY42s4PkN9BqfhFLIX1k5sBjOIOspA3m+xqX8TVOB+4t\n6SGdx69Q0buGNNkVLWYHyW+g1fwilkIKmWkMN3oasxlka+fGb0vafh1efiFlj/er3QddjLwhTXZr\n1evsIPnNovf5RYxCTi0NmW7PRmMyg2z9AngsqyfE+yTw9s5e77zWt0/bBsmu5ewg+bWeX8SopJCp\nuo2oxmwGWZXh2a+1fYOk7YCrgB0o88q8xPa5Q8tvafu/l3xF11GyW2P5prKD5De0fHP5RYxaCpkh\nGqMZZCVtSJnd9/aU9T/a9lWd598IbAAc5ToYmMqMwJ8GXm/7iy3t+SW7drOD5Nd6fhGjMrF9ZAYd\n5IYeO4hySefbbD8c+ATwMMp4O68HTpJ0FPAO4JWUQavW0JeGtHoi8FfKnCzLgJdJemjn+eOBe1HG\noQDA9nWUxnTXer93DWmyAxrNDpJf1Wx+EX0zkYXMYE/H4z+D7N6UPde/UOaeuQx4XN3zw/b/UMag\neLykAyWdXM/l/w+lQ6JmeuNRSXbtZgfJr/X8InrJPRjMZqlu1IGm6u/rUwbV+jHwQeAV9fEnA5d3\nltuTsne43zTvp1Fv0wzbuaz+fAzwEUofAoAHUAYBe2Jn2R2AG4GfAI+vj/V2kLBk1152ya/9/HLL\nrc+3Pu/RLBiN8QyyklaoTCi3ildf+XEJZQ/vSfX+hcBvKSObImkr4BXAa23v7DJHDbavWYp1n4tk\n1252kPxoPL+IFox1Z1+N+Qyykg6gXIJ6BWUQsA/a/pGk9VyukrgF8ARgX8pcMj+X9ApgM9uHq4yQ\nutz29fX9Vtj+64g2Zw3Jrt3sIPm1nl9ES8b9iMy4zyC7H/Cy+vNq4GSA2pDKq2f9vRR4j6R9KJ0P\nf1SXW2n7+tr5Uj1rSJNdu9lB8ms9v4hmjGUh0+kEeCjwt5rbDLIH09AMsvXQ9A2Uvdc/uXSIvIWk\n59dF1oPSqdD2K4GPAk8F3mf7/d33crV0az+zZAc0mh0kv7pIs/lFNMk96KizGDcmYAZZYAp4duf+\nnpQ9wG7Hyt1neO2yUa9/shu/7JJf+/nllltrt7HuIzNMYzKDrOrAX/Vw9Xttb9V57lTgVNsfl7QH\n8GjbR3ae7/32TSfZtbF9M0l+bWxfRIuaPbWkMZ9BVtIuku5Qf1+v+1xtSGX7TOD7ko7rPP0Hykii\n2P5mtyGtj418+5Jdu9lB8ms9v4hx02QhozGeQbbzJfEk4AOwqgPhppJ27iw6yO5gYCdJJ0j6APA3\nwLVD79mbnJMd0Gh2kPyqZvOLGEdN/ZHVPbf9gB0pez+PGd477F7pMB3b19k+22Vk0V6NninpxcD/\nlfQo28cAW0naqz79aOBBg2XrnuGyelj+eZQvhwuB3Wxf0n3fnnxJJLuqtewg+dF4fhHjrIk+Mhrz\nGWQl3RP4d8oIn2+kdAb8gaQHAJfOd1vUr8nzkt383q832UHyaz2/iEnQ60JGEzKDrKQXAFvafsUc\nl5+xsezL9ia7GZfvfXaQ/NayfBP5RUySvp9aGssZZFWGNt9H0h1UxtS4P/D1+tx69ee0h94lHQi8\nYLrnoFfbm+xu+toDaSM7SH7TvfZA2skvYmL0vZAZ1xlkb0s5t34/lxE9b0Nt/IGVsLpRHGxrp9Pg\n6dQRU3su2bWbHSS/1vOLmBi9LGQ6DcfHgftL2tD2ZcC3KbPFPrKz+FnAsyhXQ3yuNkKn2f5Kn/aQ\nJN1X0tcl3dv2Lyl7rg+TtBFlVtyHS7pt7Qi5fn3N1pQvlG6nwd/Yvq6vXxTJrt3sIPm1nl/EJBp5\nIaPJmUH274Ff2/5Bvf8+4CSXSfW+Q9m2twK4zMGyHXACZRj3VTkNviD68EWR7NrNDpJf6/lFRDHS\nQkZlBtnzgTdJer2ke9THB4NQ/T/KnuA+ku5i+2rKeft71OevAg6xfXR93Yol3YA56Oy9nQ3cStId\n6/0bbf8EoHaWfCFwa0mfkPQfwBeA79n+d/fwEs5k1252kPxazy8iVhv1EZmxn0G2s/f2E8re36H1\n8VUNpMqVENcCTwGOBM4EHmj72Pp8Hw9lJzuazQ6SH9B0fhFRjayQ0eTNIPs74LPADpK26T7hejmn\n7d/ZvtD2B2z/WtLy+iXRq21Ldqu1lh0kv+4TLeYXEWsaWSFj+0rKDLgP6zQWhwEvrXtJ1wNI2r0u\nf6LtZ9o+qT4+6qNJ81IbzA2ofQxmUxvRlX1sSJPd2vU5O0h+s+l7fhGxppE0SJKW119fD7xu8Ljt\nb1DOy+9Xl9tj8Hvntcvqsk2du67bvD+wHDhO0lPr49Nm0NdGNNm1mx0kPxrPLyJuatFG9pW0C/Bb\n27+UtJ7tG4ael21L+gxlvIqX1cdPBt5q+/xFWbEekPR0YH/bj5J0a9u/HfU6dSW7mfU9O0h+a9NC\nfhExPwt+REaZQXYuvgBcJulnwOv6sn3Jbk56mR0kvznqbX4RsW4W9IiMygyyewKn2P6MpAuA59me\nkvQ04Ba2391ZfpnLIFRbU0bXvAdw7PAe5DiqXzr3s33WqNcFkt189C07SH7z0cf8ImLdLUgho8wg\nOy99uhoi2c1Pn7KD5DdffcsvIm6+hSpkMoNso5Jd25JfREy6dTo/rMwg26xk17bkFxGxpnXt6JYZ\nZNuV7NqW/CIiOuZcyCgzyDYr2bUt+UVEzGw+R2Qyg2y7kl3bkl9ExAxmLWQ6e2+ZQbYxya5tyS8i\nYnZzvmpJ0hbASwBsHzn03HLbKyVtDtyeMrDWGbZ/XZ/P1RAjlOzalvwiImY2n0JmOfAgyt7fi2xf\nPsfX3JiGdLSSXduSX0TEzFbMdcG61zfvGWTXec1iwSS7tiW/iIiZzeeqpcwg26hk17bkFxExs3Ua\n2VeZQbZZya5tyS8iYk3rOiBeZpBtV7JrW/KLiOhY57mWlBlkm5Xs2pb8IiJWW9dTS7mks1HJrm3J\nLyJiTQsy+3VERETEKOT8ekRERDQrhUxEREQ0K4VMRERENCuFTERERDQrhUxEREQ06/8DwjVa9shM\nl0oAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xa402128>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Decision Tree tuning results show the best score and the best parameters\n",
    "#Unfortunately If ipython kernel restarts, the graph will be lost."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score: 0.453253143794\n",
      "Parameters : \n",
      "  max_depth: 13\n",
      "  max_features: 'auto'\n",
      "  random_state: 28\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmQAAAGECAYAAACYvTyjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XvcZWP9//HX+76HGYzzsZzPp1AOUyJfUg4ldBJKJRUK\nlZIOdJQUoi+llCQ/pVKOoaJukhKppJxDTvF1ypmZ8fn9cV17Zs12z8w9M3vfa133ej8fj+sxe+29\n1pr3uLa9rrXWta5LEYGZmZmZ1Weg7gBmZmZmbecGmZmZmVnN3CAzMzMzq5kbZGZmZmY1c4PMzMzM\nrGZukJmZmZnVzA0ysxaRtIOkGyXdLOnQYT7/H0mPSro2l8PqyGlm1jbj6g5gZqND0gBwIrAtcC9w\ntaRzI+LGrlUvj4idRz2gmVmL+QqZWXtMAm6JiDsjYjJwJrDLMOtpdGOZmZkbZGbtsTxwV2X57vxe\nt80l/VXSLyStNzrRzMzazbcszazqz8BKEfGUpB2Bc4C1as5kZjbmuUFm1h73ACtVllfI700TEU9U\nXl8k6ZuSloiIh6vrSfIkuGZmcyEihu0W4luWZu1xNbCGpJUlzQ/sDpxXXUHSspXXkwB1N8Y6ImKW\n5bOf/exs12lCcU7nbHJxzrGVcVZ8hcysJSJiqqQDgF+RTsZOiYgbJO2bPo6TgbdI2h+YDDwNvK2+\nxGZm7eEGmVmLRMTFwNpd73278vobwDdGO5eZWdv5lqWZ9cXWW29dd4QRcc7ecs7ecs7eaXpGze6e\npplZN0nh3w4zszkjiXCnfjMzM7NmcoPMzMzMrGZukJmZmZnVzA0yMzMzs5q5QWZmZmZWMzfIzMzM\nzGrmBpmZmZlZzdwgMzMzM6uZG2RmZmZmNXODzMzMzKxmbpCZmZmZ1cwNMjMzM7OauUFmZmZmVjM3\nyMzMzEbZcsutgqSelOWWW6Xuf471gCKi7gxmVhhJ4d8Os7knCejV/0PC/z+WQRIRoeE+8xUyMzMz\ns5q5QWZmZmZWMzfIzMzMzGrmBpmZmZlZzdwgMzMzM6uZG2RmZmZmNXODzMzMzKxmbpCZ2ZjlwTfN\nrBQeGNbM5lgpA8N68E1rKn8328kDw5qZmZk1mBtkZmY2Ir26Bezbv9ZrY6F7gm9Zmtkc8y3Ldurd\nf0//t/R3s7dK+e/pW5ZmZmZmDeYGmZmZmVnN3CAzMzMzq5kbZGYtImkHSTdKulnSobNYbzNJkyW9\naTTzmVmzjIXO8qVwg8ysJSQNACcC2wPrA3tIWmcm6x0F/HI2+/OPdI/4oGdNdf/9d5I6y897Sfuy\nmXGDzKw9JgG3RMSdETEZOBPYZZj1DgTOAh6Y9e78I90rPuj1jhu3VqpxdQcws1GzPHBXZfluUiNt\nGkkvBnaNiG0kzfCZWQmmN257sa9hRycw6ws3yMys6nig2rdsFkekz1Veb52LmZl1DA0NMTQ0NKJ1\nPTCsWUtIegXwuYjYIS9/AoiI+EplnX91XgJLAU8C74+I87r2FaUMwuicvVPCwLDt+28JzllWzpkN\nDOsrZGbtcTWwhqSVgfuA3YE9qitExGqd15JOBc7vboyZmVnvuUFm1hIRMVXSAcCvSA/0nBIRN0ja\nN30cJ3dvMuohzcxayrcszWyO+ZZlb7UvZwkZwTmds9c8l6WZmZlZg7lBZmZmZlYzN8jMzMzMauYG\nmZmZmVnN3CAzMzMzq5kbZGZmZmY1c4PMzMzMrGZukJmZmZnVzA0yMzMzs5q5QWZmZmZWMzfIzMzM\nzGrmBpmZmZlZzdwgMzMzM6uZG2RmZmZmNXODzMzMzKxmbpCZmZmZ1cwNMjMzM7OauUFmZmZmVjM3\nyMzMzMxq5gaZmZmZWc3cIDMzMzOrmRtkZmZmZjVzg8zMzMysZm6QmZmZmdXMDTIzMzOzmrlBZmZm\nZlYzN8jMzMzMauYGmZmZmVnN3CAzMzMzq5kbZGZmZmY1c4PMzMzMrGZukJm1iKQdJN0o6WZJhw7z\n+c6S/ibpL5KukfTqOnKambWNIqLuDGY2CiQNADcD2wL3AlcDu0fEjZV1FoyIp/LrDYCzI2KNYfYV\n0KvfDtGv3yFJOGfv9C5nCRnBOZ2z1yQRERruM18hM2uPScAtEXFnREwGzgR2qa7QaYxlE4EHRzGf\nmVlruUFm1h7LA3dVlu/O781A0q6SbgAuBA4apWxmZq02ru4AZtYsEXEOcI6kLYHTgbWHX/Nzlddb\n52JmZh1DQ0MMDQ2NaF33ITNrCUmvAD4XETvk5U8AERFfmcU2twGTIuKhrvfdh6yH2pezhIzgnM7Z\na+5DZmaQOvGvIWllSfMDuwPnVVeQtHrl9cYA3Y0xMzPrPd+yNGuJiJgq6QDgV6STsVMi4gZJ+6aP\n42TgzZLeCTwHPAm8rb7EZmbt4VuWZjbHfMuyt9qXs4SM4JzO2Wu+ZWlmZmbWYG6QmZmZmdXMDTIz\nMzOzmrlBZmZmZlYzN8jMzMzMauYGmZmZmVnN3CAzMzMzq5kbZGZmZmY1c4PMzMzMrGZukJmZmZnV\nzA0yMzMzs5q5QWZmZmZWMzfIzMzMzGrmBpmZmZlZzdwgMzMzM6uZG2RmZmZmNXODzMzMzKxmbpCZ\nmZmZ1cwNMjMzM7OauUFmZmZmVjM3yMzMzMxq5gaZmZmZWc3cIDMzMzOrmRtkZmZmZjVzg8zMzMys\nZm6QmZmZmdXMDTIzMzOzmrlBZmZmZlYzN8jMzMzMauYGmZmZmVnN3CAzMzMzq5kbZGYtImkHSTdK\nulnSocN8vqekv+VyhaQN6shpZtY2bpCZtYSkAeBEYHtgfWAPSet0rfYvYKuI2Ag4AvjO6KY0M2sn\nN8jM2mMScEtE3BkRk4EzgV2qK0TEHyPiv3nxj8Dyo5zRzKyV3CAza4/lgbsqy3cz6wbXe4GL+prI\nzMwAGFd3ADNrHknbAHsDW9adxcysDdwgM2uPe4CVKssr5PdmIGlD4GRgh4h4ZOa7+1zl9da5mJlZ\nx9DQEENDQyNaVxHR3zRm1giSBoGbgG2B+4A/AXtExA2VdVYCLgX2iog/zmJfAb367RD9+h2ShHP2\nTu9ylpARnNM5e00SEaHhPvMVMrOWiIipkg4AfkXqP3pKRNwgad/0cZwMHA4sAXxT6RduckRMqi+1\nmVk7+AqZmc0xXyHrrfblLCEjOKdz9tqsrpD5KUszMzOzmrlBZmZmZlYzN8jMzMzMauYGmZmZmVnN\n3CAzMzMzq5kbZGZmZmY1c4PMzMzMrGZukJmZmZnVzA0yMzMzs5q5QWZmZmZWMzfIzMzMzGrmBpmZ\nmZlZzdwgMzMzM6uZG2RmZmZmNXODzMzMzKxmbpCZmZmZ1cwNMjMzM7OauUFmZmZmVjM3yMzMzMxq\n5gaZmZmZWc3cIDMzMzOrmRtkZmZmZjVzg8zMzMysZm6QmZmZmdXMDTIzMzOzmrlBZmZmZlYzN8jM\nzMzMauYGmZmZmVnN3CAzMzMzq5kbZGZmZmY1c4PMzMzMrGZukJm1iKQdJN0o6WZJhw7z+dqSrpT0\njKSD68hoZtZG4+oOYGajQ9IAcCKwLXAvcLWkcyPixspqDwEHArvWENHMrLV8hcysPSYBt0TEnREx\nGTgT2KW6QkQ8GBF/BqbUEdDMrK3cIDNrj+WBuyrLd+f3zMysZm6QmZmZmdXMfcjM2uMeYKXK8gr5\nvbn0ucrrrXMxM7OOoaEhhoaGRrSuIqK/acysESQNAjeROvXfB/wJ2CMibhhm3c8CT0TEsTPZV0Cv\nfjtEv36HJOGcvdO7nCVkBOd0zl6TRERouM98hcysJSJiqqQDgF+RuiucEhE3SNo3fRwnS1oWuAZY\nGHhe0oeA9SLiifqSm5mNfb5CZmZzzFfIeqt9OUvICM7pnL02qytk7tRvZmZmVjM3yMzMzMxq5gaZ\nmZmZWc3cIDMzMzOrmRtkZmZmZjVzg8zMzMysZm6QmZmZmdXMDTIzMzOzmrlBZmZmZlYzN8jMzMzM\nauYGmZmZmVnN3CAzMzMzq5kbZGZmZmY1c4PMzMzMrGZukJmZmZnVzA0yMzMzs5q5QWZmZmZWMzfI\nzMzMzGrmBpmZmZlZzdwgMzMzM6uZG2RmZmZmNXODzMzMzKxmbpCZmZmZ1cwNMjMzM7OauUFmZmZm\nVjM3yMzMzMxq5gaZmZmZWc3cIDMzMzOrmRtkZmZmZjVzg8zMzMysZm6QmZmZmdXMDTIzMzOzmrlB\nZtYiknaQdKOkmyUdOpN1/lfSLZL+Kumlc/+3Dc39pqNqqO4AIzRUd4ARGqo7wAgN1R1ghIbqDjBC\nQ3UHGIGhugPMkhtkZi0haQA4EdgeWB/YQ9I6XevsCKweEWsC+wLfmvu/cWjuNx1VQ3UHGKGhugOM\n0FDdAUZoqO4AIzRUd4ARGqo7wAgM1R1gltwgM2uPScAtEXFnREwGzgR26VpnF+AHABFxFbCopGVH\nN6aZWfu4QWbWHssDd1WW787vzWqde4ZZx8zMekwRUXcGMxsFkt4MbB8R78/L7wAmRcRBlXXOB74c\nEVfm5UuAj0fEtV378g+HmdlciAgN9/640Q5iZrW5B1ipsrxCfq97nRVns85Mf1DMzGzu+JalWXtc\nDawhaWVJ8wO7A+d1rXMe8E4ASa8AHo2I+0c3pplZ+/gKmVlLRMRUSQcAvyKdjJ0SETdI2jd9HCdH\nxIWSXifpVuBJYO86M5uZtYX7kJmZmZnVzLcszczMzGrmBpmZzRNJRXTwLyWn9U4pdV5KTusvN8jM\nbI5Jep+k70haafZr16egnNtKWqruHLNTQs6C6ryUnI2vcygn56y4QWZmI9J1Fr82MAH4NLBuPYmG\nV0rODklfAr4JfCZPXdXIKyal5KSAOs8an7OUOi8l5+y4QWZmsyVpMWDa2WdEfCwi9gIGgc0lLVBb\nuIqCclYPFl8DXgrcBOwoaamICEmD9aSbroSckpaQtEpnucF1XkrOan0eRwPrHMrJOSfcIDOzWZL0\neeAS4CRJ+3QdOH4BbAXsJemdkmobSqegnB8Hjpe0M0BEPBQRT5PGiXueND4cQK1n+CXklHQY8Fvg\nKEmfaHCdl5LzU8Apkt4HEBEPNq3OoZycc8oNMjMblqTxkk4B1gG2Bn4MvAx4RWediDgb+H/AbsD7\ngamSRvV3paCcL5P0Z2A94B/AvpI+WFnlWuBi4E2SngS2G818HSXkzHV+Mum237bAqcBawCKdqyIN\nqfNScm4s6WrS/0NnAbtJendlldrrHMrJObc8MKyZDSsinpX0I+CfEfGEpAuB9wBTIN0yiIipEfFr\nSf8DXB9pYMNRHdywlJykSdqPjYgf5lyPANtIGhcRUyJiCnBxvk14YERcOMr5ismZ6/zIiLgjZxwg\nTfM1ntQv60lJqrvOS8kJLAB8JCKuyDk3JzVuOv+O2us8KyXnXPEVMjMDUv8rScdIelfnvYi4JCLu\nlTQ+Ip4EHgcWzp9NrWw+GXippFWdc4ackypv/YZ09t7xYuCZiJjS1VfraeBRSa/K++nrbZcScnbX\neW7E3JFf70nq33Y/cBTwBUhTT+TNa/tuNjxntc6vjIgrJC0g6TjgUOAASV/N63faCnV/NxuZs1fc\nIDMzJC0PDJFu870tL0/rOJvP9Bcmnd1fPcwujgU2BJZ0TpC0O3AX8AdJ4/PbT0fEw10Hh0cgHZQr\n798EXAe8Jh/Q+3a1pIScw9V5V4ft3wDr5Q7yXwFWk7RhZRe1fTcbmnO4Ou/U6XLANTnDkcDekraI\niOfz53V/NxuXs5fcIDMzIuIe4BOkzrB/BQ7I71evLq0L3BgR/ydpJ+UOtfkAvSLpYDLROQFYBtgS\n+BFwYn5vIGftHBy2B36Xs23UeT8i7gMWB54fhQNJ43POrs4j4j+Vv/9h0tXRO3PeARry3WxKToav\nc+WMt0fEGRHx33xl73vAtCtUDfhuNjFn70SEi4tLiwt5TtvK8hbA+cDL8/J8+c8dSWedpwNXAq+s\nbLMh8Pq25+xkBJbIfy4DPAasm5cH85+LAueQHj74IWnC96XyZ2vmf9fmzjnbOu/kVM5zJvB9Ul+j\nzr+xCd/N2nOOtM4r669FunW9cdd7jfhu1p2zL//2ugO4uLg0qwCLAIcAp3W9fxwwFXh33RkLy3kE\n8Luu95bPGf8BHFR3xlJyzqLO3wfcCny47oyF5Ryuztck9XX7O3BA3RlLyjmvpdMSNbMxTtKaEXHL\nCNYTsD7wUeBnpM7G15Ce/Ho+Ih7N6w3GjLcKW5WzkkExve/KDJ9F/oGVdDfw9oi4TNKLSE/Y7Qsc\nERFPtD3nPNT5c6SroU+R7rI+3q+MJeWsZJibOp8K7AGcGhGP9SNbiTlHRd0tQhcXl/4XYAfgG8A6\nlfdWms02x5EGWfw9sGzl/XFtzkm6zfQdYO1hPuu+ddW5jbo96WD8R+DjXesMtjxnL+u8LxlLydmD\nOj9klOq8iJyjXdyp32wMqzwGfitwN6l/FZI2Ad4jaYmZbPcZYGdgt4jYIiLu73wWaayfVubM+306\nZ/xYzjBO0tGS1ol8dKisO1nSksCewAPA8RHx1Up+RZ+ukDQ9Z5/qvB9XxYrImfc7r3V+dCV/k7+b\no5JztPmWpVlLSHo18FykcXwWjnzbpGsdRURIWjki7qy837dbKyXmlLQscAzwyYi4W9IrI+LKmay7\nGanD8bcr7w3EMLdoWpyz8XVeSs6C6ryInKPJDTKzMSSfSf4C2DEiHlGaVmR+4KKIuGs2275gvJ4+\n9sUpJecrgfER8du8vC5wT0Q8JmnBiHhqDvc3rk9XGBufs6A6LyVn4+u8pJxN4AaZ2RigNCGx8uX9\nn5IeE/8r8Dbgb6TR1j8aEf+qHiDyWeqewMmRRrh3Tma4ynEsqQ/Q+cDnSZ3cHwPeFBFPSpovIiZX\ntntJRFzf73wl5SyozkvJ2fg6Lylnk7gPmVnhJK1JGr9osiSRJijeCtgoIrYEPkIatuAQeEH/lcWB\n34/SgaTxOSUNSFqLNNwDwMnAUqQD7g8iYnPgUeBb+fOplW0FvFczjr7e9pyNr/NSchZU50XkbCJf\nITMrVOUMdALpFsu/ge2ATUmjhb8jIjbJZ/5rkfprHBMRvxnNy/6l5MxZFwHeBCwGLEG6QrIMsBdw\ndESck9f5N7B1RPy1jlsoTc9ZSp2XkjNnbXSdl5aziXyFzKwwSgY6fVUi4hnSLYDXAm+OiPsi4jhg\nnKQ98g/dbcAlwCfzNn3/8Sso54Dyk3SRxjNaljQQ5TbARaRpWW4AlpK0ZF7naNJo6y/ImM/yW5mz\noDovJWfj67yknE3nBplZQXLflYiI5yVtJOn9klYEPkiaJmj1fPYJ8AXgo5ImRMSzwBnAh51zhpwD\nEfF8ztm5xXIRcBZwLml8o+eAC4HNSfNkEhFfAu6QtEz3waNzkG9bzoLqvJScja/zknKWwLcszQoj\naSLweuBg4A5gPuBU4F7gKGC/iLgtr3sFcGFEHOmcM805nnS2viNppP9TgVtIo6xfFRGn5/W+DjwB\nnBQRdzvnsBlLqfNScja+zkvK2XTj6g5gZjOn4cfaOQPYhDRx7l2StgZOBCaRfgT3krQoaRDFnYHR\n6BRdZE5JOwJvAR4B1gHeChxPmhz6RmA9SeuRDthDpBHG76ls368hDRqfs9Q6LyVnE+u8pJwl8i1L\nswbr/PBJ2kLStvntj5L6aIzP6wwBVwFvJj1W/iTwIuC8iHg4Ip7t9O9oc87qgUTSMvntR0kHjoci\nYmpEnAlcDxwE/CBn/AVwOHBJRPywejul3we8Jucsoc5LyVlKnZeSs1S+ZWnWILkvy3IRcXVeXpLU\nIXYCcDPpx+1TwLeBJSPiTXm9/wUujogLVRnXR3rhQJUty7kwaWT1Z/PyWqR+QeNIB+CTSEMaLEaa\nQPsRSa8Avkiau1DAWhHxz8o+e561hJwF1XkpORtf5yXlHBOiARNquri4pAK8B/gXebJc4A3Agfn1\n90idjlchHVyeAr4OfDxvs1nXvgbanBPYmNSXZYu8vBBwds66Hmmwz4NIV0KGgM8CKwCnAF8cZn/9\nmmi5lJyNr/NSchZU50XkHCvFtyzNGkDTHxn/HnA/8N780ZKk/ixXkAZQ3Cki7oj0mP5HgHeTHsff\nJPIVgY7owzxvJeSsZLyWdMDdWNJSwMrAQ/m9U4DLgdMi4j7gh6Sxkw7L+Y/t3m/0aXLtgnI2ts5L\nyVlgnTc655hTd4vQxcUlFdKYPacCFwAPkwZTfDXwW2CPynq7kCbahdRX49359XjnnCHnS4EfARcD\nrwIWJB14rwJWr6y3Kak/7fnAPpX3+3YVp7ScBdV5KTkbX+cl5RwrxVfIzBog99P4AunAsTvpVsBn\n85+/AfaWtKWkH5A6HQ/mTQ8DPgQQuY+Hc4KkA0gH5rNIV0jeQbqtchxwf0TcJmklST8Bdot0JeQM\n4D2SVp7Jk3mtzFlQnZeSs/F1XlLOMaXuFqGLi0sArE16ImmJvDwReBB4WV7+MHAC8LnKNnLOmeY8\nEXhvfr0ecBrpcfwJpD4wZwB/Bz7btd0bnbPYOi8lZ+PrvKScY6l4HDKzUTKbJ4seIY1gvaykRyLi\nCUlXkybgfXlEHN/1yHnfxu4pIeeszr6VBql8CJhP0sSI+Kek+4EDgT9ExBvzk3eKiAfzNuMiYkpE\nnN3SnI2v81JyFlTnReRsE9+yNBslERGSVgVmmKst/zA+QBqr5wvAJEkbAncC90taLG//vLJ+HfBK\nyTmzA0n+7FngP6Qn6V6e374WWJrUOXkwIh6KiAeV5uBT9GlewoJyNr7OS8lZUJ0XkbNNPA6Z2SiR\ntDSpf8ur88Gj877ygUbAIaRHzTcCDo2I85xz2t8/kA+o8wOTga8CP4yIvwyTcTHS8Ac7kQauXBr4\nUkRc7JzDZm5knZeSs5Q6LyVnW/mWpVmPdd9WkbQDcCtpZPA/R8QD1XXyj1/n9sFXJS0OPNY5g+/j\nrZUicnZUzug7B4yVgSW61omc+VFJx5PGRtoAOKNzBt/9725TzlLqvJScHU2u8xJztpUbZGY9ks/O\nVb0VkG+b7AKsQeoAO1WVUcA78lnrQHoZj+RtO30yej3GUJE5Jc0HfBlYS9IlQAB/zZ9N6w9TOVBE\npHGUrs3rDEaa2qXnjZym5yy1zkvJ2cQ6LymnZdGAJwtcXMZSIZ1NfhbYsPLe+sC5wDOkOd2Wrnwm\nKiNYA8u0PSeV8YuAlZj+5NyipCe9vg08D3yQrtG/u3N276+NOUuo81JyllLnpeR0mV7ch8xsHlQv\n3UsaBI4CtgL+H6kz7DURcXz+fC/SOD6rkx7RvyQijqjsayHS/G9LAvtHxFNty9mVeSHgS6SBPa8H\nLo2IU/JnqwKXAdeQRg2/kjyRceXf2dn+mxFxcz8yNjlnKXVeSs6uzI2s81JzWuKnLM3mQeeHK1sM\n+EtEvJw0dcvLgAMlrZQ/34r0Y7cvsAfwtc6GSoMwXgJcQRo1vKcHkqbnzLeaun0CeDQiNgTuA75Q\nybgc8GNgr/znROCpyoFkv5zzD8AtvchYUk5ofp2XkrOUOi8lp81C3ZfoXFxKKqRL+aosv4E06vcm\neXlcXr4UeDHwHeDU/Nn3gO3y64H850TgU6RH9Xs2bUtJObuW1wdWzq8XydnOJ525n0eaNw9gR+DX\nw+xvLdIZ/+Fty1lSnZeSs+l1XlJOlxHUZd0BXFxKKczYJ2Mh0tn7ZaRbLL8iTVoM8E1g0/z6aFI/\njTU6P5LD7HeCczIfaXLivwFXV3LtVTkYb5AzbglMqqwzmP8UsAKVvkVtyVlonZeSs5F1XlJOl5EV\nP2VpNgKdp4tyH5cjSeMc3Q0cHBF/lrQ3afyjC0h9XHaQ9F7Sj9/OEXFrZV8zPDIeEc+0LWfe3/OS\nxgEfIZ2RXxMRe0r6KOlW1FakaVo6g4FuT7odNX9EXFHZz9T8Z+R/a081PWcpdV5Kzry/Rtd5aTlt\nZNyHzGwmJC0r6feQfrAkvZo00e5jpHneNgLWzI+S/wB4QtI7SIMpPgHMDxwWERdU91s9kLQs5ysl\nrV9Z3pJ0O+q1pI7YW+W/91hgQUlvA34H3EsaDHQl4HUR8Zte5ioxZ0F1XkrOxtd5STltLtV9ic7F\npckFOAc4Mr9+G+lsffm8/GnSmD5r5OXXAPcAi3Tto++Pi5eQkzSx86HAwqQDx9PAAfmznUi3qV6T\nl18H3A4smpdXds7y6ryUnAXVeRE5Xeau+AqZ2TDybRWAA4ANJU2IiB+TbqcckD87nfSk0haSFoyI\nS4BdIuIxKc2zp1lM4NuWnJWnv74BbAisGRGXA78mDV8A8BfSwWN7SQtExIWkM/rVACLiTiXOWUCd\nl5KzoDovIqfNo7pbhC4uTSh0PamU33vBWSSpE/LfgdXz8kHAt4DlasxeRM5Krk4H4s1I0+G8OC+/\nijT21Dsa8H1oTE5/N9tX52Mhp8ucFw8Ma60mabuI+NUI1ps2F56kI0kjiO+U+74sGBH/7XPOEc3F\n14CcC0TE0yPNKek7pC5B75e0MOnJr2s6Oes+m68zp7+bPc/p76Y1mm9ZWitJeqmknwEXS3rr7Nbv\nOuB8E3hU0mLA1Ij4b+f2Sp9yXgwcI2m3/N7gzNavMedGkq4C9p1dxk7U/OcngD0kbRARj0fEpdWc\nfWjkbCxpH0nrjnCTUc/p72bPc/q7aUXwFTJrFUkTSANIbgt8nzTFys0R8cPKOjM8Ul8XSbsCRwDH\nAg8CpwArRX7Evwk5JS1KGidqTdL0K+Mj4jUj3LZzhr9hRFzX55zzk/5bvp407tW2wMsj4tGm5PR3\ns7f83bTS+AqZtc2SwM3A5hFxAqlD8YYw/cy5eiDR8NORzPazHlke+E5EnAr8EvhR9e+uO2e+PfJB\n4BFgO1JH7dslLTeS7WP62EejcSBZkTTEwmYR8QHgLlJfHGZ3ZWYUc/q72SP+blqJ3CCzMU/SDpIO\nkLRaRNwTEd+NiOfyxz8DtpM0Xz7THJC0lqQvzm6/fbhlMS1nfmsA2FjSUcA/gVcC50vaLNKAkOvW\nlPMl+aD7OPCViDg0IiaTGhAbA4/n9TpPya2Wb02Nqk7OvDgReAZ4o6Q3kKaT2UzSqp3Gg6Q1Rjun\nv5v+btJpQe48AAAgAElEQVTQ76bVIBrwZIGLSz8KaQqQs4HfAycB55IGRayuswnwXWCVynvzAw+Q\nOh3XkfMCYGvSQW950hWId+d1DwEuz6/Hj3LOdUhjSv0p59wnvz/I9HkFrwT2rGyzGHAh8Mq8/IIn\nBvuc81vA2/P725EGH/0Hacymr5PmRtyINK7TqOX0d9PfzaZ+N13qK75CZmPZS4E7ImKLiNgfuJ40\ncW71VskDpKeSns/vzx/pCsXXgFVqyvk34KUx/erBY8AvACLiaGCZ3IH32dHKKWlt4HjSAXhLcl8X\nSctHum0SSlO4XEw60HVuXT1KmtLlHTl/X/sVzSTnTpJWiPTE4l+AMyKN/H4IaYqe1SNdUbl2tHLi\n72bP+LtpY4UbZDaW/ZI0CnjHraRxe4h0W2UgIu4CbgLent/v3C66gHRWWlfOSTnPPaSrFLtKWlzS\n+3Pe20Y5563ACRFxUv5vdANpMuOncs6IiCnA4sCqeZvO02zHA38YhYzD5fwnMA54Kh+UJ+bXC+fP\nH8qfQ2pAjFZOfzd7x99NGxPcILMxKyImR8QDlbfWJp2FTlsl/xDeDNyb++h0HhW/PiKmzK5TbR9z\nXltZPpzU/+VCYAfg8IjoHGxGJWe+0nBx5a0HSH1dBmGGoQQuIh2gx0Xqu0NEPBwRp/cz3whyzpcP\nytcALwG+LWmIdJD+bQ05/d3sXUZ/N21MGDf7VcyaK19JeD7/6D4/3OX8yq2eF5H6kXQeiX8mIp6V\n9M18tv8Cvbo9MI85L42ISyStFRE35/dnGFZgNG5jxIzjSG0C/F9EPNj12Y2kfjpTujOOlpnkvD9/\ndpGkPwJvAe6OiIv6lcPfzd7mnBV/N20s8BUyK5akbUm3HIiIqdUf2Eo/HCLiuXyWPh64RdJhpIEp\nX5Q/vydv06+BKecl50nAyvnzzgFvsJcHE023fV4eyYnasuSzfUmflPT6nPHfEXF1ft3TA14Pcn5K\n0k4R8UhEfKdzwNPsBwqdm6z+bvYmn7+b1hpukFnJbgXWl7Q5gKRNJR0Dwz5Ovz6wPWm8pFWBD0fE\nHdUV+njGPC85PzRMztlOUzMn8r97KeBnSo/aT8k5t5jFZpsCb5d0BbAScEUvM/Up54rA7zofVG4B\n9vS/Z+bvZg/4u9mX76Y1VTTgUU8Xl5EUQORJdSvvvR/4eH69GPCSmWz7ImAI2Ljy3gsmPm5hzgGm\nDwvwBeBn+fWCpCEXFhlmuwmkx/PPBzYZpXpvdM7C6ryUnI2u85JyupRRPHWSFUfSUqTL/f+MufgC\n57NPRZ/neGtyzk6/ofx6oYh4Mr/eJiJ+O7vtVJmqxTln+HsbW+ddf09jc5ZS56XktHK4QWZFkfQx\n0hn9EPAEcEREPFz9cczrTYqIP0kzdt7NfVz6fhugqTmH+Xs+Q3pK7izgT5H7As3B/lqds+vvaGSd\nl5KzlDovJaeVx33IrJEkTex0kM3L4yRtCKwWEWuROsPuDWwAM/Z3kbQZ8NbuH868Xk9/+ArKuULe\nb2cqlkFJXyfdVvkM8G5gH6WJjqvbLSvpDEmLDLffNuYsqM5Lydn4Oi8pp5XLw15Yk71C0k6k/hYX\nAU8CA5J+Rhqj500RcVn3QSPSk1RXO+cM9pZ0P2l+xJ2B00kHklOBfUkDaZ4R0wcf7WS8X9KREfGY\nc86ghDovJWcpdV5KTiuUr5BZY2jGR7yfBdYjTRfy74j4eX5/EvC7iHh1RPxW0qakp5WG219fvt8F\n5Zw2mCjwa9LceZcA/wUWAJYhzaF4bUT8T0RcpzR59fi8fedJr3/0I19JOQuq81JyNr7OS8ppY4Mb\nZNYYnUv3SnO7TQa+TZpgd0Je5bek6UaWkrSVpP1IZ6kbzGR/fekg2/ScnYNoRFQH+VwGuAr4Vz4w\nTwH+SOr3cmbe7nDgk6Szfirb9kUpOfPf0eg6LyVnKXVeSk4bW9yp3xpD0huAzwJ/JT2av7ek9YGj\ngBMj4peSVge2Bl5LeuT8MxFxk3MO29n4jcCOwGURcUZ+72Fgl4j4Xc74PmBz0gH7TuCwmMNOyWM1\nZ1fmRtZ5KTlLqfNSctoYFQ0Ye8OlXYV0ZXYSsHpeXoR0hn4RsCFp9O+ngA/mzz9KOgPt/PgNAgt1\n7U8tzjkRWLqyPD/wA+AXpOlZrmf6OFMfA/6SX69Iuu2yPLBpZfueZywlZ0F1XkrOxtd5STldxnbx\nLUurwwCpQ/FXJB0FHEyaaPeNpIPMz4CvA8cqPZl0EjCZ9FRYRJripTPmz0DMeFuhjTk3AE7If88B\nwPPAacBbgc1IB+udJW0aEccA4ySdSzrYrBMR90TENZWc/bpsXkLOUuq8lJwl1HlJOW0sq7tF6NKO\nQtcZOPB20hhIQ5X3Fgd+DqyXl29i+qjXE4EFnXOmOf8PuBf4IumMXcDhwGn589OAs/LrFYHdgEWd\ns+g6LyVn4+q8pJwu7Sm+QmajIvIZuKTtJO0PXA4cBjwoacG82rrAM8AESVuROiAvKGkC8HREPKU+\nT7ZbQs7qFQ1J6+an5M4FJkTE4RHxdEQE8GJSp2OA24ANlQb7vCsifhIR/3XOMuq8lJyl1HkpOa1d\n3KnfRoWkBYBTSGeWp0TE9yUtRpr77fGI+HRe7zOkDrKrA3tFxFXOOWzOFYCjSX2F9omIGyRdANwR\nEQdIGkca9PONQJAO0sdHxAwTGEeffwBKyFlQnZeSs/F1XlJOaw83yGxUSFoH2B/4REQ8LWlh4DlS\nf5dPAUcASwP/Bh6MiH9Xth21qUUKynkGcGNEfLHy3oqkCYtXiIjHJG0ALAVsFBHHj0auEnMWVOel\n5Gx8nZeU09rDI/XbaFkQWBv4haS/A68h3SL4Hmkcn9OBXwEfi4gnIE31EhFTRutAUkrO3En7X6Sx\npPYnTRL9DPC1XH6Zb2F9NyJOIN22mnabZjQylpSTAuq8lJyl1HkpOa1dfIXM5pkkAQOz+9FXmkdv\nYeAGYDXgEODDEXGf0kCWdzvnDKN7z/R/TkmvAN4F/A1YEliJ9Cj+tyTtCtwVEX+u7rPXt1ZKyFlY\nnZeSs9F1XlJOsypfIbN5ln+kpkpaFJhaOTvvnh/vuvz+i4C3AYvl1RQRd+cfUfXrDLSEnNUsszob\nj4g/kjsb5zP500jDHhAR53T21fl39+OAV0LOEuq8lJyl1HkpOc26+SlLmyvqmuNO0j7ArcDXJL23\n8/Yw270c+D5pnJ/XRcS9nR+6/JvXl6laCsg52Nl3Xv4c8GVJ21Y/79pmGUkHAn8BbgTOqX7ep4Nd\n43MWVOel5Gx8nZeU02xmfIXM5lj1rDP/2C0ALEQa0Xp10mCUl0bE7cOcoV4D7BERD+ft+9bZuISc\nmv74fWcOwsWAbYG1SP1WfixprU6OLv8lHcBfG7kDd79uq5SWM79uZJ2XkrO0Om96TrPZcR8ymyuS\nVgP2A15NatjfDuweEc9K+iqwXES8s2ubcRExpbLc9ye/Csq5AnAq6aD8ALB3RDwi6TTS+FH7da0/\nQD6B72QE+jXaelE5C6rzUnI2vs5Lymk2M75labPVfalf0tKkOfMWjohNSSNbP0yaQw/gOGADSa/L\n688HEBFTJE2UdJCk5Xp9ICktp6RBJV8EPk6aO+8wYDypfxCkefO2UeqATF5/MKYPajlR0kKRpsLp\nyy2gJucsrc5LydnkOi8pp9mccIPMZqtyK2BjSctExP8BZ5MGn4TU7+IpYOv8+X2kR/FXy9tPztt/\ngDS6+L8i4j9tyylN6yDcOYh2bo0sBGwPXJD/3h8CL5O0dv43/Az4ct42Kv/O/YFrgTV6lbGknNWM\nTa3zUnKWUuel5DSbK9GA+ZtcmlWALYElK8ubAdeTJtL9PbAVsAwwRLrFAvA/pB+9Nwyzv01ITzN9\nGhjfwpxLdC2/gXQA/jqwBenE6O/ADvnzdYEvAV+obLNK5fVOwCVtzFlQnZeSs/F1XlJOF5d5KbUH\ncGleAX4KnJ5fjwP+lzQFC6Rxj44GtgZ2Bn5f2e5DwNpd+xoHbAcs3cacpAmgjwdWy8uTgKuBHYBD\ngZ/kA/GuwFWV7XbPB5QlSONTkQ8665Eez1+upTkbX+el5CyozovI6eIyr6X2AC7NKOQHPPLrF5Ee\nAX9ZXv4Z8P78ekngKOCdwCBwGWkqF+ecMee4/OfSpH4tb8kHg48Dx+TPJpLO1H+Uly8HPtn5bHb/\n/rbkLKjOS8nZ+DovKaeLS6+K+5AZMMPYPfuT5sR7jnRWD3AlsHDu2/IQMAVYL1I/jPcA3+jsp9PH\nwzmnPQn3KtJceG8BFgf+BEySND7S4J8PkSYuhjRJ9EJ5+87AoANd+w16qIScBdV5KTkbX+cl5TTr\nFTfIWqrzpFHXe5OA95NuD+wGrKE0hcgQqXPxkZI2IfXZuAYgIm6LiMc7P3q9/rErKWdX5gUlfRd4\nN2kQzy1It1DuJk3VckRefRlgfqXhDC6JiMOq+43eD/LZ+Jwl1XkpObsyN67OS8pp1jd1X6JzGf1C\n7k+RXy8ODObXuwDHVz57I3ATMAFYETgG+DnwbuccPmflvQmkTsOr5OXdgO8ALwdWJY0Mfhapo/cO\n1f0Mt7+25CyxzkvJ2dQ6Lymni0s/S+0BXGqs/NS5+G7SmEdbAZsCvyP33cjrPAR8Ob8e17X9qPTF\nKCjnETnrxqSrz6cC21U+HwJOBObPB5sNaqr3xucsqM5Lydn4Oi8pp4tLP4pvWbZAdx8KSVtI+gFp\n8MTXkB4X/zTwV+Ae4AhJa0h6PXAFMJj3MbW6v4jo9a2VUnKqa/mlkr4FrAzMB5wd6TbJvcDGkjpj\nHN0ErA2sHxHPRMTf8/YvmGOvLTkLqvNScja+zkvKaTaaPHXSGKcZ58ybEBHPSFqX1Mn4UxFxkqQl\nSGemt5EGo/wQ6THyQWC/iPinc07LOcNUNZLWJp2xPxERb8zv/Qb4DXACcDjw0rz6Q8DnIuIG5yyq\nzkvJ2fg6Lymn2air+xKdS/8L6Qmlb5M6Gk/K7x0B/CS/FulWyxXAxvm9lbv20fc+GQXlXALYF1gm\nL+8DnAm8JC+vTjpwLJeXdwJ2raHeG5+zoDovJWfj67yknC4uo1l8y3KMGebWynakEa2vBq4Dvi5p\nY+DzwIaSXhsRAfwT+CWwFkBE3Jm3H8zLvX6iqtScB5D6CG0AfDo/QfcT0hAHG0taOCJuAy4mjZ1E\nRFwQEecMt7825Sy4zkvJ2bg6LymnWd18y3KMkvQG0lnmhcDzpLF5jgGWJf3QfYI0MOXHIuIleZsZ\nbiW0PWfXrarNSINQrg98lzSVzamkg/CBpFsqOwGnRcSVuY/MihHxb+d8Qd7G1nkpOUup81JymjVC\n3ZfoXOa9kPpZdKZlWYr0aPi0p5OAVUgHlVeQDjA3MH3+vNOBFbr215cnv0rICSwHbAsslJdXBQ7K\nuVYgPdn1EeAqYA/ga8Cn87r/j3SAno/+DxNQSs7G13kpOQuq8yJyurg0rdQewGUeKm/6Fc6dgL/n\n1xOBu4DjKuu9DLg5vx5Huu1yZOcH0zlnyPo64Ifkx+lJ099cCCyVlxcBflRZPoM0wOemnfecs5w6\nLyVnCXVeWk4Xl6YV34svWERE/vMC4DpJr440XcihwPaVVW/Jn59P6gdzBulJpSeh/30ySsmZM15I\netR+m/zW50iP4k/Jt6MeI91aOUjSPqSDyxHAdRHxYM7Z16lvSshZSp2XkjNnbHSdl5bTrGnch6wQ\nkhTDVFanj4bStCFTOu8BPyb9wH0xv7c4adqRyyPiH9Vt25hzJtk7GVcEJkfEf/L7lwNnRcT/5uX1\nSNPjrEGaFPr6fmdrcs5S6ryUnDPJ3qg6Lz2nWRO5QVYASS8GloyIv8+q03DnrDIiInegPYPUB+aO\nrvUG8mo9rfyCcs4fEc+NYL35ImKypFeS+hRtExEPdO+j+u9pW86C6ryUnI2v85JympXEtyzLsC1p\nzjZmdiDJn007QETE1cC5wLrVdfJVguf79MPX+JydA4Ok8bNbNx9IFBFXAreTRmLvfNY5kAxW/z1t\ny0kBdV5KzlLqvJScZqXxFbKG6j4Dl/Q70uPg3+18HrO5VTKzWzQtzTkhIp7Jry8GTomIn45gu84t\nmEVIT3s92vacBdV5KTkbX+cl5TQrla+QNVDnBywionIW+h7g8s461QOJZtKhOG/ft86xJeRUmnPw\nUuB4SXvlt08E1u7kmdXfXcn/eEQ8OrN/Q4tyNr7OS8lZUJ0XkdOsdL5C1lCSlgc+CTwM/Jz0SP7U\nytnmAOmx+y9HxH9Hcrbfppz5AHEg8D7SiOpPAV8Hto+Ifw2z/sKkPkZ39DtbiTm7MjSyzkvJWUqd\nl5LTbKzwmUoDSFq1a3k10hNed+e3PgPsAulss9OHBViJ9Lg4o3QgKSWnAAHXA2+MiLMiPYp/DbBn\nZb3q9/9DwP6V7fuuhJwF1XkpORtf5yXlNBtL3CCrmaT5gaMlrSZpG0lbkqZmeToijoqIzwAXAJtI\nelFns/zn8cAjkhZwzhmf1MoH1z8Cd0gal1e5G/hbZ/3OgTkv/hjYTmkevX73GSolZ+PrvJScBdV5\nETnNxiI3yGrSObOM9KTRvaT53A4GHgeeBf6jNFYPwF+BrYHJeZvOmfz/Ad+IiKedUwOdg4Ck1SUd\nFxFPRRpXqpPjJcAT1e1yH6GBiLiFdDtmoX6e3ZeQs6Q6LyVn0+u8pJxmY5UbZDXJZ5YT862V/5AO\nDKdHxN+AKaSpW96YV78eeBQY37WP2yPifueclnNRSceTOm9/SNKb8seStAywdET8VtIikrbunPV3\nDs4R8f2I+E8/z+5LyFlYnZeSs9F1XlJOs7HKDbJ6fR44ICKOBN5Bun1CRFwHnAdsLumnpLngfhcR\n9zhnouGf1PoysDhpougjKzmnAqsC/5B0CKkfzCb5zN85h9e4Oi8lZyl1XkpOs9aIBkyoOZYL+UnW\nyvJLgB3z6zVJB43X5eVLgS/m1+sC85MGtFzVOWfINlB53ZmgeAFSH5Y1Kp9dDhyRX+9Nuu1yCrC6\nc5ZT56XkLKHOS8vp4tKm4itkfaQ8AnVleSFgB2A3SUtH6nNxGfAWSROADwDvlnQhcDQwPiIujYjb\nJQ30sc9Q43NKWlbSUjDt1spaks4CTpf0JdLVXgFvq2z2c+ADkhYDriJN27JPRNzmnM2v81JyFlTn\nReQ0a626W4RjrTB9bLfOnwuQHrnfClgUWIV0G+ADlc9vAD6UlycBuzjnDFkXIXUWPiwvD5KmtDkk\n5/wW8CNgOeABYGNgYeBQ4E/A0V37G2hjzlLqvJScJdR5aTldXNpcfIWsh5Se6PoSTHvy6G2ks8qF\ngFcBJ0QaNPFPwEslrR7p6a5bga0lLRcRf4qIc/P+Bluec31JX4yIx0hXQVaUtAmwGOnWyQkRcUdE\n7AdsCSwBfAr4IPAPUsfurwGP5f11Hunv6XhTJeQsqM5Lydn4Oi8pp5m5U3+vjQOWlbRzXn4O2BH4\nPqkfy2vzAeZM0hNh35c0BNwGfDgi/lPdWcxiEuSW5BwEVpC0HXAxcB/whoh4iNQ/aFJl3TOALSPN\nU7g/sAFwdn59b87Zrye/SshZSp2XkrOEOi8pp5nVfYluLBVgArAH8GlSX4z5ge2AvwCvBd4KXAsM\n5vX3BnaqbC/nHDbnp/Lya0kdijcEdiMNZbAKabT1XwOb5/XmJw13cB3wVucsss5LydnYOi8pp4uL\nS3guy16TNDEinqgsHwRMiYhvSnoz6cfwhIg4vGu7UZ3vr8SckhYH3gWsFhEHSfoisDzpLP/0iPhK\nZbulIuJB5xw+Y15ufJ2XkrOpdV5STrO2c4NsDuU+Kv8ZwXqdCYyPIHWUvRPYFDgd+E1EPOycc5Rz\nMNLE0JsB7wSujYhTJS1C6mD8aF6vLwflEnKOwTovJae/m2Y2z9yHbA5I2hPYT9JKlfeWns1mRwO/\nAtYnPal0VkQ8rOEHZXTOmeSM6X2BriN1Nl5W0njgiYh4VNKgNG3C6NblHIt1XkpOfzfNrBd8hWwE\nJI2LiCn56aTdgGsi4qeStiadsZ8SEY+McF/TJu91zrnLqTR58eO9zlVizrbUeSk5/d00s7nlBtks\nDHfpXtJbgP9ExBWSloz0tNJc7885nXOsZnTO9uY0sznnBtkI5NsCuwPHRsRldeeZGefsrRJylpAR\nnLPXSslpZiPnPmRdOv1SlKYFmU/SYcC7gaOqP3yqDDipNCXJnpIWdk7nbHNG52xvTjObN26QZZ0f\nvfxU14SIeD4iJgPrAacCd0vaUtIueb3qgJOrAXeOUt8R52xZzhIyOmd7c5pZb7T+lqUkVTsHS/oY\nsBNpUt2rgAWB04Bf5FV2Bb4QESeNZv8L52xfzhIyOmd7c5pZb42rO0BdpBc+qSXpYNKZ5T7AJ4Dt\ngV2AjTpPLEm6nvzfbbQOys7ZrpwlZHTO9uY0s/5o7S3LyCStLOkbkpYB1gZ+AuxLGvPoqxExBXhU\n0haSfkz6Yfy1czpnmzM6Z3tzmll/tKpB1jkDrSwfDHwFuD0iHgCmkCbTvSMiXhkRl0laB1iCNLbP\nNRGxcUTc6JzO2baMztnenGY2CqIBE2qOdgFWzn+eAtxaeX9X4Exg/bx8IHA+ad636vaDzumcbc3o\nnO3N6eLi0r9Se4C+/wPzgwuV5Z2A64GJwArAv4Ct8meLAR8Afg9cDpwHbDyr/Tmnc47ljM7Z3pwu\nLi6jW2oP0Jd/FMwHLFJZXgR4dWX5QuDg/PpDwK+7tl+c1Gm2s9yvH2bnbFnOEjI6Z3tzuri41Fdq\nD9DzfxCMB7YBds/Lk4CDgZOASfm9VwJ/BVYFJgC/AQ7Mn3WfvfblVoBzti9nCRmds705XVxc6i1j\nplN/5ZHxZ4HngQ9Luh3YGLgI+C/wSqVJda8E7gS+FBHPACcCy+bto7rfmHGwRed0zjGZ0Tnbm9PM\nmqH4BpmSwa4frUFgJeAPEfGtiLgB+AuwIrBlXufXwC6S1o+In0fEYc7pnG3L6JztzWlmzTJmRuqX\ntDTpCaQLgWtJgym+H/hLRJyuNKfb+4CtgQAeBc6MiIsq++j7KNfO2b6cJWR0zvbmNLNmKLJBJr1g\napH9gf2An5GeVJovIj6S398IOCginpO0PLAOsBlwdL8v/Ttn+3KWkNE525vTzJqrqAaZKpPtVt5b\nANgZ+CXph+040q2BPYAbgU8Cq5CecjomIoaq++vH2adzti9nCRmds705zawA0YAnC2ZXgJWZ8ZHx\njUijWW9feW8v4BpgE9Jj45fk95cGjgW269pnP4Y0cM6W5Swho3O2N6eLi0s5pfYAswyXHjo4HPgt\n00eqPgS4AngzcDFwQn7/M0x/rHw/4Bngfc7pnG3N6Jztzeni4lJeGUezLQq8jjSS9URJE4GngLeQ\nzkhXBK7M674YWFzShsDqpNsDo9U51jnbl7OEjM7Z3pxmVpq6W4QzK8BA/vP3wEPAMcAawB+Aq4Ef\nAy+rrL8E8E7gR+Qz1/x+X28DOGf7cpaQ0Tnbm9PFxaXMUnuAWYZL04WcCzwOrJHf+yppLJ/OOmsD\n3wWW79p21H70nLN9OUvI6Jztzeni4lJeacRTlsNduq8+Ri7pI8DbI2JTSfOTbgn8AViANOXISRFx\nwqz255zOOVYzOmd7c5rZ2NGIBtnMVH/ElKYc+UykARXXBJYnjd1zSkQ87JzO6YzO6ZxmVqy6Ls0x\nvT/GfKSOsN8AJgyz3mD+c1fgsZnsa5A+3Q5wzvblLCGjc7Y3p4uLy9gstc1lGRHPS1o8IiYDk0nT\niowfZr2p+c9zgEskbVD9PN9GmBoRfbnU55zty1lCRudsb04zG5tG7Zal0mS7UyvLy5I6x15EepR8\nSkR8XNK4iJgyq22d0znbltE525vTzNphVK6Qdc4Y8+u1JS0UEfeTBku8FdgSeFc+04yubQeA7s61\nfRk/zTnbl7OEjM7Z3pxm1iL9vB9aLcCmwKXA2aQ53parfPYR4E7g28BPST+Gg+Q+HXmdzYBvOKdz\ntjGjc7Y3p4uLSztKf3aaO71WlhcFzgFel5dvB04HxuXlQ4E9gXHA54FtK9suDpwIXAis45zOOdYz\nOmd7c7q4uLS39HZnXU8VkaYSWSK/XhR4DXAtcCRwE/DW/Nm3gcOG2d/bgRuoTNjrnM45VjM6Z3tz\nuri4uPRuRzNeyl8DuBH4NXBZ5f1vAe/Ir78B3AYsRZobbpn8fudBgwFgXWC+nv6DnbN1OUvI6Jzt\nzeni4uIS0YNhLyQJpj0yvr6kg4ENgIMj4rXAoKRP546wU4FlJa2bN78h/2heGBEP5I620dlfRNwQ\n6RH0eeac7ctZQkbnbG9OM7MZzE0rjtSvYgdyJ1jSdCFvAC4ndZC9Fdgvf7YpcAvworzOGaT+Gu/o\nd2vTOduXs4SMztnenC4uLi4zK3O3Ufohq3aI/SZwM7BRXv4IcBywdF7+LnBmfr0EsGBlX4Nzk8E5\nnbPUjM7Z3pwuLi4uMysjvmUp6WWSfidpo4i4Dzgf2E7SfMCXAQGL5dWvII3d8+a8fAjwnKQFgUcj\n4ilJgzB91Otecc725Swho3O2N6eZ2YiMtOVG+gE7u7I8Dli3snwU8PP8en7gXcBPgDVHs4XpnO3L\nWUJG52xvThcXF5eRlNmvMP0Jo61IgyiumJcHuv4UcB2wc15eC3hV174G5jWwczpnSRmds705XVxc\nXOakjHguS0lLAR8FiIhPdn02GBFTJe0HHBgR649op33gnL1VQs4SMuYsztlDpeQ0MxuJORn24hHS\nyNRrSlqh+kHkPhcR8S3gTb2LN1ecs7dKyFlCRnDOXislp5nZbI24QZZ/4MYDC85mvZs64wDVwTl7\nq4ScJWTMf79z9lApOc3MRmJOnrIcBHYnTbB7tKQ98/sv2EeM9D5oHzhnb5WQs4SM4Jy9VkpOM7OR\nGHEfshk2kvYCdo+I10taIiIe7n20eeecvVVCzhIygnP2Wik5zcxmZm6nTvo1cJekW4EjhjsjbQjn\n7FBYq98AAACGSURBVK0ScpaQEZyz10rJaWY2rLm6QgYgaTywcUT8obeRess5e6uEnCVkBOfstVJy\nmpkNZ25vWaqEPhnO2Vsl5CwhIzhnr5WS08xsZub6CpmZmZmZ9Yb7WZiZmZnVzA0yMzMzs5q5QWZm\nZmZWMzfIzMzMzGrmBpmZmZlZzdwgMzMzM6vZ/wfNMgQ6LfP57QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x12e8c8d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>3.</b>Classifying the data and evaluating the accuracy and other scores\n",
    "\n",
    "        After, I had classified the development data with the best classifier so far, I observed that we can increase the accuracy by adding more valuable features to the training, for example if the tweet contains words which are in our lexicon's positive words or negative words, we can leverage from this. \n",
    "        \n",
    "<b>4.</b>Implementing lexicons and evaluating them to find the best lexicon\n",
    "\n",
    "        As specified in the assignment document, I implemented three lexicons and copied one manually annotated lexicon [-naming them as lexicons might be wrong, but I wanted to connect the terms with the specification document-] and created lists of positive and negative words from them. One of the lexicons was relatively very successful of finding positive and negative lists, which was sentiword_net. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# cosine similarity accuracy   : 0.476216511755\n",
    "\n",
    "# ppmi accuracy              : 0.358119190815\n",
    "\n",
    "# sentiword_net accuracy    :  0.835975943138 "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>5.</b> Error analysing and finding patterns in the data, Improvements on preprocessing\n",
    "    \n",
    "        After retraining the classifier, I filtered the errors and analysed them to find some patterns in order to fix and increase the performance, accuracy etc. Most of my strategies were on preprocessing step, because I thought if I create new better features and remove inefficient ones from the model, the weight of the efficient features in the model will increase and this will create a better model. For example, I thought if I remove numbers from the tweets, which are intuitively not effective to polarity, that will result into increase of the weights of the other features. Other examples were removing some of the punctuations, applying max_match algorithm, lemmatization etc. However, after I had tested the model, I observed that those differences made very little improvement (actually some of them made it worse). Since, I quickly ran out of time, I wasn't able to try other things to increase accuracy. However, I will discuss these in the following section. \n",
    "        The graphs below show the classifer's accuracy and other values after applying some improvements."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Logistic regression after adding polarity data\n",
    "# Accuracy = 0.513395297977\n",
    "#              precision    recall  f1-score   support\n",
    "\n",
    "#          -1       0.42      0.28      0.33       360\n",
    "#           0       0.46      0.68      0.55       700\n",
    "#           1       0.66      0.47      0.55       769\n",
    "\n",
    "# avg / total       0.53      0.51      0.51      1829"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Logistic regression after adding only cosine distance polarity data and applying improved preprocessing\n",
    "# Accuracy = 0.515582285402\n",
    "#              precision    recall  f1-score   support\n",
    "\n",
    "#          -1       0.46      0.14      0.22       360\n",
    "#           0       0.45      0.74      0.56       700\n",
    "#           1       0.66      0.48      0.56       769\n",
    "\n",
    "# avg / total       0.54      0.52      0.49      1829"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>6.</b> Final testing\n",
    "\n",
    "            I tested the test data on the best Logistic regression classifier by retraining it with the preprocessing improvements. I added only the cosine similarity lexicon(word2vec)'s feature to the model. In the end, the result was very slightly increased. Final result is as below. More importantly presicion value has increased and that means the improvements on preprocessing resulted into creating more accurate results. However, if we look more carefully to the precision and recall values, those values have changed dramatically. For example, for positive: precision value increased while the recall remained stable. But, for negative: precision value decreased with the equal rate and recall increased dramatically. That may suggest us that the lexicon, that I developed and used in this model as a feauture(cosine similarity), has problems with negative polarity in itself."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Logistic Regression on test set\n",
    "# Accuracy = 0.520199225235\n",
    "#              precision    recall  f1-score   support\n",
    "\n",
    "#          -1       0.41      0.19      0.26       290\n",
    "#           0       0.43      0.73      0.54       625\n",
    "#           1       0.71      0.48      0.57       892\n",
    "\n",
    "# avg / total       0.56      0.52      0.51      1807\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>Consequently,</b> although I achieved a very little improvement on the classifier, it still has many problems. \n",
    "I think the most important strategy to achieve better results is to reduce the randomicity of features as much as possible in this model. Just like I mentioned above, the features which have no relation with polarity should be removed from the model.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The performance of the whole project is very reasonable, also it includes testing time\n",
    "\n",
    "--- Total Exec. time : 440.164999962 seconds ---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}