FireAI_041_SentimentAnalysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "np.random.seed(37) # 使得每次运行得到的随机数都一样"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000\n",
      "1000\n",
      "['pos/cv000_29590.txt', 'pos/cv001_18431.txt', 'pos/cv002_15918.txt', 'pos/cv003_11664.txt', 'pos/cv004_11636.txt']\n",
      "['neg/cv000_29416.txt', 'neg/cv001_19502.txt', 'neg/cv002_17424.txt', 'neg/cv003_12683.txt', 'neg/cv004_12641.txt']\n",
      "['films', 'adapted', 'from', 'comic', 'books', 'have', ...]\n"
     ]
    }
   ],
   "source": [
    "# 1, 准备数据集\n",
    "from nltk.corpus import movie_reviews\n",
    "pos_fileIds=movie_reviews.fileids('pos') # 加载积极文本文件\n",
    "neg_fileIds=movie_reviews.fileids('neg') # 消极文本文件\n",
    "\n",
    "print(len(pos_fileIds)) # 1000\n",
    "print(len(neg_fileIds)) # 1000 \n",
    "\n",
    "print(pos_fileIds[:5])\n",
    "print(neg_fileIds[:5])\n",
    "\n",
    "# 由此可看出，movie_reviews.fileids是加载各种类别文本的文件，\n",
    "# 并返回该文件名组成的list\n",
    "\n",
    "# 如果想要查看某个文本文件的内容，可以使用\n",
    "print(movie_reviews.words(fileids=['pos/cv000_29590.txt']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[({'films': True, 'adapted': True, 'from': True, 'comic': True, 'books': True, 'have': True, 'had': True, 'plenty': True, 'of': True, 'success': True, ',': True, 'whether': True, 'they': True, \"'\": True, 're': True, 'about': True, 'superheroes': True, '(': True, 'batman': True, 'superman': True, 'spawn': True, ')': True, 'or': True, 'geared': True, 'toward': True, 'kids': True, 'casper': True, 'the': True, 'arthouse': True, 'crowd': True, 'ghost': True, 'world': True, 'but': True, 'there': True, 's': True, 'never': True, 'really': True, 'been': True, 'a': True, 'book': True, 'like': True, 'hell': True, 'before': True, '.': True, 'for': True, 'starters': True, 'it': True, 'was': True, 'created': True, 'by': True, 'alan': True, 'moore': True, 'and': True, 'eddie': True, 'campbell': True, 'who': True, 'brought': True, 'medium': True, 'to': True, 'whole': True, 'new': True, 'level': True, 'in': True, 'mid': True, '80s': True, 'with': True, '12': True, '-': True, 'part': True, 'series': True, 'called': True, 'watchmen': True, 'say': True, 'thoroughly': True, 'researched': True, 'subject': True, 'jack': True, 'ripper': True, 'would': True, 'be': True, 'saying': True, 'michael': True, 'jackson': True, 'is': True, 'starting': True, 'look': True, 'little': True, 'odd': True, '\"': True, 'graphic': True, 'novel': True, 'if': True, 'you': True, 'will': True, 'over': True, '500': True, 'pages': True, 'long': True, 'includes': True, 'nearly': True, '30': True, 'more': True, 'that': True, 'consist': True, 'nothing': True, 'footnotes': True, 'other': True, 'words': True, 'don': True, 't': True, 'dismiss': True, 'this': True, 'film': True, 'because': True, 'its': True, 'source': True, 'can': True, 'get': True, 'past': True, 'thing': True, 'might': True, 'find': True, 'another': True, 'stumbling': True, 'block': True, 'directors': True, 'albert': True, 'allen': True, 'hughes': True, 'getting': True, 'brothers': True, 'direct': True, 'seems': True, 'almost': True, 'as': True, 'ludicrous': True, 'casting': True, 'carrot': True, 'top': True, 'well': True, 'anything': True, 'riddle': True, 'me': True, ':': True, 'better': True, 'set': True, 'ghetto': True, 'features': True, 'violent': True, 'street': True, 'crime': True, 'than': True, 'mad': True, 'geniuses': True, 'behind': True, 'menace': True, 'ii': True, 'society': True, '?': True, 'question': True, 'course': True, 'whitechapel': True, '1888': True, 'london': True, 'east': True, 'end': True, 'filthy': True, 'sooty': True, 'place': True, 'where': True, 'whores': True, 'unfortunates': True, 'are': True, 'nervous': True, 'mysterious': True, 'psychopath': True, 'has': True, 'carving': True, 'through': True, 'their': True, 'profession': True, 'surgical': True, 'precision': True, 'when': True, 'first': True, 'stiff': True, 'turns': True, 'up': True, 'copper': True, 'peter': True, 'godley': True, 'robbie': True, 'coltrane': True, 'not': True, 'enough': True, 'calls': True, 'inspector': True, 'frederick': True, 'abberline': True, 'johnny': True, 'depp': True, 'blow': True, 'crack': True, 'case': True, 'widower': True, 'prophetic': True, 'dreams': True, 'he': True, 'unsuccessfully': True, 'tries': True, 'quell': True, 'copious': True, 'amounts': True, 'absinthe': True, 'opium': True, 'upon': True, 'arriving': True, 'befriends': True, 'an': True, 'unfortunate': True, 'named': True, 'mary': True, 'kelly': True, 'heather': True, 'graham': True, 'isn': True, 'so': True, 'proceeds': True, 'investigate': True, 'horribly': True, 'gruesome': True, 'crimes': True, 'even': True, 'police': True, 'surgeon': True, 'stomach': True, 'i': True, 'think': True, 'anyone': True, 'needs': True, 'briefed': True, 'on': True, 'won': True, 'go': True, 'into': True, 'particulars': True, 'here': True, 'unique': True, 'interesting': True, 'theory': True, 'both': True, 'identity': True, 'killer': True, 'reasons': True, 'chooses': True, 'slay': True, 'bother': True, 'cloaking': True, 'screenwriters': True, 'terry': True, 'hayes': True, 'vertical': True, 'limit': True, 'rafael': True, 'yglesias': True, 'les': True, 'mis': True, 'rables': True, 'do': True, 'good': True, 'job': True, 'keeping': True, 'him': True, 'hidden': True, 'viewers': True, 'until': True, 'very': True, 'funny': True, 'watch': True, 'locals': True, 'blindly': True, 'point': True, 'finger': True, 'blame': True, 'at': True, 'jews': True, 'indians': True, 'after': True, 'all': True, 'englishman': True, 'could': True, 'capable': True, 'committing': True, 'such': True, 'ghastly': True, 'acts': True, 'ending': True, 'whistling': True, 'stonecutters': True, 'song': True, 'simpsons': True, 'days': True, 'holds': True, 'back': True, 'electric': True, 'car': True, '/': True, 'made': True, 'steve': True, 'guttenberg': True, 'star': True, 'worry': True, 'll': True, 'make': True, 'sense': True, 'see': True, 'now': True, 'onto': True, 'appearance': True, 'certainly': True, 'dark': True, 'bleak': True, 'surprising': True, 'how': True, 'much': True, 'looks': True, 'tim': True, 'burton': True, 'planet': True, 'apes': True, 'did': True, 'times': True, 'sleepy': True, 'hollow': True, '2': True, 'print': True, 'saw': True, 'wasn': True, 'completely': True, 'finished': True, 'color': True, 'music': True, 'finalized': True, 'no': True, 'comments': True, 'marilyn': True, 'manson': True, 'cinematographer': True, 'deming': True, 'word': True, 'ably': True, 'captures': True, 'dreariness': True, 'victorian': True, 'era': True, 'helped': True, 'flashy': True, 'killing': True, 'scenes': True, 'remind': True, 'crazy': True, 'flashbacks': True, 'twin': True, 'peaks': True, 'though': True, 'violence': True, 'pales': True, 'comparison': True, 'black': True, 'white': True, 'oscar': True, 'winner': True, 'martin': True, 'childs': True, 'shakespeare': True, 'love': True, 'production': True, 'design': True, 'original': True, 'prague': True, 'surroundings': True, 'one': True, 'creepy': True, 'acting': True, 'solid': True, 'dreamy': True, 'turning': True, 'typically': True, 'strong': True, 'performance': True, 'deftly': True, 'handling': True, 'british': True, 'accent': True, 'ians': True, 'holm': True, 'joe': True, 'gould': True, 'secret': True, 'richardson': True, '102': True, 'dalmatians': True, 'log': True, 'great': True, 'supporting': True, 'roles': True, 'big': True, 'surprise': True, 'cringed': True, 'time': True, 'she': True, 'opened': True, 'her': True, 'mouth': True, 'imagining': True, 'attempt': True, 'irish': True, 'actually': True, 'half': True, 'bad': True, 'however': True, '00': True, 'r': True, 'gore': True, 'sexuality': True, 'language': True, 'drug': True, 'content': True}, 'Pos'), ({'every': True, 'now': True, 'and': True, 'then': True, 'a': True, 'movie': True, 'comes': True, 'along': True, 'from': True, 'suspect': True, 'studio': True, ',': True, 'with': True, 'indication': True, 'that': True, 'it': True, 'will': True, 'be': True, 'stinker': True, 'to': True, 'everybody': True, \"'\": True, 's': True, 'surprise': True, '(': True, 'perhaps': True, 'even': True, 'the': True, ')': True, 'film': True, 'becomes': True, 'critical': True, 'darling': True, '.': True, 'mtv': True, 'films': True, '_election': True, 'high': True, 'school': True, 'comedy': True, 'starring': True, 'matthew': True, 'broderick': True, 'reese': True, 'witherspoon': True, 'is': True, 'current': True, 'example': True, 'did': True, 'anybody': True, 'know': True, 'this': True, 'existed': True, 'week': True, 'before': True, 'opened': True, '?': True, 'plot': True, 'deceptively': True, 'simple': True, 'george': True, 'washington': True, 'carver': True, 'having': True, 'student': True, 'elections': True, 'tracy': True, 'flick': True, 'an': True, 'over': True, '-': True, 'achiever': True, 'her': True, 'hand': True, 'raised': True, 'at': True, 'nearly': True, 'question': True, 'way': True, 'mr': True, '\"': True, 'm': True, 'sick': True, 'of': True, 'megalomaniac': True, 'encourages': True, 'paul': True, 'popular': True, 'but': True, 'slow': True, 'jock': True, 'run': True, 'nihilistic': True, 'sister': True, 'jumps': True, 'in': True, 'race': True, 'as': True, 'well': True, 'for': True, 'personal': True, 'reasons': True, 'dark': True, 'side': True, 'such': True, 'sleeper': True, 'success': True, 'because': True, 'expectations': True, 'were': True, 'so': True, 'low': True, 'going': True, 'fact': True, 'was': True, 'quality': True, 'stuff': True, 'made': True, 'reviews': True, 'more': True, 'enthusiastic': True, 'than': True, 'they': True, 'have': True, 'any': True, 'right': True, 'you': True, 'can': True, 't': True, 'help': True, 'baggage': True, 'glowing': True, 'which': True, 'contrast': True, 'negative': True, 'reviewers': True, 'likely': True, 'good': True, 'does': True, 'not': True, 'live': True, 'up': True, 'its': True, 'hype': True, 'what': True, 'makes': True, '_election_': True, 'disappointing': True, 'contains': True, 'significant': True, 'details': True, 'lifted': True, 'directly': True, '_rushmore_': True, 'released': True, 'few': True, 'months': True, 'earlier': True, 'similarities': True, 'are': True, 'staggering': True, ':': True, 'president': True, 'extraordinary': True, 'number': True, 'clubs': True, 'involved': True, 'play': True, 'max': True, 'fischer': True, 'most': True, 'tension': True, 'potential': True, 'relationship': True, 'between': True, 'teacher': True, 'his': True, 'single': True, 'parent': True, 'home': True, 'has': True, 'contributed': True, 'drive': True, 'male': True, 'bumbling': True, 'adult': True, 'pursues': True, 'extramarital': True, 'affair': True, 'gets': True, 'caught': True, 'whole': True, 'life': True, 'ruined': True, 'he': True, 'bee': True, 'sting': True, 'bill': True, 'murray': True, 'several': True, 'stings': True, 'on': True, 'happened': True, 'how': True, 'individual': True, 'screenplay': True, 'novel': True, 'contain': True, 'many': True, 'points': True, 'yet': True, 'both': True, 'probably': True, 'aware': True, 'each': True, 'other': True, 'two': True, 'different': True, 'studios': True, 'genre': True, 'geeks': True, 'revenge': True, 'hadn': True, 'been': True, 'fully': True, 'formed': True, 'strengths': True, 'rely': True, 'upon': True, 'fantastic': True, 'performances': True, 'newcomer': True, 'jessica': True, 'campbell': True, 'anti': True, 'social': True, 'tammy': True, 'here': True, 'playing': True, 'rooney': True, 'role': True, '_ferris': True, 'bueller_': True, 'seems': True, 'fun': True, 'had': True, 'since': True, 'revelation': True, 'early': True, 'year': True, 'teenagers': True, 'little': True, 'clout': True, 'my': True, 'money': True, 'deserves': True, 'oscar': True, 'nomination': True, 'once': True, 'character': True, 'like': True, 'speech': True, 'gymnasium': True, 're': True, 'won': True, 'one': True, 'thing': True, 'bothering': True, 'me': True, 'i': True, 've': True, 'seen': True, 'there': True, 'amount': True, 'sexuality': True, 'suppose': True, 'coming': True, 'should': True, 'expect': True, 'no': True, 'less': True, 'starts': True, 'off': True, 'light': True, 'airy': True, 'sitcom': True, 'screws': True, 'tighten': True, 'tensions': True, 'mount': True, 'alexander': True, 'payne': True, 'decides': True, 'add': True, 'elements': True, 'frankly': True, 'distract': True, 'story': True, 'bad': True, 'enough': True, 'doesn': True, 'determination': True, 'win': True, 'all': True, 'costs': True, 'throw': True, '/': True, 'logical': True, 'reason': True, 'why': True, 'when': True, 'lot': True, 'tonal': True, 'nosedive': True, 'takes': True, 'explicitly': True, 'sex': True, 'driven': True, 'mark': True, 'disappointment': True}, 'Pos'), ({'you': True, \"'\": True, 've': True, 'got': True, 'mail': True, 'works': True, 'alot': True, 'better': True, 'than': True, 'it': True, 'deserves': True, 'to': True, '.': True, 'in': True, 'order': True, 'make': True, 'the': True, 'film': True, 'a': True, 'success': True, ',': True, 'all': True, 'they': True, 'had': True, 'do': True, 'was': True, 'cast': True, 'two': True, 'extremely': True, 'popular': True, 'and': True, 'attractive': True, 'stars': True, 'have': True, 'them': True, 'share': True, 'screen': True, 'for': True, 'about': True, 'hours': True, 'then': True, 'collect': True, 'profits': True, 'no': True, 'real': True, 'acting': True, 'involved': True, 'there': True, 'is': True, 'not': True, 'an': True, 'original': True, 'or': True, 'inventive': True, 'bone': True, 's': True, 'body': True, '(': True, 'basically': True, 'complete': True, 're': True, '-': True, 'shoot': True, 'of': True, 'shop': True, 'around': True, 'corner': True, 'only': True, 'adding': True, 'few': True, 'modern': True, 'twists': True, ')': True, 'essentially': True, 'goes': True, 'against': True, 'defies': True, 'concepts': True, 'good': True, 'contemporary': True, 'filmmaking': True, 'overly': True, 'sentimental': True, 'at': True, 'times': True, 'terribly': True, 'mushy': True, 'mention': True, 'very': True, 'manipulative': True, 'but': True, 'oh': True, 'how': True, 'enjoyable': True, 'that': True, 'manipulation': True, 'must': True, 'be': True, 'something': True, 'other': True, 'casting': True, 'makes': True, 'movie': True, 'work': True, 'as': True, 'well': True, 'does': True, 'because': True, 'i': True, 'absolutely': True, 'hated': True, 'previous': True, 'ryan': True, '/': True, 'hanks': True, 'teaming': True, 'sleepless': True, 'seattle': True, 'couldn': True, 't': True, 'been': True, 'directing': True, 'both': True, 'films': True, 'were': True, 'helmed': True, 'by': True, 'same': True, 'woman': True, 'haven': True, 'quite': True, 'yet': True, 'figured': True, 'out': True, 'what': True, 'liked': True, 'so': True, 'much': True, 'again': True, 'really': True, 'important': True, '?': True, 'if': True, 'like': True, 'why': True, 'even': True, 'question': True, 'storyline': True, 'cliched': True, 'come': True, 'tom': True, 'plays': True, 'joe': True, 'fox': True, 'insanely': True, 'likeable': True, 'owner': True, 'discount': True, 'book': True, 'chain': True, 'meg': True, 'kathleen': True, 'kelley': True, 'more': True, 'proprietor': True, 'family': True, 'run': True, 'children': True, 'called': True, 'nice': True, 'homage': True, 'soon': True, 'become': True, 'bitter': True, 'rivals': True, 'new': True, 'books': True, 'store': True, 'opening': True, 'up': True, 'right': True, 'across': True, 'block': True, 'from': True, 'small': True, 'business': True, 'little': True, 'know': True, 'are': True, 'already': True, 'love': True, 'with': True, 'each': True, 'over': True, 'internet': True, 'neither': True, 'party': True, 'knows': True, 'person': True, 'true': True, 'identity': True, 'rest': True, 'story': True, 'isn': True, 'serve': True, 'mere': True, 'backdrop': True, 'sure': True, 'some': True, 'mildly': True, 'interesting': True, 'subplots': True, 'fail': True, 'comparison': True, 'utter': True, 'cuteness': True, 'main': True, 'relationship': True, 'this': True, 'course': True, 'leads': True, 'predictable': True, 'climax': True, 'foreseeable': True, 'ending': True, 'damn': True, 'cute': True, 'done': True, 'doubt': True, 'any': True, 'entire': True, 'year': True, 'contains': True, 'scene': True, 'evokes': True, 'pure': True, 'joy': True, 'part': True, 'when': True, 'discovers': True, 'her': True, 'online': True, 'filled': True, 'such': True, 'lack': True, 'word': True, 'happiness': True, 'first': True, 'time': True, 'actually': True, 'left': True, 'theater': True, 'smiling': True}, 'Pos')]\n"
     ]
    }
   ],
   "source": [
    "# 2, 处理数据集\n",
    "def extract_features(word_list):\n",
    "    '''专门一个函数来提取特征'''\n",
    "    return dict([(word,True) for word in word_list]) # 此处加True的作用是构成dict,实质意义不大\n",
    "\n",
    "pos_features=[(extract_features(movie_reviews.words(fileids=[f])),'Pos') \n",
    "              for f in pos_fileIds]\n",
    "neg_features=[(extract_features(movie_reviews.words(fileids=[f])),'Neg') \n",
    "              for f in neg_fileIds]\n",
    "print(pos_features[:3]) # 打印下看看内容是否正确\n",
    "\n",
    "dataset=pos_features+neg_features # 将两部分结合起来作为一个dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num of train_set:  1600 \n",
      "Num of test_set:  400\n",
      "Accuracy: 69.25%\n"
     ]
    }
   ],
   "source": [
    "# 构建模型，训练模型\n",
    "from nltk import NaiveBayesClassifier\n",
    "from nltk.classify import accuracy as nltk_accuracy\n",
    "\n",
    "np.random.shuffle(dataset)\n",
    "rows=int(len(dataset)*0.8) # 80%为train set\n",
    "train_set,test_set=dataset[:rows],dataset[rows:]\n",
    "print('Num of train_set: ',len(train_set),\n",
    "      '\\nNum of test_set: ',len(test_set))\n",
    "clf=NaiveBayesClassifier.train(train_set)\n",
    "\n",
    "# 查看该模型在test set上的表现\n",
    "acc=nltk_accuracy(clf,test_set)\n",
    "print('Accuracy: {:.2f}%'.format(acc*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "insulting\n",
      "astounding\n",
      "3000\n",
      "outstanding\n",
      "offbeat\n",
      "palpable\n",
      "wonderfully\n",
      "stupidity\n",
      "hatred\n",
      "slip\n"
     ]
    }
   ],
   "source": [
    "# 查看模型内部信息\n",
    "# 该分类器是分析某段文本中哪些单词与“积极”的关联最大，\n",
    "# 哪些与“消极”的关联最大，进而分析这些关键词的出现来判断某句话是积极或消极\n",
    "\n",
    "# 打印这些关键词\n",
    "for key_word in clf.most_informative_features()[:10]:\n",
    "    print(key_word[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sample: It is an amazing movie, Type: Pos, Probability: 58.08%\n",
      "Sample: This is a dull movie. I would never recommend it to anyone., Type: Neg, Probability: 77.31%\n",
      "Sample: The cinematography is pretty great in this movie, Type: Pos, Probability: 64.29%\n",
      "Sample: The direction was terrible and the story was all over the place, Type: Neg, Probability: 65.96%\n"
     ]
    }
   ],
   "source": [
    "# 用该模型来预测新样本，查看新句子的情感是积极还是消极\n",
    "new_samples = [\n",
    "        \"It is an amazing movie\", \n",
    "        \"This is a dull movie. I would never recommend it to anyone.\",\n",
    "        \"The cinematography is pretty great in this movie\", \n",
    "        \"The direction was terrible and the story was all over the place\" \n",
    "    ]\n",
    "\n",
    "for sample in new_samples:\n",
    "    predict_P=clf.prob_classify(extract_features(sample.split()))\n",
    "    pred_sentiment=predict_P.max()\n",
    "    print('Sample: {}, Type: {}, Probability: {:.2f}%'.format(\n",
    "        sample,pred_sentiment,predict_P.prob(pred_sentiment)*100))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}