HW1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f09253d6",
   "metadata": {},
   "source": [
    "Task 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fc3ead84",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pymystem3 import Mystem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d51d10d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "file = open('Приглашение_на_казнь.txt', 'r', encoding='utf-8')\n",
    "cincinnat = file.read()\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "b0fe8dd7",
   "metadata": {},
   "outputs": [],
   "source": [
    "m = Mystem()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "8f4b45e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "lemmas = m.lemmatize(cincinnat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "1a007c31",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['владимир',\n",
       " ' ',\n",
       " 'набоков',\n",
       " '.',\n",
       " '\\n',\n",
       " 'приглашение',\n",
       " ' ',\n",
       " 'на',\n",
       " ' ',\n",
       " 'казнь',\n",
       " '\\n',\n",
       " '\\n',\n",
       " '\\n',\n",
       " '\\n',\n",
       " 'Comme',\n",
       " ' ',\n",
       " 'un',\n",
       " ' ',\n",
       " 'fou',\n",
       " ' ']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lemmas[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "c6ba2644",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<function TextIOWrapper.close()>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file = open('lemmatized_text.txt', 'w', encoding = 'utf-8')\n",
    "file.write(''.join(lemmas))\n",
    "file.close"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "44477abf",
   "metadata": {},
   "source": [
    "Task 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "dbd61e28",
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6b65aee8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\NewUser\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     C:\\Users\\NewUser\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
      "[nltk_data]       date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('punkt')\n",
    "nltk.download('averaged_perceptron_tagger')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e3a1f359",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokens = nltk.word_tokenize(cincinnat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4da119ec",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Владимир',\n",
       " 'Набоков',\n",
       " '.',\n",
       " 'Приглашение',\n",
       " 'на',\n",
       " 'казнь',\n",
       " 'Comme',\n",
       " 'un',\n",
       " 'fou',\n",
       " 'se',\n",
       " 'croit',\n",
       " 'Dieu',\n",
       " 'nous',\n",
       " 'nous',\n",
       " 'croyons',\n",
       " 'mortels',\n",
       " '.',\n",
       " '/Delalande',\n",
       " '.',\n",
       " 'Discours']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokens[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3b2d28e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pymorphy2 import MorphAnalyzer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "af96287f",
   "metadata": {},
   "outputs": [],
   "source": [
    "morph = MorphAnalyzer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "83cd4884",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "4654010b",
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open('analyzed_text.jsonl', 'w', encoding = 'utf-8')\n",
    "\n",
    "for i in tokens:\n",
    "    dic = {}\n",
    "    a = morph.parse(i)[0]\n",
    "    dic['lemma'] = a.normal_form\n",
    "    dic['word'] = a.word\n",
    "    dic['POS'] = a.tag.POS\n",
    "    json.dump(dic, f, ensure_ascii = False)\n",
    "    f.write('\\n')\n",
    "\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4293b964",
   "metadata": {},
   "source": [
    "Task 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4742618d",
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open('analyzed_text.jsonl', encoding='utf8')\n",
    "data = [json.loads(line) for line in f]\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b9b68912",
   "metadata": {},
   "outputs": [],
   "source": [
    "#getting rid of everything that has NULL part of speech\n",
    "words = list(filter(lambda line: line['POS'] != None, data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "fdbb2c0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#a function to get values corresponding to a certain key from each dictionary in list\n",
    "def test(lsts, key):\n",
    "  return [x.get(key) for x in lsts]\n",
    "\n",
    "#creating a list of parts of speech\n",
    "pos = test(words, 'POS')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "7eb2cec0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('INTJ', 0.002268659726248393),\n",
       " ('PRTS', 0.0031257089561644528),\n",
       " ('COMP', 0.004058380176955458),\n",
       " ('NUMR', 0.004058380176955458),\n",
       " ('PRED', 0.004612941443371732),\n",
       " ('ADJS', 0.008368833656827406),\n",
       " ('PRTF', 0.01716619192861284),\n",
       " ('GRND', 0.021350608757026542),\n",
       " ('INFN', 0.02145143807819314),\n",
       " ('PRCL', 0.0509692218497139),\n",
       " ('NPRO', 0.06896725567795114),\n",
       " ('ADVB', 0.0836127145773991),\n",
       " ('PREP', 0.10047641854251216),\n",
       " ('CONJ', 0.1006528698545537),\n",
       " ('ADJF', 0.1187265256736659),\n",
       " ('VERB', 0.13816137732852712),\n",
       " ('NOUN', 0.25197247359532154)]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#getting unique parts of speech from our list\n",
    "unique_pos = set(pos)\n",
    "\n",
    "#calculating the frequency of each part of speech\n",
    "dic_pos = {}\n",
    "n = len(pos)\n",
    "for p in unique_pos:\n",
    "    dic_pos[p] = pos.count(p)/n\n",
    "\n",
    "#displaying the sorted frequencies (from the rarest part of speech (interjection) to the most frequent one(noun))\n",
    "sorted(dic_pos.items(), key = lambda x: x[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "39eaed73",
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "dd536efa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('быть', 418),\n",
       " ('сказать', 204),\n",
       " ('мочь', 118),\n",
       " ('знать', 107),\n",
       " ('хотеть', 62),\n",
       " ('говорить', 55),\n",
       " ('казаться', 43),\n",
       " ('спросить', 38),\n",
       " ('продолжать', 38),\n",
       " ('видеть', 36),\n",
       " ('сидеть', 32),\n",
       " ('смотреть', 30),\n",
       " ('стоять', 30),\n",
       " ('идти', 29),\n",
       " ('понимать', 28),\n",
       " ('выйти', 26),\n",
       " ('стать', 26),\n",
       " ('проговорить', 26),\n",
       " ('просить', 26),\n",
       " ('начать', 25)]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#creating a list of dictionaries that contains only verbs\n",
    "verbs = list(filter(lambda word: word['POS'] == 'VERB', words))\n",
    "\n",
    "#creating a list of lemmas of the verbs\n",
    "verb_lemmas = test(verbs, 'lemma')\n",
    "\n",
    "#displaying top-20 verbs by frequency\n",
    "Counter(verb_lemmas).most_common(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "84fff139",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('ещё', 159),\n",
       " ('уже', 111),\n",
       " ('только', 107),\n",
       " ('тут', 93),\n",
       " ('опять', 76),\n",
       " ('теперь', 76),\n",
       " ('где', 68),\n",
       " ('ничего', 60),\n",
       " ('там', 57),\n",
       " ('вдруг', 53),\n",
       " ('сейчас', 47),\n",
       " ('очень', 42),\n",
       " ('потом', 38),\n",
       " ('совсем', 36),\n",
       " ('наконец', 34),\n",
       " ('тогда', 32),\n",
       " ('что-то', 30),\n",
       " ('сразу', 28),\n",
       " ('тихо', 27),\n",
       " ('затем', 26)]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#creating a list of dictionaries that contains only adverbs\n",
    "adverbs = list(filter(lambda word: word['POS'] == 'ADVB', words))\n",
    "\n",
    "#creating a list of lemmas of the adverbs\n",
    "adverb_lemmas = test(adverbs, 'lemma')\n",
    "\n",
    "#displaying top-20 adverbs by frequency\n",
    "Counter(adverb_lemmas).most_common(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3537c7d6",
   "metadata": {},
   "source": [
    "Task 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "0f74d097",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(('м-сье', 'пьер'), 199),\n",
       " (('родрига', 'иванович'), 58),\n",
       " (('сказать', 'цинциннат'), 46),\n",
       " (('не', 'мочь'), 36),\n",
       " (('мочь', 'быть'), 34),\n",
       " (('сказать', 'м-сье'), 34),\n",
       " (('что', 'я'), 32),\n",
       " (('ничего', 'не'), 32),\n",
       " (('и', 'вот'), 31),\n",
       " (('я', 'не'), 29),\n",
       " (('и', 'с'), 28),\n",
       " (('не', 'знать'), 28),\n",
       " (('то', 'есть'), 28),\n",
       " (('к', 'цинциннат'), 28),\n",
       " (('на', 'стол'), 27),\n",
       " (('у', 'я'), 27),\n",
       " (('как', 'бы'), 26),\n",
       " (('у', 'он'), 26),\n",
       " (('так', 'что'), 25),\n",
       " (('цинциннат', 'и'), 24),\n",
       " (('спросить', 'цинциннат'), 24),\n",
       " (('и', 'в'), 24),\n",
       " (('и', 'не'), 24),\n",
       " (('и', 'всё'), 23),\n",
       " (('и', 'я'), 23)]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#creating a list of lemmas\n",
    "lemmas = test(words, 'lemma')\n",
    "\n",
    "#displaying top-25 bigrams\n",
    "Counter(nltk.bigrams(lemmas)).most_common(25)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a1e8b9a7",
   "metadata": {},
   "source": [
    "Commentary on the bigram results.\n",
    "\n",
    "The bigrams can be divided into several types.\n",
    "The first type consists of bigrams 'м-сье Пьер' and 'Родрига Иванович'. These are the personal names and, apparantely, one part of such bigram cannot be used without the other.\n",
    "The bigrams 'сказать цинциннат', 'сказать м-сье' and 'спросить цинциннат' are taken from reported speech. М-сье Пьер and Цинциннат apparentely speak more than other characters (I don't remember 'cause I read Приглашение на казнь several years ago).\n",
    "The bigrams 'мочь быть', 'то есть', 'как бы', 'так что' likely are so frequent because these pairs of words are used as constructions (the first one is может быть, all the other don't change).\n",
    "High frequency of other bigrams is hard to interpret."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "c811cf9e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(('сказать', 'м-сье', 'пьер'), 34),\n",
       " (('м-сье', 'пьер', 'и'), 19),\n",
       " (('в', 'всякий', 'случай'), 10),\n",
       " (('на', 'сам', 'дело'), 8),\n",
       " (('и', 'м-сье', 'пьер'), 8),\n",
       " (('продолжать', 'м-сье', 'пьер'), 8),\n",
       " (('я', 'казаться', 'что'), 7),\n",
       " (('о', 'тот', 'что'), 7),\n",
       " (('родрига', 'иванович', 'и'), 7),\n",
       " (('в', 'конец', 'конец'), 6),\n",
       " (('сказать', 'цинциннат', 'и'), 6),\n",
       " (('м-сье', 'пьер', 'в'), 6),\n",
       " (('произнести', 'м-сье', 'пьер'), 6),\n",
       " (('у', 'он', 'в'), 5),\n",
       " (('я', 'ещё', 'не'), 5),\n",
       " (('на', 'цинциннат', 'который'), 5),\n",
       " (('с', 'тот', 'же'), 5),\n",
       " (('я', 'знать', 'что'), 5),\n",
       " (('до', 'сей', 'пора'), 5),\n",
       " (('в', 'сам', 'дело'), 5),\n",
       " (('не', 'знать', 'как'), 5),\n",
       " (('всё', 'так', 'же'), 5),\n",
       " (('говорить', 'м-сье', 'пьер'), 5),\n",
       " (('ещё', 'не', 'приехать'), 5),\n",
       " (('ходить', 'по', 'камера'), 4)]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#displaying top-25 trigrams\n",
    "Counter(nltk.trigrams(lemmas)).most_common(25)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "179eb323",
   "metadata": {},
   "source": [
    "Commentary on the trigram results.\n",
    "\n",
    "Here we have even more actions by м-ье Пьер: he talks, he continues to talk, he utters things, he says something.\n",
    "Here also we have some collocations: \"во всяком случае\", \"на самом деле\", \"в конце концов\", \"до сих пор\", \"в самом деле\".\n",
    "Other trigrams are more contextual and as I don't remember the text well I cannot say any more about them."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f98eb6d6",
   "metadata": {},
   "source": [
    "Task 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "f6f2b05d",
   "metadata": {},
   "outputs": [],
   "source": [
    "s = 'Описав около него круг, они плавно вернулись в камеру, и тут Цинциннат пожалел, что так кратко было дружеское пожатие обморока.'\n",
    "t = nltk.word_tokenize(s)\n",
    "m = {}\n",
    "for i in t:\n",
    "    m[i] = morph.parse(i)[0]\n",
    "m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "e90a710a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'описав около него круги , их плавно вернулись в камере и тут цинциннат пожалело что так кратко было дружеская пожатие обморока .'"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m['круг'] = m['круг'].inflect({'plur'})\n",
    "m['они'] = m['они'].inflect({'accs'})\n",
    "m['камеру'] = m['камеру'].inflect({'datv'})\n",
    "m['пожалел'] = m['пожалел'].inflect({'neut'})\n",
    "m['дружеское'] = m['дружеское'].inflect({'femn'})\n",
    "' '.join(m[i].word for i in m.keys())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}