workflow_example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# w2widget\n",
    "*Words to Widget*"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Import widget"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "import pandas as pd\n",
    "from gensim.models import Word2Vec\n",
    "from nltk.tokenize import WordPunctTokenizer\n",
    "\n",
    "from w2widget.utils import tokenizer_with_ws\n",
    "from w2widget.widget import Widget\n",
    "\n",
    "%aimport w2widget.utils\n",
    "%aimport w2widget.widget\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import inaugural, twitter_samples, reuters\n",
    "\n",
    "# import nltk\n",
    "# nltk.download('twitter_samples')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Twitter samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Twitter Samples\n",
      "\n",
      "These samples of Tweets (or 'status updates') were collected from the\n",
      "Twitter Streaming and REST APIs (see\n",
      "https://dev.twitter.com/overview/documentation,\n",
      "https://dev.twitter.com/streaming/overview). Each file consists of\n",
      "line-separated JSON-formatted tweets, i.e. one Tweet per line. For a\n",
      "detailed description of the JSON fields in a Tweet, see\n",
      "https://dev.twitter.com/overview/api/tweets.\n",
      "\n",
      "Any use of this data is subject to the Twitter Developer Agreement and\n",
      "Developer Policy:\n",
      "https://dev.twitter.com/overview/terms/agreement-and-policy.\n",
      "\n",
      "####################################################\n",
      "* tweets.20150430-223406.json:\n",
      "\n",
      "Collected from the public stream of the Streaming API using the\n",
      "'statuses / filter' endpoint. The value of 'track' was set to the\n",
      "following keywords: \n",
      "\n",
      "\"david cameron, miliband, milliband, sturgeon, clegg, farage, tory,\n",
      "tories, ukip, snp, libdem\"\n",
      "\n",
      "####################################################\n",
      "* positive_tweets.json\n",
      "* negative_tweets.json\n",
      "\n",
      "These were collected in July 2015 by searching against the following strings:\n",
      "\n",
      "positive\n",
      "-------\n",
      "    ':-)', ':)', ';)', ':o)', ':]', ':3', ':c)', ':>', '=]', '8)', '=)', ':}',\n",
      "    ':^)', ':-D', ':D', '8-D', '8D', 'x-D', 'xD', 'X-D', 'XD', '=-D', '=D',\n",
      "    '=-3', '=3', ':-))', \":'-)\", \":')\", ':*', ':^*', '>:P', ':-P', ':P', 'X-P',\n",
      "    'x-p', 'xp', 'XP', ':-p', ':p', '=p', ':-b', ':b', '>:)', '>;)', '>:-)',\n",
      "    '<3'\n",
      "    }\n",
      "\n",
      "negative\n",
      "------\n",
      "    ':L', ':-/', '>:/', ':S', '>:[', ':@', ':-(', ':[', ':-||', '=L', ':<',\n",
      "    ':-[', ':-<', '=\\\\', '=/', '>:(', ':(', '>.<', \":'-(\", \":'(\", ':\\\\', ':-c',\n",
      "    ':c', ':{', '>:\\\\', ';('\n",
      "   \n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "twitter_samples._readme = 'README.txt'\n",
    "print(twitter_samples.readme())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[FileSystemPathPointer('C:\\\\Users\\\\tobia\\\\AppData\\\\Roaming\\\\nltk_data\\\\corpora\\\\twitter_samples\\\\negative_tweets.json'),\n",
       " FileSystemPathPointer('C:\\\\Users\\\\tobia\\\\AppData\\\\Roaming\\\\nltk_data\\\\corpora\\\\twitter_samples\\\\positive_tweets.json'),\n",
       " FileSystemPathPointer('C:\\\\Users\\\\tobia\\\\AppData\\\\Roaming\\\\nltk_data\\\\corpora\\\\twitter_samples\\\\tweets.20150430-223406.json')]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_samples.abspaths()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets_path = twitter_samples.abspaths()[-1]\n",
    "# with open(tweets_path, 'r') as f:\n",
    "tweets = []\n",
    "with twitter_samples.open(tweets_path) as f:\n",
    "    for line in f:\n",
    "        tweets.append(json.loads(line.strip()))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "docs = [tweet[\"text\"] for tweet in tweets if \"retweeted_status\" not in tweet]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reuters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "docs = []\n",
    "\n",
    "for path in reuters.fileids():\n",
    "    with reuters.open(path) as f:\n",
    "        docs.append(f.read())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "      The Reuters-21578 benchmark corpus, ApteMod version\n",
      "\n",
      "This is a publically available version of the well-known Reuters-21578\n",
      "\"ApteMod\" corpus for text categorization.  It has been used in\n",
      "publications like these:\n",
      "\n",
      " * Yiming Yang and X. Liu. \"A re-examination of text categorization\n",
      "   methods\".  1999.  Proceedings of 22nd Annual International SIGIR.\n",
      "   http://citeseer.nj.nec.com/yang99reexamination.html\n",
      "\n",
      " * Thorsten Joachims. \"Text categorization with support vector\n",
      "   machines: learning with many relevant features\".  1998. Proceedings\n",
      "   of ECML-98, 10th European Conference on Machine Learning.\n",
      "   http://citeseer.nj.nec.com/joachims98text.html\n",
      "\n",
      "ApteMod is a collection of 10,788 documents from the Reuters financial\n",
      "newswire service, partitioned into a training set with 7769 documents\n",
      "and a test set with 3019 documents.  The total size of the corpus is\n",
      "about 43 MB.  It is also available for download from\n",
      "http://kdd.ics.uci.edu/databases/reuters21578/reuters21578.html ,\n",
      "which includes a more extensive history of the data revisions.\n",
      "\n",
      "The distribution of categories in the ApteMod corpus is highly skewed,\n",
      "with 36.7% of the documents in the most common category, and only\n",
      "0.0185% (2 documents) in each of the five least common categories.\n",
      "In fact, the original data source is even more skewed---in creating\n",
      "the corpus, any categories that did not contain at least one document\n",
      "in the training set and one document in the test set were removed from\n",
      "the corpus by its original creator.\n",
      "\n",
      "In the ApteMod corpus, each document belongs to one or more\n",
      "categories.  There are 90 categories in the corpus.  The average\n",
      "number of categories per document is 1.235, and the average number of\n",
      "documents per category is about 148, or 1.37% of the corpus.\n",
      "\n",
      " -Ken Williams\n",
      "  ken@mathforum.org\n",
      "\n",
      "         Copyright & Notification \n",
      "\n",
      "(extracted from the README at the UCI address above)\n",
      "\n",
      "The copyright for the text of newswire articles and Reuters\n",
      "annotations in the Reuters-21578 collection resides with Reuters Ltd.\n",
      "Reuters Ltd. and Carnegie Group, Inc. have agreed to allow the free\n",
      "distribution of this data *for research purposes only*.  \n",
      "\n",
      "If you publish results based on this data set, please acknowledge\n",
      "its use, refer to the data set by the name \"Reuters-21578,\n",
      "Distribution 1.0\", and inform your readers of the current location of\n",
      "the data set (see \"Availability & Questions\").\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(reuters.readme())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Inaugural speeches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "docs = []\n",
    "\n",
    "for doc in inaugural.abspaths():\n",
    "    # with open(doc, 'r') as f:\n",
    "    f = inaugural.open(doc)\n",
    "    docs.append(f.read())\n",
    "    f.close()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### NPR\n",
    "https://www.kaggle.com/datasets/shuyangli94/interview-npr-media-dialog-transcripts?resource=download&select=utterances.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/npr/utterances.csv', nrows=200_000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "docs = df['utterance'].dropna().to_list()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Text preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm.notebook import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = WordPunctTokenizer()\n",
    "\n",
    "tokenizer.tokenize_with_ws = tokenizer_with_ws(tokenizer.tokenize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "051a62685dce4dc496d12b8f648ee2e0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizing:   0%|          | 0/199999 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "document_tokens = [\n",
    "    [token.lower() for token in tokenizer.tokenize(doc) if token.isalnum()]\n",
    "    for doc in tqdm(docs, smoothing=0, desc='tokenizing')\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "356e3fc23cf74b568a07724bdb449886",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizing with ws:   0%|          | 0/199999 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tokens_with_ws = [tokenizer.tokenize_with_ws(doc) for doc in tqdm(docs, smoothing=0, desc='tokenizing with ws')]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train word2vec model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "w2v = Word2Vec()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "wv_model = Word2Vec(\n",
    "    document_tokens,\n",
    "    vector_size=200,\n",
    "    window=10,\n",
    "    workers=4,\n",
    "    seed=42,\n",
    "    epochs=20,\n",
    "    min_count=2,\n",
    ").wv\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('ukraine', 0.7139810919761658),\n",
       " ('putin', 0.7039670348167419),\n",
       " ('moscow', 0.646641194820404),\n",
       " ('iran', 0.6392899751663208),\n",
       " ('russian', 0.6354030966758728),\n",
       " ('kremlin', 0.6271708011627197),\n",
       " ('turkey', 0.6120085716247559),\n",
       " ('crimea', 0.6026292443275452),\n",
       " ('nato', 0.5902634859085083),\n",
       " ('russians', 0.58718341588974)]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wv_model.most_similar('russia')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reduce dimensions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "\n",
    "# from sklearn.manifold import TSNE\n",
    "from openTSNE import TSNE\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "normed_vectors = wv_model.get_normed_vectors()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca = PCA(n_components=50)\n",
    "pca_embedding = pca.fit_transform(normed_vectors)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------------------------------------\n",
      "TSNE(random_state=420, verbose=1)\n",
      "--------------------------------------------------------------------------------\n",
      "===> Finding 90 nearest neighbors using Annoy approximate search using euclidean distance...\n",
      "   --> Time elapsed: 38.83 seconds\n",
      "===> Calculating affinity matrix...\n",
      "   --> Time elapsed: 2.76 seconds\n",
      "===> Calculating PCA-based initialization...\n",
      "   --> Time elapsed: 0.12 seconds\n",
      "===> Running optimization with exaggeration=12.00, lr=3931.42 for 250 iterations...\n",
      "Iteration   50, KL divergence 7.0616, 50 iterations in 9.7868 sec\n",
      "Iteration  100, KL divergence 7.0623, 50 iterations in 10.1660 sec\n",
      "Iteration  150, KL divergence 7.0608, 50 iterations in 8.7114 sec\n",
      "Iteration  200, KL divergence 7.0616, 50 iterations in 7.8212 sec\n",
      "Iteration  250, KL divergence 7.0606, 50 iterations in 7.4543 sec\n",
      "   --> Time elapsed: 43.94 seconds\n",
      "===> Running optimization with exaggeration=1.00, lr=3931.42 for 500 iterations...\n",
      "Iteration   50, KL divergence 5.5609, 50 iterations in 8.5419 sec\n",
      "Iteration  100, KL divergence 5.2112, 50 iterations in 8.1025 sec\n",
      "Iteration  150, KL divergence 5.0414, 50 iterations in 8.0992 sec\n",
      "Iteration  200, KL divergence 4.9324, 50 iterations in 8.9152 sec\n",
      "Iteration  250, KL divergence 4.8539, 50 iterations in 10.3471 sec\n",
      "Iteration  300, KL divergence 4.7953, 50 iterations in 10.6564 sec\n",
      "Iteration  350, KL divergence 4.7495, 50 iterations in 11.7916 sec\n",
      "Iteration  400, KL divergence 4.7130, 50 iterations in 13.4388 sec\n",
      "Iteration  450, KL divergence 4.6834, 50 iterations in 24.3539 sec\n",
      "Iteration  500, KL divergence 4.6580, 50 iterations in 24.6813 sec\n",
      "   --> Time elapsed: 128.93 seconds\n",
      "===> Finding 15 nearest neighbors in existing embedding using Annoy approximate search...\n",
      "   --> Time elapsed: 9.40 seconds\n",
      "===> Calculating affinity matrix...\n",
      "   --> Time elapsed: 0.80 seconds\n",
      "===> Running optimization with exaggeration=4.00, lr=0.10 for 0 iterations...\n",
      "   --> Time elapsed: 0.00 seconds\n",
      "===> Running optimization with exaggeration=1.50, lr=0.10 for 250 iterations...\n",
      "Iteration   50, KL divergence 809168.4907, 50 iterations in 2.4345 sec\n",
      "Iteration  100, KL divergence 802068.1558, 50 iterations in 3.4284 sec\n",
      "Iteration  150, KL divergence 796999.6054, 50 iterations in 2.1424 sec\n",
      "Iteration  200, KL divergence 793194.0318, 50 iterations in 1.9820 sec\n",
      "Iteration  250, KL divergence 790216.3044, 50 iterations in 2.8310 sec\n",
      "   --> Time elapsed: 12.82 seconds\n"
     ]
    }
   ],
   "source": [
    "TSNE_embedding = TSNE(\n",
    "    n_components=2, learning_rate=\"auto\", random_state=420, verbose=1\n",
    ").fit(pca_embedding)\n",
    "\n",
    "wv_tsne_embedding = TSNE_embedding.transform(pca_embedding)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## doc2vec (optional)\n",
    "\n",
    "https://towardsdatascience.com/word-embeddings-and-document-vectors-when-in-doubt-simplify-8c9aaeec244e\n",
    "\n",
    "https://towardsdatascience.com/document-embedding-techniques-fed3e7a6a25d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\tobia\\anaconda3\\envs\\w2widget\\lib\\site-packages\\w2v_widget\\utils.py:2: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
      "  from tqdm.autonotebook import tqdm\n"
     ]
    }
   ],
   "source": [
    "%aimport w2v_widget.utils\n",
    "from w2widget.doc2vec import Doc2Vec, calculate_inverse_frequency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_weights = calculate_inverse_frequency(document_tokens)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b877abf964ab40a69f01cc6d47456af9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Calculating document vectors:   0%|          | 0/199999 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dv_model = Doc2Vec(wv_model, word_weights)\n",
    "\n",
    "dv_model.add_doc2vec(document_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------------------------------------\n",
      "TSNE(random_state=420, verbose=1)\n",
      "--------------------------------------------------------------------------------\n",
      "===> Finding 90 nearest neighbors using Annoy approximate search using euclidean distance...\n",
      "   --> Time elapsed: 205.94 seconds\n",
      "===> Calculating affinity matrix...\n",
      "   --> Time elapsed: 17.99 seconds\n",
      "===> Calculating PCA-based initialization...\n",
      "   --> Time elapsed: 1.59 seconds\n",
      "===> Running optimization with exaggeration=12.00, lr=16663.58 for 250 iterations...\n",
      "Iteration   50, KL divergence 7.7611, 50 iterations in 73.6381 sec\n",
      "Iteration  100, KL divergence 7.5314, 50 iterations in 71.3147 sec\n",
      "Iteration  150, KL divergence 7.5052, 50 iterations in 71.8796 sec\n",
      "Iteration  200, KL divergence 7.4970, 50 iterations in 70.9963 sec\n",
      "Iteration  250, KL divergence 7.4898, 50 iterations in 56.6100 sec\n",
      "   --> Time elapsed: 344.48 seconds\n",
      "===> Running optimization with exaggeration=1.00, lr=16663.58 for 500 iterations...\n",
      "Iteration   50, KL divergence 5.6335, 50 iterations in 56.0788 sec\n",
      "Iteration  100, KL divergence 5.2053, 50 iterations in 60.4125 sec\n",
      "Iteration  150, KL divergence 4.9896, 50 iterations in 62.3525 sec\n",
      "Iteration  200, KL divergence 4.8542, 50 iterations in 57.9933 sec\n",
      "Iteration  250, KL divergence 4.7623, 50 iterations in 63.1253 sec\n",
      "Iteration  300, KL divergence 4.7080, 50 iterations in 68.8580 sec\n",
      "Iteration  350, KL divergence 4.6780, 50 iterations in 66.9019 sec\n",
      "Iteration  400, KL divergence 4.6355, 50 iterations in 57.4848 sec\n",
      "Iteration  450, KL divergence 4.6206, 50 iterations in 66.8434 sec\n",
      "Iteration  500, KL divergence 4.5971, 50 iterations in 64.5589 sec\n",
      "   --> Time elapsed: 624.61 seconds\n",
      "===> Finding 15 nearest neighbors in existing embedding using Annoy approximate search...\n",
      "   --> Time elapsed: 54.17 seconds\n",
      "===> Calculating affinity matrix...\n",
      "   --> Time elapsed: 3.52 seconds\n",
      "===> Running optimization with exaggeration=4.00, lr=0.10 for 0 iterations...\n",
      "   --> Time elapsed: 0.00 seconds\n",
      "===> Running optimization with exaggeration=1.50, lr=0.10 for 250 iterations...\n",
      "Iteration   50, KL divergence 3731548.6504, 50 iterations in 13.5814 sec\n",
      "Iteration  100, KL divergence 3710449.5580, 50 iterations in 10.7312 sec\n",
      "Iteration  150, KL divergence 3697249.4263, 50 iterations in 10.0312 sec\n",
      "Iteration  200, KL divergence 3688093.9200, 50 iterations in 10.4794 sec\n",
      "Iteration  250, KL divergence 3681112.6070, 50 iterations in 11.8254 sec\n",
      "   --> Time elapsed: 56.68 seconds\n"
     ]
    }
   ],
   "source": [
    "dv_model.reduce_dimensions(verbose=1)\n",
    "\n",
    "dv_tsne_embedding = dv_model.TSNE_embedding_array\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving dv_tsne embeddings\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "\n",
    "print(\"Saving tokens with ws\")\n",
    "with open(\"data/npr/tokens_with_ws.pkl\", \"wb\") as f:\n",
    "    pickle.dump(tokens_with_ws, f)\n",
    "\n",
    "print(\"Saving wv_model\")\n",
    "wv_model.save_word2vec_format('data/npr/word_vectors.txt')\n",
    "\n",
    "print(\"Saving wv_tsne embeddings\")\n",
    "with open(\"data/npr/wv_tsne_embedding.pkl\", \"wb\") as f:\n",
    "    pickle.dump(wv_tsne_embedding, f)\n",
    "    \n",
    "print(\"Saving dv_model\")\n",
    "with open(\"data/npr/dv_model.pkl\", \"wb\") as f:\n",
    "    pickle.dump(dv_model, f)\n",
    "    \n",
    "print(\"Saving dv_tsne embeddings\")\n",
    "with open(\"data/npr/dv_tsne_embedding.pkl\", \"wb\") as f:\n",
    "    pickle.dump(dv_tsne_embedding, f)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Display widget"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b358cbba4a214b8588a2e10e67dd532a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HTML(value='<style>\\n.widget-button {\\n    margin-right: 160px;\\n}\\n\\n.widget-select-multiple {…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": "\n                // Javascript injection to help plotly graph fit it's container\n                selector = \"div.js-plotly-plot:nth-child(1) > div:nth-child(1) > div:nth-child(1) > div:nth-child(4) > div:nth-child(1) > div:nth-child(3) > a:nth-child(4)\";\n                \n                function waitForElementToDisplay(selector, callback, checkFrequencyInMs, timeoutInMs) {\n                    var startTimeInMs = Date.now();\n                    (function loopSearch() {\n                        if (document.querySelector(selector) != null) {\n                        callback();\n                        return;\n                        }\n                        else {\n                        setTimeout(function () {\n                            if (timeoutInMs && Date.now() - startTimeInMs > timeoutInMs)\n                            return;\n                            loopSearch();\n                        }, checkFrequencyInMs);\n                        }\n                    })();\n                }\n                \n                waitForElementToDisplay(selector,function(){document.querySelector(selector).click()},1000,9000);\n                console.warn('test')\n            ",
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "wv_widget = Widget(\n",
    "    wv_model=wv_model,\n",
    "    two_dim_word_embedding=TSNE_embedding,\n",
    "    tokens_with_ws=tokens_with_ws,\n",
    "    dv_model=None,\n",
    "    two_dim_doc_embedding=None,\n",
    "    initial_search_words=[],\n",
    ")\n",
    "\n",
    "wv_widget.display_widget()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'test': ['cameron', 'snap', 'followback']}"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wv_widget.queries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{}"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wv_widget.topics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ipywidgets.embed import embed_minimal_html, dependency_state\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "embed_minimal_html(\n",
    "    \"export.html\",\n",
    "    views=wv_widget.view,\n",
    "    title=\"w2widget\",\n",
    "    state=dependency_state(wv_widget.view),\n",
    ")\n"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "ec010e966837cae837deaaf4e5496ec78be523a218bce7b35f26aada2ba560b3"
  },
  "kernelspec": {
   "display_name": "Python 3.8.13 ('w2widget')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}