02_gutenberg_cleaning_concat_ces.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "run_control": {
     "frozen": false,
     "read_only": false
    }
   },
   "source": [
    "## Downloading and cleaning some books:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-03T18:41:40.820831Z",
     "start_time": "2018-06-03T18:41:40.194054Z"
    }
   },
   "outputs": [],
   "source": [
    "import urllib.request\n",
    "import re\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-03T18:41:48.897867Z",
     "start_time": "2018-06-03T18:41:40.822534Z"
    }
   },
   "outputs": [],
   "source": [
    "books = pd.read_csv('books.csv', encoding='latin-1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-03T18:41:48.917393Z",
     "start_time": "2018-06-03T18:41:48.899502Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(18807125, 4)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "books.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gutenberg_id</th>\n",
       "      <th>text</th>\n",
       "      <th>title</th>\n",
       "      <th>author</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>11</td>\n",
       "      <td>ALICE'S ADVENTURES IN WONDERLAND</td>\n",
       "      <td>Alice's Adventures in Wonderland</td>\n",
       "      <td>Carroll, Lewis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Alice's Adventures in Wonderland</td>\n",
       "      <td>Carroll, Lewis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11</td>\n",
       "      <td>Lewis Carroll</td>\n",
       "      <td>Alice's Adventures in Wonderland</td>\n",
       "      <td>Carroll, Lewis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Alice's Adventures in Wonderland</td>\n",
       "      <td>Carroll, Lewis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11</td>\n",
       "      <td>THE MILLENNIUM FULCRUM EDITION 3.0</td>\n",
       "      <td>Alice's Adventures in Wonderland</td>\n",
       "      <td>Carroll, Lewis</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   gutenberg_id                                text  \\\n",
       "0            11    ALICE'S ADVENTURES IN WONDERLAND   \n",
       "1            11                                 NaN   \n",
       "2            11                       Lewis Carroll   \n",
       "3            11                                 NaN   \n",
       "4            11  THE MILLENNIUM FULCRUM EDITION 3.0   \n",
       "\n",
       "                              title          author  \n",
       "0  Alice's Adventures in Wonderland  Carroll, Lewis  \n",
       "1  Alice's Adventures in Wonderland  Carroll, Lewis  \n",
       "2  Alice's Adventures in Wonderland  Carroll, Lewis  \n",
       "3  Alice's Adventures in Wonderland  Carroll, Lewis  \n",
       "4  Alice's Adventures in Wonderland  Carroll, Lewis  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "books.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-03T18:41:51.477016Z",
     "start_time": "2018-06-03T18:41:48.919505Z"
    }
   },
   "outputs": [],
   "source": [
    "books = books.fillna('') #get rid of NaNs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-03T18:43:26.285825Z",
     "start_time": "2018-06-03T18:43:21.932603Z"
    }
   },
   "outputs": [],
   "source": [
    "books[['text', 'title', 'author']] = books[['text', 'title', 'author']].astype(str)\n",
    "books = books.fillna('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-03T18:43:30.794637Z",
     "start_time": "2018-06-03T18:43:30.791838Z"
    }
   },
   "outputs": [],
   "source": [
    "book_titles = list(books['title'].unique())\n",
    "book_authors = list(books['author'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-03T18:43:30.823366Z",
     "start_time": "2018-06-03T18:43:30.819680Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3515"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(book_titles) #some books have same title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "950"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(book_authors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-03T18:54:01.466243Z",
     "start_time": "2018-06-03T18:54:01.299105Z"
    }
   },
   "outputs": [],
   "source": [
    "book_ids = books[['gutenberg_id','author','title']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-03T18:54:02.462125Z",
     "start_time": "2018-06-03T18:54:01.468083Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3527"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "book_ids = book_ids.drop_duplicates(['author','title'])\n",
    "len(book_ids) #keep unique author+title combos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gutenberg_id</th>\n",
       "      <th>author</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>11</td>\n",
       "      <td>Carroll, Lewis</td>\n",
       "      <td>Alice's Adventures in Wonderland</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3339</th>\n",
       "      <td>12</td>\n",
       "      <td>Carroll, Lewis</td>\n",
       "      <td>Through the Looking-Glass</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7246</th>\n",
       "      <td>13</td>\n",
       "      <td>Carroll, Lewis</td>\n",
       "      <td>The Hunting of the Snark: An Agony in Eight Fits</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8086</th>\n",
       "      <td>15</td>\n",
       "      <td>Melville, Herman</td>\n",
       "      <td>Moby Dick</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8248</th>\n",
       "      <td>19</td>\n",
       "      <td>Longfellow, Henry Wadsworth</td>\n",
       "      <td>The Song of Hiawatha</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      gutenberg_id                       author  \\\n",
       "0               11               Carroll, Lewis   \n",
       "3339            12               Carroll, Lewis   \n",
       "7246            13               Carroll, Lewis   \n",
       "8086            15             Melville, Herman   \n",
       "8248            19  Longfellow, Henry Wadsworth   \n",
       "\n",
       "                                                 title  \n",
       "0                     Alice's Adventures in Wonderland  \n",
       "3339                         Through the Looking-Glass  \n",
       "7246  The Hunting of the Snark: An Agony in Eight Fits  \n",
       "8086                                         Moby Dick  \n",
       "8248                              The Song of Hiawatha  "
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "book_ids.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gutenberg_id</th>\n",
       "      <th>author</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>474573</th>\n",
       "      <td>225</td>\n",
       "      <td>MacDonald, George</td>\n",
       "      <td>At the Back of the North Wind</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12229114</th>\n",
       "      <td>18614</td>\n",
       "      <td></td>\n",
       "      <td>At the Back of the North Wind</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17972259</th>\n",
       "      <td>31686</td>\n",
       "      <td>Smith, Evelyn E.</td>\n",
       "      <td>Collector's Item</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17483938</th>\n",
       "      <td>28924</td>\n",
       "      <td>Young, Robert F.</td>\n",
       "      <td>Collector's Item</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18645803</th>\n",
       "      <td>51305</td>\n",
       "      <td>Harmon, Jim</td>\n",
       "      <td>Confidence Game</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18043092</th>\n",
       "      <td>32243</td>\n",
       "      <td>McKimmey, James</td>\n",
       "      <td>Confidence Game</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6909308</th>\n",
       "      <td>7439</td>\n",
       "      <td></td>\n",
       "      <td>English Fairy Tales</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11417454</th>\n",
       "      <td>17034</td>\n",
       "      <td>Steel, Flora Annie Webster</td>\n",
       "      <td>English Fairy Tales</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17650117</th>\n",
       "      <td>29832</td>\n",
       "      <td>Wells, Basil</td>\n",
       "      <td>Second Sight</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15784792</th>\n",
       "      <td>22997</td>\n",
       "      <td>Nourse, Alan Edward</td>\n",
       "      <td>Second Sight</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17300527</th>\n",
       "      <td>26140</td>\n",
       "      <td>Kenyon, Ernest M.</td>\n",
       "      <td>Security</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15333018</th>\n",
       "      <td>22239</td>\n",
       "      <td>Anderson, Poul</td>\n",
       "      <td>Security</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14989754</th>\n",
       "      <td>21823</td>\n",
       "      <td></td>\n",
       "      <td>The Butterfly's Ball and the Grasshopper's Feast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14016490</th>\n",
       "      <td>20860</td>\n",
       "      <td>Roscoe, William</td>\n",
       "      <td>The Butterfly's Ball and the Grasshopper's Feast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18442349</th>\n",
       "      <td>50332</td>\n",
       "      <td>O'Brien, Fitz-James</td>\n",
       "      <td>The Diamond Lens</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15856528</th>\n",
       "      <td>23169</td>\n",
       "      <td>O'Brien, Fitz James</td>\n",
       "      <td>The Diamond Lens</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17802946</th>\n",
       "      <td>30438</td>\n",
       "      <td>McKimmey, James</td>\n",
       "      <td>The Eyes Have It</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17908231</th>\n",
       "      <td>30833</td>\n",
       "      <td>Garrett, Randall</td>\n",
       "      <td>The Eyes Have It</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17956325</th>\n",
       "      <td>31516</td>\n",
       "      <td>Dick, Philip K.</td>\n",
       "      <td>The Eyes Have It</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>505891</th>\n",
       "      <td>285</td>\n",
       "      <td>Hyne, Charles John Cutcliffe Wright</td>\n",
       "      <td>The Lost Continent</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>351712</th>\n",
       "      <td>149</td>\n",
       "      <td>Burroughs, Edgar Rice</td>\n",
       "      <td>The Lost Continent</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17417733</th>\n",
       "      <td>27797</td>\n",
       "      <td>Vance, Gerald</td>\n",
       "      <td>Vital Ingredient</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17853712</th>\n",
       "      <td>30491</td>\n",
       "      <td>De Vet, Charles V. (Charles Vincent)</td>\n",
       "      <td>Vital Ingredient</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          gutenberg_id                                author  \\\n",
       "474573             225                     MacDonald, George   \n",
       "12229114         18614                                         \n",
       "17972259         31686                      Smith, Evelyn E.   \n",
       "17483938         28924                      Young, Robert F.   \n",
       "18645803         51305                           Harmon, Jim   \n",
       "18043092         32243                       McKimmey, James   \n",
       "6909308           7439                                         \n",
       "11417454         17034            Steel, Flora Annie Webster   \n",
       "17650117         29832                          Wells, Basil   \n",
       "15784792         22997                   Nourse, Alan Edward   \n",
       "17300527         26140                     Kenyon, Ernest M.   \n",
       "15333018         22239                        Anderson, Poul   \n",
       "14989754         21823                                         \n",
       "14016490         20860                       Roscoe, William   \n",
       "18442349         50332                   O'Brien, Fitz-James   \n",
       "15856528         23169                   O'Brien, Fitz James   \n",
       "17802946         30438                       McKimmey, James   \n",
       "17908231         30833                      Garrett, Randall   \n",
       "17956325         31516                       Dick, Philip K.   \n",
       "505891             285   Hyne, Charles John Cutcliffe Wright   \n",
       "351712             149                 Burroughs, Edgar Rice   \n",
       "17417733         27797                         Vance, Gerald   \n",
       "17853712         30491  De Vet, Charles V. (Charles Vincent)   \n",
       "\n",
       "                                                     title  \n",
       "474573                       At the Back of the North Wind  \n",
       "12229114                     At the Back of the North Wind  \n",
       "17972259                                  Collector's Item  \n",
       "17483938                                  Collector's Item  \n",
       "18645803                                   Confidence Game  \n",
       "18043092                                   Confidence Game  \n",
       "6909308                                English Fairy Tales  \n",
       "11417454                               English Fairy Tales  \n",
       "17650117                                      Second Sight  \n",
       "15784792                                      Second Sight  \n",
       "17300527                                          Security  \n",
       "15333018                                          Security  \n",
       "14989754  The Butterfly's Ball and the Grasshopper's Feast  \n",
       "14016490  The Butterfly's Ball and the Grasshopper's Feast  \n",
       "18442349                                  The Diamond Lens  \n",
       "15856528                                  The Diamond Lens  \n",
       "17802946                                  The Eyes Have It  \n",
       "17908231                                  The Eyes Have It  \n",
       "17956325                                  The Eyes Have It  \n",
       "505891                                  The Lost Continent  \n",
       "351712                                  The Lost Continent  \n",
       "17417733                                  Vital Ingredient  \n",
       "17853712                                  Vital Ingredient  "
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "duplicates = book_ids[book_ids.duplicated('title',keep=False)]\n",
    "duplicates.sort_values(\"title\") #we have some duplicate titles but they may be other books/authors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[7439, 18614, 21823]"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "blank_dupes = list(duplicates.loc[duplicates['author'] == '', 'gutenberg_id'])\n",
    "blank_dupes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3524"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#I want to get rid of these blank dupes (3 books with no author and identical titles)\n",
    "for number in blank_dupes:\n",
    "    book_ids = book_ids[book_ids.gutenberg_id != number]\n",
    "    \n",
    "len(book_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-03T18:54:11.504290Z",
     "start_time": "2018-06-03T18:54:02.471260Z"
    }
   },
   "outputs": [],
   "source": [
    "# make a list of complete book text for each book\n",
    "\n",
    "books_list=[]\n",
    "for book in book_ids.gutenberg_id.unique():\n",
    "    books_list.append((' ').join(books.loc[books['gutenberg_id'] == book]['text']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'books' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-80df49b3193f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mbooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mbooks\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'gutenberg_id'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mbook\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'text'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'books' is not defined"
     ]
    }
   ],
   "source": [
    "books.loc[books['gutenberg_id'] == book]['text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3524"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(books_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-03T18:54:11.509758Z",
     "start_time": "2018-06-03T18:54:11.506000Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"The War of the Worlds  by H. G. Wells [1898]        But who shall dwell in these worlds if they be      inhabited? .  .  .  Are we or they Lords of the      World? .  .  .  And how are all things made for man?--           KEPLER (quoted in The Anatomy of Melancholy)    BOOK ONE  THE COMING OF THE MARTIANS    CHAPTER ONE  THE EVE OF THE WAR   No one would have believed in the last years of the nineteenth century that this world was being watched keenly and closely by intelligences greater than man's and yet as mortal as his own; that as men busied themselves about their various concerns they were scrutinised and studied, perhaps almost as narrowly as a man with a microscope might scrutinise the transient creatures that swarm and multiply in a drop of water.  With infinite complacency men went to and fro over this globe about their little affairs, serene in their assurance of their empire over matter.  It is possible that the infusoria under the microscope do the same.  No one gave a tho\""
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "books_list[6][:1000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-03T18:54:12.764036Z",
     "start_time": "2018-06-03T18:54:12.724410Z"
    }
   },
   "outputs": [],
   "source": [
    "book_ids['text'] = books_list\n",
    "books_df = book_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-03T18:55:35.046127Z",
     "start_time": "2018-06-03T18:55:33.451450Z"
    }
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "pickler = open(\"books_df.pkl\",\"wb\")\n",
    "pickle.dump(books_df, pickler)\n",
    "pickler.close()\n",
    "\n",
    "pickler = open(\"book_authors.pkl\",\"wb\")\n",
    "pickle.dump(book_authors, pickler)\n",
    "pickler.close()\n",
    "\n",
    "pickler = open(\"books_list.pkl\",\"wb\")\n",
    "pickle.dump(books_list, pickler)\n",
    "pickler.close()\n",
    "\n",
    "pickler = open(\"book_titles.pkl\",\"wb\")\n",
    "pickle.dump(book_titles, pickler)\n",
    "pickler.close()"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  },
  "toc": {
   "nav_menu": {
    "height": "123px",
    "width": "252px"
   },
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}