FireAI_037_TextChunking.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import word_tokenize\n",
    "def split(dataset,words_num):\n",
    "    '''\n",
    "    将dataset这一整段文本分割成N个小块，\n",
    "    使得每个小块中含有单词的数目等于words_num'''\n",
    "    words=dataset.split(' ') # 此处用空格来区分单词是否合适？\n",
    "    # words=word_tokenize(dataset) # 用分词器来分词是否更合适一些？\n",
    "    \n",
    "    rows=int(np.ceil(len(words)/words_num)) # 即行数\n",
    "    result=[] # 预计里面装的元素是rows行words_num列，最后一行可能少于words_num，故不能用np.array\n",
    "\n",
    "    # words是list，可以用切片的方式获取\n",
    "    for row in range(rows):\n",
    "        result.append(words[row*words_num:(row+1)*words_num])\n",
    "    return result\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "192427\n",
      "34\n",
      "['[', 'Emma', 'by', 'Jane', 'Austen', '1816', ']', 'VOLUME', 'I', 'CHAPTER', 'I', 'Emma', 'Woodhouse', ',', 'handsome', ',', 'clever', ',', 'and', 'rich', ',', 'with', 'a', 'comfortable', 'home', 'and', 'happy', 'disposition', ',', 'seemed']\n",
      "30\n",
      "['its', 'separate', 'lawn', ',', 'and', 'shrubberies', ',', 'and', 'name', ',']\n",
      "10\n"
     ]
    }
   ],
   "source": [
    "# 测试一下\n",
    "# 数据集暂时用简·奥斯丁的《爱玛》中的文本\n",
    "dataset=nltk.corpus.gutenberg.words('austen-emma.txt')\n",
    "print(len(dataset)) # 192427 代表读入正常\n",
    "result=split(\" \".join(dataset[:1000]), 30) # 只取前面的1000个单词，每30个单词分一个块，一共有34个块\n",
    "print(len(result))\n",
    "print(result[0])\n",
    "print(len(result[0]))\n",
    "print(result[-1])\n",
    "print(len(result[-1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}