2_1텍스트추출.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "2-1텍스트추출.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "mount_file_id": "1-9JivZdJ7Xjf5XWIDeJ25r6ipSrqFZyi",
      "authorship_tag": "ABX9TyNMzdVraUHJEu6L6iJ3OyuN",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/dlwlgus53/KoreanEmbedding/blob/master/2_1%ED%85%8D%EC%8A%A4%ED%8A%B8%EC%B6%94%EC%B6%9C.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zPJ5xH72BJUS"
      },
      "source": [
        "## 데이터 파일 읽기 \n",
        "\n",
        "- wiki 백과\n",
        "- koquard\n",
        "- 네이버 영화 리뷰\n",
        "\n",
        "파일을 읽고, 필요한 text를 추출하는 방법 코드입니다\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wd_FOwlxtBIX"
      },
      "source": [
        "#### 필요 모듈 load"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2H9y7TBW-YEE"
      },
      "source": [
        "root_path = '/content/drive/MyDrive/2021-1/AI데이터활용교재개발/code'\n",
        "import re, json, glob\n",
        "from gensim.corpora import WikiCorpus, Dictionary\n",
        "from gensim.utils import to_unicode\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2kilgUwptEw4"
      },
      "source": [
        "### Wiki 백과 파일에서 text 추출하기\n",
        "- 데이터 : https://dumps.wikimedia.org/kowiki/latest/kowiki-latest-pages-articles.xml.bz2"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G7Xfc5ilBRNN"
      },
      "source": [
        "#### 정규표현식 지정"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SYNxA9-9_MUU"
      },
      "source": [
        "WIKI_REMOVE_CHARS = re.compile(\"'+|(=+.{2,30}=+)|__TOC__|(ファイル:).+|:(en|de|it|fr|es|kr|zh|no|fi):|\\n\", re.UNICODE)\n",
        "WIKI_SPACE_CHARS = re.compile(\"(\\\\s|゙|゚|　)+\", re.UNICODE)\n",
        "EMAIL_PATTERN = re.compile(\"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\\.[a-zA-Z0-9-.]+$)\", re.UNICODE)\n",
        "URL_PATTERN = re.compile(\"(ftp|http|https)?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+\", re.UNICODE)\n",
        "WIKI_REMOVE_TOKEN_CHARS = re.compile(\"(\\\\*$|:$|^파일:.+|^;)\", re.UNICODE)\n",
        "MULTIPLE_SPACES = re.compile(' +', re.UNICODE)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kutwzTAiEle3"
      },
      "source": [
        "def tokenize(content, token_min_len=2, token_max_len=100, lower=True):\n",
        "    content = re.sub(EMAIL_PATTERN, ' ', content)  # remove email pattern\n",
        "    content = re.sub(URL_PATTERN, ' ', content) # remove url pattern\n",
        "    content = re.sub(WIKI_REMOVE_CHARS, ' ', content)  # remove unnecessary chars\n",
        "    content = re.sub(WIKI_SPACE_CHARS, ' ', content)\n",
        "    content = re.sub(MULTIPLE_SPACES, ' ', content)\n",
        "    tokens = content.replace(\", )\", \"\").split(\" \")\n",
        "    result = []\n",
        "    for token in tokens:\n",
        "        if not token.startswith('_'):\n",
        "            token_candidate = to_unicode(re.sub(WIKI_REMOVE_TOKEN_CHARS, '', token))\n",
        "        else:\n",
        "            token_candidate = \"\"\n",
        "        if len(token_candidate) > 0:\n",
        "            result.append(token_candidate)\n",
        "    return result\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qEeOiIlvE9J_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "426da3c7-269a-42d8-b7b8-935d1ab6b552"
      },
      "source": [
        "\"\"\"Convert Wikipedia xml dump file to text corpus\"\"\"\n",
        "in_f = root_path + '/data/raw/kowiki-latest-pages-articles.xml.bz2' #\n",
        "out_f = root_path + '/data/processed/processed_wiki_ko.txt' #\n",
        "\n",
        "output = open(out_f, 'w', encoding = \"utf-8\")\n",
        "wiki = WikiCorpus(in_f, tokenizer_func=tokenize, dictionary=Dictionary(), processes = 1)\n",
        "i = 0\n",
        "for text in wiki.get_texts():\n",
        "    output.write(bytes(' '.join(text), 'utf-8').decode('utf-8') + '\\n')\n",
        "    i = i + 1\n",
        "    if (i % 10000 == 0):\n",
        "      print('Processed ' + str(i) + ' articles')\n",
        "output.close()\n",
        "print('Processing complete!')\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Processed 10000 articles\n",
            "Processed 20000 articles\n",
            "Processed 30000 articles\n",
            "Processed 40000 articles\n",
            "Processed 50000 articles\n",
            "Processed 60000 articles\n",
            "Processed 70000 articles\n",
            "Processed 80000 articles\n",
            "Processed 90000 articles\n",
            "Processed 100000 articles\n",
            "Processed 110000 articles\n",
            "Processed 120000 articles\n",
            "Processed 130000 articles\n",
            "Processed 140000 articles\n",
            "Processed 150000 articles\n",
            "Processed 160000 articles\n",
            "Processed 170000 articles\n",
            "Processed 180000 articles\n",
            "Processed 190000 articles\n",
            "Processed 200000 articles\n",
            "Processed 210000 articles\n",
            "Processed 220000 articles\n",
            "Processed 230000 articles\n",
            "Processed 240000 articles\n",
            "Processed 250000 articles\n",
            "Processed 260000 articles\n",
            "Processed 270000 articles\n",
            "Processed 280000 articles\n",
            "Processed 290000 articles\n",
            "Processed 300000 articles\n",
            "Processed 310000 articles\n",
            "Processed 320000 articles\n",
            "Processed 330000 articles\n",
            "Processed 340000 articles\n",
            "Processed 350000 articles\n",
            "Processed 360000 articles\n",
            "Processed 370000 articles\n",
            "Processing complete!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9Yj6TexbFvdT"
      },
      "source": [
        "###  KorQuAD \n",
        "- 데이터 : https://github.com/korquad/korquad.github.io/tree/master/dataset 에서KorQuAD_v1.0_dev.json, KorQuAD_v1.0_train.json 다운로드\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_lLNZOiwGBnb"
      },
      "source": [
        "def process_korQuAD(corpus_fname, output_fname):\n",
        "    with open(corpus_fname) as f1, open(output_fname, 'w', encoding='utf-8') as f2:\n",
        "        dataset_json = json.load(f1)\n",
        "        dataset = dataset_json['data']\n",
        "        for article in dataset:\n",
        "            w_lines = []\n",
        "            for paragraph in article['paragraphs']:\n",
        "                w_lines.append(paragraph['context'])\n",
        "                for qa in paragraph['qas']:\n",
        "                    q_text = qa['question']\n",
        "                    for a in qa['answers']:\n",
        "                        a_text = a['text']\n",
        "                        w_lines.append(q_text + \" \" + a_text)\n",
        "            for line in w_lines:\n",
        "                f2.writelines(line + \"\\n\")\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m5nTVaFPGHOb"
      },
      "source": [
        "# train파일 process\n",
        "in_f_train = root_path + '/data/raw/KorQuAD_v1.0_train.json'\n",
        "out_f_train = root_path + '/data/processed/processed_korquad_train.txt'\n",
        "process_korQuAD(in_f_train, out_f_train)\n",
        "\n",
        "# dev 파일 process\n",
        "in_f_test = root_path + '/data/raw/KorQuAD_v1.0_dev.json'\n",
        "out_f_test = root_path + '/data/processed/processed_korquad_dev.txt'\n",
        "process_korQuAD(in_f_test, out_f_test)\n",
        "# 두 파일을 합친 파일 생성\n",
        "out_f_combined = root_path + '/data/processed/processed_korquad.txt'\n",
        "with open(out_f_train, encoding='utf-8') as f1, open(out_f_test, encoding='utf-8') as f2, open(out_f_combined, 'w', encoding='utf-8') as f3:\n",
        "    f3.write(f1.read() + '\\n' + f2.read())\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-Kh5HDHrHV41"
      },
      "source": [
        "### 네이버 영화 리뷰 \n",
        "데이터 : \n",
        "- https://github.com/e9t/nsmc/raw/master/ratings.txt\n",
        "- https://github.com/e9t/nsmc/raw/master/ratings_train.txt\n",
        "- https://github.com/e9t/nsmc/raw/master/ratings_test.txt\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Drq2-88NHkAJ"
      },
      "source": [
        "def process_nsmc(corpus_path, output_fname, with_label=True, with_id = False):\n",
        "  # file open with encoding\n",
        "  with open(corpus_path, 'r', encoding='utf-8') as f1, \\\n",
        "                open(output_fname, 'w', encoding='utf-8') as f2:\n",
        "            next(f1)  # skip head line\n",
        "            for line in f1:\n",
        "              # strip -> split\n",
        "                id, sentence, label = line.strip().split('\\t')\n",
        "                if not sentence: continue\n",
        "                if with_label:\n",
        "                    f2.writelines(sentence + \"\\u241E\" + label + \"\\n\")\n",
        "                elif with_id:\n",
        "                    f2.writelines(sentence + \"\\u241E\" + id + \"\\n\")\n",
        "                else:\n",
        "                    f2.writelines(sentence + \"\\n\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TcW122S3Hsnd"
      },
      "source": [
        "input_path = root_path + '/data/raw/ratings.txt'\n",
        "output_path = root_path + '/data/processed/processed_ratings.txt'\n",
        "process_nsmc(input_path, output_path, with_label=False, with_id = False)\n",
        "\n",
        "input_path = root_path + '/data/raw/ratings.txt'\n",
        "output_path = root_path + '/data/processed/processed_ratings_movieid.txt'\n",
        "process_nsmc(input_path, output_path, with_label=False, with_id = True)\n",
        "\n",
        "input_path = root_path + '/data/raw/ratings_train.txt'\n",
        "output_path = root_path + '/data/processed/processed_ratings_train.txt'\n",
        "\n",
        "process_nsmc(input_path, output_path, with_label=True, with_id = False)\n",
        "\n",
        "input_path  =root_path + '/data/raw/ratings_test.txt'\n",
        "output_path=  root_path + '/data/processed/processed_ratings_test.txt'\n",
        "process_nsmc(input_path, output_path, with_label=True, with_id = False)\n",
        "\n",
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}