Copy_of_BioNLP09_IOB_to_CoNLL.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Copy of BioNLP09_IOB_to_CoNLL.ipynb",
      "provenance": [],
      "mount_file_id": "1xtJFKVbfcrPA2z4t5P_yTmqtNv2SpK5y",
      "authorship_tag": "ABX9TyMIperXJUTcUQILcuIbBxSu",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/gokhanturer/NER_Model_SparkNLP/blob/main/Copy_of_BioNLP09_IOB_to_CoNLL.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wqPYIuoFZjmZ"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "pd.set_option(\"max_rows\",100)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!wget -q https://raw.githubusercontent.com/cambridgeltl/MTL-Bioinformatics-2016/master/data/BioNLP09-IOB/train.tsv\n",
        "!wget -q https://raw.githubusercontent.com/cambridgeltl/MTL-Bioinformatics-2016/master/data/BioNLP09-IOB/test.tsv"
      ],
      "metadata": {
        "id": "kky9qv8PZk0Q"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "with open(\"train.tsv\") as f:\n",
        "  train_data = f.read()"
      ],
      "metadata": {
        "id": "-dCneZgLZkwf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "with open(\"test.tsv\") as f:\n",
        "  test_data = f.read()"
      ],
      "metadata": {
        "id": "LZUm_sdFamNI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def data_pre_process(data):\n",
        "\n",
        "  result = []\n",
        "\n",
        "  for i in data.split(\"\\n\"):\n",
        "    if i == \"\": # Each sentence ends \"\"\n",
        "      result.append(\"None\\tNone\") # We add \"None\" between each sentences\n",
        "    else:\n",
        "      result.append(i)\n",
        "  \n",
        "  df = pd.DataFrame(result)\n",
        "\n",
        "  return df"
      ],
      "metadata": {
        "id": "mSTcxYLJZkp6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def conll_generator(data, filename):\n",
        "\n",
        "  data = data_pre_process(data)\n",
        "\n",
        "  data.to_csv(filename) # We saved the return value of the data_prepare_function as csv file\n",
        "\n",
        "  df = pd.read_csv(filename, sep = \"\\t\", usecols = [0,1],names = ['token','entity'])\n",
        "\n",
        "  # We read csv file , add column name and assigned a NaN value to first row \n",
        "  df.iloc[0,:] = np.nan\n",
        "  \n",
        "  # After reading the data, we cleaned the unnecessary characters in the columns.\n",
        "  df[\"token\"] = df[\"token\"].map(lambda x: x.split(\",\")[1], na_action = \"ignore\")\n",
        "  df[\"token\"] = df[\"token\"].map(lambda x: ',' if x == '\"' else x, na_action = \"ignore\")\n",
        "  df[\"entity\"] = df[\"entity\"].map(lambda x: x.strip('\"'), na_action = \"ignore\")\n",
        "\n",
        "  # We assigned the value np.nan to the places that are None\n",
        "  df[\"token\"]= df[\"token\"].map(lambda x: np.nan if x == 'None' else x)\n",
        "  df[\"entity\"] = df[\"entity\"].map(lambda x: np.nan if x == 'None' else x)\n",
        "\n",
        "  # We added two new columns with value \"NN\" excluding NaN values\n",
        "  df[\"pos1\"] = df[\"token\"].map(lambda x: np.nan if type(x)== float else 'NN')\n",
        "  df[\"pos2\"] = df[\"token\"].map(lambda x: np.nan if type(x)== float else 'NN')\n",
        "\n",
        "  df = df[[\"token\",\"pos1\",\"pos2\",\"entity\"]]\n",
        "\n",
        "  # We changed the column names according to the conll format\n",
        "  df.columns = [\"-DOCSTART-\", \"-X-\", \"-X-\", \"O\"]\n",
        "\n",
        "  return df"
      ],
      "metadata": {
        "id": "9R6Xh2Q3Zkmp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data = train_data\n",
        "\n",
        "filename = \"train_data.csv\"\n",
        "\n",
        "train_data = conll_generator(data, filename)"
      ],
      "metadata": {
        "id": "Kvwo-CsTZkjS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_data.to_csv(\"/content/BioNLP09_IOB_train.conll\", index = False, sep = \" \")"
      ],
      "metadata": {
        "id": "t7Dan9MvAHb0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data = test_data\n",
        "\n",
        "filename = \"test_data.csv\"\n",
        "\n",
        "test_data = conll_generator(data, filename)"
      ],
      "metadata": {
        "id": "XBl3yttlA2Pw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "test_data.to_csv(\"/content/BioNLP09_IOB_test.conll\", index = False, sep = \" \")"
      ],
      "metadata": {
        "id": "t9xFsfy3A2Mv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "cUMC9eSeA2J3"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}