label_studio_tagtog형식맞추기.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "label-studio_tagtog형식맞추기.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "RYDEmvH4cuBs"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Label-studio 파일 읽어오기"
      ],
      "metadata": {
        "id": "QCvJMyJ2GAf-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "file_name = 'label-studio.csv'\n",
        "DF = pd.read_csv(file_name)"
      ],
      "metadata": {
        "id": "kWEWS2VrClg_"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## TagTog 형식 맞추기"
      ],
      "metadata": {
        "id": "yUCZUnUQGXpO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "new_df = pd.DataFrame(columns=[\"sentence\",\"sentence_with_entity\",\"subject_entity\",\"object_entity\",\"class\"])"
      ],
      "metadata": {
        "id": "-egwh9ZgCv-D"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "new_df['sentence'] = DF['text']"
      ],
      "metadata": {
        "id": "IRsk1oSrDSNW"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "DF['label'] = DF['label'].str.slice(1,-1)"
      ],
      "metadata": {
        "id": "DttinlNWE1cj"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "DF['label'] = DF['label'].map(lambda x : x.replace('labels','type'))"
      ],
      "metadata": {
        "id": "6gfDhoPWFUSz"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def return_sbj(data):\n",
        "  result = [{'start':0,'end':0,'text':'','type':''},{'start':0,'end':0,'text':'','type':''}]\n",
        "  sp = data.split(',')\n",
        "\n",
        "  result[0]['start'] = int(sp[0].split(':')[1])\n",
        "  result[0]['end'] = int(sp[1].split(':')[1]) - 1\n",
        "  result[0]['text'] = sp[2].split(':')[1].strip()\n",
        "  tps = sp[3].split('-')\n",
        "  result[0]['type'] = tps[1].lower()\n",
        "\n",
        "\n",
        "  if 'SBJ' in tps[0]:\n",
        "    sbj_idx = 0\n",
        "  elif 'OBJ' in tps[0]:\n",
        "    sbj_idx = 1\n",
        "\n",
        "  result[1]['start'] = int(sp[4].split(':')[1])\n",
        "  result[1]['end'] = int(sp[5].split(':')[1]) - 1\n",
        "  result[1]['text'] = sp[6].split(':')[1].strip()\n",
        "  result[1]['type'] = sp[7].split('-')[1].lower()\n",
        "\n",
        "  if sbj_idx == 0:\n",
        "    return result[0]\n",
        "  elif sbj_idx == 1:\n",
        "    return result[1]\n",
        "  else:\n",
        "    print('err')"
      ],
      "metadata": {
        "id": "D_8fWXNMfoZM"
      },
      "execution_count": 135,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def return_obj(data):\n",
        "  result = [{'start':0,'end':0,'text':'','type':''},{'start':0,'end':0,'text':'','type':''}]\n",
        "  sp = data.split(',')\n",
        "\n",
        "  result[0]['start'] = int(sp[0].split(':')[1])\n",
        "  result[0]['end'] = int(sp[1].split(':')[1]) - 1\n",
        "  result[0]['text'] = sp[2].split(':')[1].strip()\n",
        "  tps = sp[3].split('-')\n",
        "  result[0]['type'] = tps[1].lower()\n",
        "\n",
        "\n",
        "  if 'SBJ' in tps[0]:\n",
        "    sbj_idx = 0\n",
        "  elif 'OBJ' in tps[0]:\n",
        "    sbj_idx = 1\n",
        "\n",
        "  result[1]['start'] = int(sp[4].split(':')[1])\n",
        "  result[1]['end'] = int(sp[5].split(':')[1]) - 1\n",
        "  result[1]['text'] = sp[6].split(':')[1].strip()\n",
        "  result[1]['type'] = sp[7].split('-')[1].lower()\n",
        "\n",
        "  if sbj_idx == 0:\n",
        "    return result[1]\n",
        "  elif sbj_idx == 1:\n",
        "    return result[0]\n",
        "  else:\n",
        "    print('err')"
      ],
      "metadata": {
        "id": "l7TjD2bpmsAi"
      },
      "execution_count": 136,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def return_class(data):\n",
        "  result = [{'start':0,'end':0,'text':'','type':''},{'start':0,'end':0,'text':'','type':''}]\n",
        "  sp = data.split(',')\n",
        "\n",
        "  tps = sp[3].split('-')\n",
        "  tp1 = tps[1].lower()\n",
        "  cls1 = tps[2].split('\"')[0].lower()\n",
        "\n",
        "\n",
        "  if 'SBJ' in tps[0]:\n",
        "    sbj_idx = 0\n",
        "  elif 'OBJ' in tps[0]:\n",
        "    sbj_idx = 1\n",
        "\n",
        "  tps = sp[7].split('-')\n",
        "  tp2 = tps[1].lower()\n",
        "  cls2 = tps[2].split('\"')[0].lower()\n",
        "\n",
        "  if cls1=='no_relation' and cls1==cls2:\n",
        "    return 'no_relation'\n",
        "  elif sbj_idx == 0 and cls1==cls2:\n",
        "    return f'{tp1}:{cls1}'\n",
        "  elif sbj_idx == 1 and cls1==cls2:\n",
        "    return f'{tp2}:{cls1}'\n",
        "  else:\n",
        "    print('err')"
      ],
      "metadata": {
        "id": "O3vNTVJf3Vl9"
      },
      "execution_count": 137,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "new_df['subject_entity'] = DF['label'].apply(lambda x : return_sbj(x))"
      ],
      "metadata": {
        "id": "Jsi-pio1DkFQ"
      },
      "execution_count": 138,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "new_df['object_entity'] = DF['label'].apply(lambda x : return_obj(x))"
      ],
      "metadata": {
        "id": "Mv9g3jtHgobT"
      },
      "execution_count": 139,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "new_df['class'] = DF['label'].apply(lambda x : return_class(x))"
      ],
      "metadata": {
        "id": "d2vmlxdH44OW"
      },
      "execution_count": 140,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def sen_w_entity(sen,sbj,obj):\n",
        "  result = ''\n",
        "  def cleaning(word):\n",
        "    return word.replace('\"','').replace(\"'\",'')\n",
        "  if sbj['start'] < obj['start']:\n",
        "    result = f\"{sen[:sbj['start']]}<sbj:{cleaning(sbj['text'])}>{sen[sbj['end']+1:obj['start']]}<obj:{cleaning(obj['text'])}>{sen[obj['end']+1:]}\"\n",
        "  else:\n",
        "    result = f\"{sen[:obj['start']]}<obj:{cleaning(obj['text'])}>{sen[obj['end']+1:sbj['start']]}<sbj:{cleaning(sbj['text'])}>{sen[sbj['end']+1:]}\"\n",
        "  return result"
      ],
      "metadata": {
        "id": "EI_dsgQcm9ib"
      },
      "execution_count": 141,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "new_df['sentence_with_entity'] = new_df.apply(lambda x : sen_w_entity(x['sentence'],x['subject_entity'],x['object_entity']),axis=1)"
      ],
      "metadata": {
        "id": "UkUCDl7Srobz"
      },
      "execution_count": 142,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 기존 구글 스프레드 시트에 붙이기"
      ],
      "metadata": {
        "id": "BiHS8VLUGc1x"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from oauth2client.service_account import ServiceAccountCredentials\n",
        "import gspread\n",
        "import json\n",
        "import glob\n",
        "import re\n",
        "import os\n",
        "import pandas as pd\n",
        "import html"
      ],
      "metadata": {
        "id": "gLE6ZMh0xROb"
      },
      "execution_count": 144,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "scope = ['https://spreadsheets.google.com/feeds',\n",
        "        'https://www.googleapis.com/auth/drive']\n",
        "credentials = ServiceAccountCredentials.from_json_keyfile_name(\n",
        "        'api_key.json', scope)\n",
        "gc = gspread.authorize(credentials)\n",
        "\n",
        "# 자신이 만든 구글 시트 이름\n",
        "sh = gc.open('반려동물_데이터셋생성')\n",
        "sheet = sh.worksheet('시트1')"
      ],
      "metadata": {
        "id": "bcoJ6p_ExqzN"
      },
      "execution_count": 104,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 구글시트에 입력\n",
        "values = sheet.get_all_values()\n",
        "\n",
        "header, rows = values[0], values[1:]\n",
        "data = pd.DataFrame(rows, columns=header)\n",
        "\n",
        "column_list = [\"sentence\",\"sentence_with_entity\",\"subject_entity\",\"object_entity\",\"class\"]\n",
        "data = data[column_list]\n",
        "\n",
        "sen_list =  list(data.sentence_with_entity.values)\n",
        "print(f'sentence list : {len(sen_list)}')\n",
        "\n",
        "sheet.resize(len(sen_list)+len(new_df)+1,10)\n",
        "start_idx = 473\n",
        "list_range = f\"a{start_idx}:e{start_idx+len(new_df)}\"\n",
        "cell_list = sheet.range(list_range)\n",
        "\n",
        "col_num = len(column_list)\n",
        "print(f'c : {len(cell_list)}, {col_num}')\n",
        "for i in range(len(new_df)):\n",
        "    cell_list[(col_num*i)].value = new_df['sentence'][i]\n",
        "    cell_list[(col_num*i)+1].value = new_df['sentence_with_entity'][i]\n",
        "    cell_list[(col_num*i)+2].value = str(new_df['subject_entity'][i])\n",
        "    cell_list[(col_num*i)+3].value = str(new_df['object_entity'][i])\n",
        "    cell_list[(col_num*i)+4].value = new_df['class'][i]\n",
        "sheet.update_cells(cell_list)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MNajjOMOxO_D",
        "outputId": "6d4b8145-d39f-48f8-ec83-244e0dfcbcca"
      },
      "execution_count": 145,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "sentence list : 494\n",
            "c : 120, 5\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'spreadsheetId': '1dcLAlfOTb_n7NSUN3y9MTzy7_MBvrSsFoDXyR1F-AaE',\n",
              " 'updatedCells': 120,\n",
              " 'updatedColumns': 5,\n",
              " 'updatedRange': \"'시트1'!A473:E496\",\n",
              " 'updatedRows': 24}"
            ]
          },
          "metadata": {},
          "execution_count": 145
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "68NAR6uMA5S9"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}