BiasDetectionBiasGPT.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Build Word processing class"
      ],
      "metadata": {
        "id": "WyM3Sh2-ort0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%pip install pyvi underthesea sentence-transformers\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Q7z-fi_K-0t4",
        "outputId": "c340ec88-8e6b-48cc-fa06-2edc1a5163c7"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: pyvi in /usr/local/lib/python3.10/dist-packages (0.1.1)\n",
            "Requirement already satisfied: underthesea in /usr/local/lib/python3.10/dist-packages (6.8.4)\n",
            "Requirement already satisfied: sentence-transformers in /usr/local/lib/python3.10/dist-packages (3.2.1)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from pyvi) (1.5.2)\n",
            "Requirement already satisfied: sklearn-crfsuite in /usr/local/lib/python3.10/dist-packages (from pyvi) (0.5.0)\n",
            "Requirement already satisfied: Click>=6.0 in /usr/local/lib/python3.10/dist-packages (from underthesea) (8.1.7)\n",
            "Requirement already satisfied: python-crfsuite>=0.9.6 in /usr/local/lib/python3.10/dist-packages (from underthesea) (0.9.11)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from underthesea) (3.8.1)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from underthesea) (4.66.5)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from underthesea) (2.32.3)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from underthesea) (1.4.2)\n",
            "Requirement already satisfied: PyYAML in /usr/local/lib/python3.10/dist-packages (from underthesea) (6.0.2)\n",
            "Requirement already satisfied: underthesea-core==1.0.4 in /usr/local/lib/python3.10/dist-packages (from underthesea) (1.0.4)\n",
            "Requirement already satisfied: transformers<5.0.0,>=4.41.0 in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (4.44.2)\n",
            "Requirement already satisfied: torch>=1.11.0 in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (2.5.0+cu121)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (1.13.1)\n",
            "Requirement already satisfied: huggingface-hub>=0.20.0 in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (0.24.7)\n",
            "Requirement already satisfied: Pillow in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (10.4.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.20.0->sentence-transformers) (3.16.1)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.20.0->sentence-transformers) (2024.6.1)\n",
            "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.20.0->sentence-transformers) (24.1)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.20.0->sentence-transformers) (4.12.2)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence-transformers) (3.4.2)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence-transformers) (3.1.4)\n",
            "Requirement already satisfied: sympy==1.13.1 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence-transformers) (1.13.1)\n",
            "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy==1.13.1->torch>=1.11.0->sentence-transformers) (1.3.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.41.0->sentence-transformers) (1.26.4)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.41.0->sentence-transformers) (2024.9.11)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.41.0->sentence-transformers) (0.4.5)\n",
            "Requirement already satisfied: tokenizers<0.20,>=0.19 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.41.0->sentence-transformers) (0.19.1)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->underthesea) (3.4.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->underthesea) (3.10)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->underthesea) (2.2.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->underthesea) (2024.8.30)\n",
            "Requirement already satisfied: threadpoolctl>=3.1.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->pyvi) (3.5.0)\n",
            "Requirement already satisfied: tabulate>=0.4.2 in /usr/local/lib/python3.10/dist-packages (from sklearn-crfsuite->pyvi) (0.9.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.11.0->sentence-transformers) (3.0.2)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import sys\n",
        "sys.path.append('/content/drive/MyDrive/Coding projects/')"
      ],
      "metadata": {
        "id": "5F4tbf-DlNtK"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Standard library imports\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import regex as re\n",
        "\n",
        "# Visualization imports\n",
        "import matplotlib.pyplot as plt\n",
        "import plotly.express as px\n",
        "import seaborn as sns\n",
        "\n",
        "# Natural Language Processing (NLP) imports\n",
        "import nltk\n",
        "from nltk.tokenize import word_tokenize\n",
        "from spacy.lang.vi import Vietnamese\n",
        "from underthesea import word_tokenize\n",
        "import underthesea\n",
        "from pyvi import ViTokenizer\n",
        "from pyvi.ViTokenizer import tokenize\n",
        "from sentence_transformers import SentenceTransformer\n",
        "from transformers import AutoModel, AutoTokenizer\n",
        "\n",
        "# Custom helper imports\n",
        "from VietnameseBiasDetectionHelper import VietnameseEmbedder\n",
        "from HindiBiasDetectionHelper import HindiEmbedder\n",
        "\n",
        "# Machine Learning imports\n",
        "from sklearn.cluster import KMeans\n",
        "from sklearn.decomposition import PCA\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.metrics import silhouette_score"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "r1fOf1G99vRi",
        "outputId": "75f4b12b-a7dc-4ba8-9f01-4d649fa72176"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:89: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n",
            "  warnings.warn(\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Vietnamese Benchmark for gender bias\n",
        "* [Estimating gender bias in sentence embeddings](https://www.politesi.polimi.it/retrieve/4e623e81-fd73-49f7-bc03-24482b923426/Estimating%20Gender%20Bias%20in%20Sentence%20Embeddings.pdf)\n",
        "\n",
        "* [SimeCSE_Vietnamese](https://github.com/vovanphuc/SimeCSE_Vietnamese)\n",
        "\n",
        "* [Sentence Embedding by BERT and Sentence Similarity\n",
        "t](https://peaceful0907.medium.com/sentence-embedding-by-bert-and-sentence-similarity-759f7beccbf1)\n",
        "\n",
        "* [How to Generate Word Embedding using BERT?](https://www.geeksforgeeks.org/how-to-generate-word-embedding-using-bert/)"
      ],
      "metadata": {
        "id": "0DjVGZPKEZ7B"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# gender_direction: single dimension comprising the majority of the gender semantics of the model\n",
        "gender_direction = [\n",
        "    [\"woman\",\"man\"],\n",
        "    [\"she\",\"he\"],\n",
        "    [\"daughter\",\"son\"],\n",
        "    [\"mother\",\"father\"],\n",
        "    [\"girl\",\"boy\"],\n",
        "    [\"gal\",\"guy\"],\n",
        "    [\"her\",\"his\"],\n",
        "    [\"hers\",\"his\"],\n",
        "    [\"female\",\"male\"],\n",
        "    [\"Mary\", \"John\"]\n",
        "]"
      ],
      "metadata": {
        "id": "Huz12qYmEllI"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Generate Gender Subspace"
      ],
      "metadata": {
        "id": "H8mG196UGXyi"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Vietnamese Gender Bias"
      ],
      "metadata": {
        "id": "K-azS4VBIk6U"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "vietnamese_embedder = VietnameseEmbedder()"
      ],
      "metadata": {
        "id": "2kDKtzFfGl6e"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sentence = \"Con trai mấy khi tỉnh táo khi yêu\"\n",
        "bias_score = vietnamese_embedder.get_gender_bias_score_of_sentence(sentence)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p9Y3YCj6Ov6p",
        "outputId": "9708a6fc-c554-4dbb-d975-954985ae066f"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Female bias score: 13.660793355666101%\n",
            "Male bias score: 3.137429803609848%\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Hindi Gender bias"
      ],
      "metadata": {
        "id": "Ui9W6AjVzCXi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "hindi_embedder = HindiEmbedder()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "X913mLiO5qTa",
        "outputId": "9ff5462a-0c61-4e7e-a502-ff1d11df3ba3"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sample_hindi_sentence = \"नमस्ते, आप कैसे हैं?\"\n",
        "bias_score = hindi_embedder.get_gender_bias_score_of_sentence(sample_hindi_sentence)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6LAyLc-WCueH",
        "outputId": "07c4b66b-0040-4752-8680-c1fe4eda4319"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Female bias score: 0%\n",
            "Male bias score: 12.3422060161829%\n"
          ]
        }
      ]
    }
  ]
}