Embedding(pickle).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "mount_file_id": "101pTX_OZBx48Q98i7BSBaS67Ncv3H7vS",
      "authorship_tag": "ABX9TyO/ZOV5L1P20WJdxOnfNDa5",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/hamin-shim/Prometheus_Cooker/blob/Embedding_model/Embedding(pickle).ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EUrYaRHL4Cr1",
        "outputId": "ca740663-6bb7-4b5b-a8dd-6654e46d2d2c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting openai\n",
            "  Downloading openai-1.35.14-py3-none-any.whl (328 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m328.5/328.5 kB\u001b[0m \u001b[31m3.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai) (3.7.1)\n",
            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai) (1.7.0)\n",
            "Collecting httpx<1,>=0.23.0 (from openai)\n",
            "  Downloading httpx-0.27.0-py3-none-any.whl (75 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m2.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pydantic<3,>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from openai) (2.8.2)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai) (1.3.1)\n",
            "Requirement already satisfied: tqdm>4 in /usr/local/lib/python3.10/dist-packages (from openai) (4.66.4)\n",
            "Requirement already satisfied: typing-extensions<5,>=4.7 in /usr/local/lib/python3.10/dist-packages (from openai) (4.12.2)\n",
            "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai) (3.7)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai) (1.2.1)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai) (2024.7.4)\n",
            "Collecting httpcore==1.* (from httpx<1,>=0.23.0->openai)\n",
            "  Downloading httpcore-1.0.5-py3-none-any.whl (77 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m4.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting h11<0.15,>=0.13 (from httpcore==1.*->httpx<1,>=0.23.0->openai)\n",
            "  Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m2.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai) (0.7.0)\n",
            "Requirement already satisfied: pydantic-core==2.20.1 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai) (2.20.1)\n",
            "Installing collected packages: h11, httpcore, httpx, openai\n",
            "Successfully installed h11-0.14.0 httpcore-1.0.5 httpx-0.27.0 openai-1.35.14\n"
          ]
        }
      ],
      "source": [
        "!pip install openai"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 필요한 라이브러리 임포트\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import torch\n",
        "from openai import OpenAI\n",
        "import torch.nn.functional as F\n",
        "import pickle\n",
        "import os\n",
        "from google.colab import drive\n",
        "import shutil\n",
        "\n",
        "# Google Drive를 Colab에 마운트\n",
        "drive.mount('/content/drive')\n",
        "\n",
        "# OpenAI 클라이언트 설정\n",
        "client = OpenAI(\n",
        "    api_key=\"up_bsQetBVyFTYszX7zw1tOHrU0RC9bm\",  # 실제 API 키로 교체해야 합니다\n",
        "    base_url=\"https://api.upstage.ai/v1/solar\"\n",
        ")\n",
        "\n",
        "# 데이터 로딩\n",
        "df = pd.read_excel('/content/drive/MyDrive/Colab Notebooks/Prometheus_Cooker/sampled.xlsx')\n",
        "\n",
        "# 결측값 처리\n",
        "df['DATA'] = df['DATA'].fillna('')\n",
        "\n",
        "# label이 1인 데이터 필터링\n",
        "filtered_df = df[df['TARGET'] == 1]\n",
        "\n",
        "# 문장과 메뉴 이름을 리스트로 변환\n",
        "texts = filtered_df['DATA'].tolist()\n",
        "menu_names = filtered_df['MENU_NAME'].tolist()  # MENU_NAME 컬럼 추가\n",
        "\n",
        "# 임베딩 저장 및 불러오기 경로\n",
        "embedding_file = '/content/embeddings.pkl'  # Colab의 임시 디렉토리 경로\n",
        "destination_dir = '/content/drive/MyDrive/dataset'  # Google Drive의 MyDrive 폴더 경로\n",
        "destination_file = destination_dir + 'embeddings.pkl'\n",
        "\n",
        "# 임베딩 생성 함수\n",
        "def get_embeddings(texts, batch_size=100):\n",
        "    embeddings = []\n",
        "    for i in range(0, len(texts), batch_size):\n",
        "        batch_texts = texts[i:i+batch_size]\n",
        "        response = client.embeddings.create(\n",
        "            input=batch_texts,\n",
        "            model=\"solar-embedding-1-large-query\"\n",
        "        )\n",
        "        batch_embeddings = [np.array(embedding.embedding) for embedding in response.data]\n",
        "        embeddings.extend(batch_embeddings)\n",
        "    return embeddings\n",
        "\n",
        "# 임베딩이 저장되어 있는지 확인하고, 저장된 경우 불러오기\n",
        "if os.path.exists(embedding_file):\n",
        "    print(\"임베딩 파일을 불러오는 중...\")\n",
        "    with open(embedding_file, 'rb') as file:\n",
        "        embeddings = pickle.load(file)\n",
        "else:\n",
        "    print(\"임베딩 생성 중...\")\n",
        "    embeddings = get_embeddings(texts)\n",
        "    with open(embedding_file, 'wb') as file:\n",
        "        pickle.dump(embeddings, file)\n",
        "\n",
        "    # 파일을 Google Drive로 복사\n",
        "    shutil.copy(embedding_file, destination_file)\n",
        "    print(f\"임베딩 파일이 Google Drive에 저장되었습니다: {destination_file}\")\n",
        "\n",
        "# 사용자 입력 문장\n",
        "queries = [\"달콤하면서도 약간의 신맛이 조화를 이루어, 마치 여름 과일을 한입 베어 문 듯한 상쾌함을 느낄 수 있는 음식 추천해줘\"]\n",
        "\n",
        "# 코사인 유사도를 기준으로 입력 문장과 가장 유사한 5개 음식 추출\n",
        "top_k = 5\n",
        "for query in queries:\n",
        "    query_embedding = get_embeddings([query])[0]\n",
        "    query_embedding = torch.tensor(query_embedding)\n",
        "\n",
        "    # 코사인 유사도 계산\n",
        "    embeddings_tensor = torch.tensor(embeddings)\n",
        "    cos_scores = F.cosine_similarity(query_embedding.unsqueeze(0), embeddings_tensor, dim=1)\n",
        "\n",
        "    # 상위 top_k 결과 추출\n",
        "    top_results = torch.topk(cos_scores, k=top_k)\n",
        "\n",
        "    print(\"\\n\\n======================\\n\\n\")\n",
        "    print(\"사용자 입력 :\", query)\n",
        "    print(\"\\nTop 5 음식 추천:\")\n",
        "\n",
        "    for idx in top_results.indices:\n",
        "        print(menu_names[idx].strip(),\" : \",texts[idx].strip(),  \"(Score: %.4f)\" % (cos_scores[idx].item()))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LEluPZDA4Dln",
        "outputId": "54972ca8-3e7a-4b2c-c833-917fb4af4271"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n",
            "임베딩 파일을 불러오는 중...\n",
            "\n",
            "\n",
            "======================\n",
            "\n",
            "\n",
            "사용자 입력 : 달콤하면서도 약간의 신맛이 조화를 이루어, 마치 여름 과일을 한입 베어 문 듯한 상쾌함을 느낄 수 있는 음식 추천해줘\n",
            "\n",
            "Top 5 음식 추천:\n",
            "토마토샐러드  :  떠먹는 토마토 샐러드는 토마토의 신선함과 소스의 달콤함이 너무 맛있답니다 (Score: 0.5817)\n",
            "사과양배추샐러드  :  유튜브에서 보고 따라해봤어요.  상큼한게 맛있어요. (Score: 0.5642)\n",
            "뚝배기배추국  :  얼큰하고 시원~한 맛이에요 (Score: 0.5439)\n",
            "복숭아멘보샤  :  달콤한 데일리 복숭아가 탱글탱글 새우를 만나 튀김으로 하나가 되었다! 달콤 바삭한 복숭아 멘보샤  자칫하면 느끼할 수 있는 튀김요리의 신의 한 수  '새콤달콤 복숭아칠리소스와 팬에 구워 달콤함이 배가된 복숭아'를 곁들여 먹는 맛난 간식! 한입 크기로 파티 핑거푸드로도 좋아요~ (Score: 0.5418)\n",
            "찜닭  :  달달한 간장양념이 가미된 찜닭요리를 해 봤어요! :) 여름보양식으로 딱이예요. (Score: 0.5381)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "l9M6sMfBK4bR"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}