00-indobart copy.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, sys\n",
    "import torch\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from datasets import load_dataset\n",
    "\n",
    "import nltk\n",
    "import evaluate\n",
    "from transformers import DataCollatorForSeq2Seq\n",
    "from transformers import AutoModelForSeq2SeqLM, Seq2SeqTrainingArguments, Seq2SeqTrainer\n",
    "from transformers import pipeline\n",
    "\n",
    "from repo.indobenchmark.toolkit.tokenization_indonlg import IndoNLGTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "using cuda\n"
     ]
    }
   ],
   "source": [
    "# check if cuda or mps available, if available, use one of them, otherwise use cpu\n",
    "\n",
    "device = torch.device(\"cpu\")\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "    device = torch.device(\"cuda\")\n",
    "    print(\"using cuda\")\n",
    "elif torch.backends.mps.is_available():\n",
    "    device = torch.device(\"mps\")\n",
    "    os.environ[\"PYTORCH_MPS_HIGH_WATERMARK_RATIO\"] = (\n",
    "        \"1\"  # This is tracked as pytorch issue #98222\n",
    "    )\n",
    "    print(\"using mps\")\n",
    "else:\n",
    "    device = torch.device(\"cpu\")\n",
    "    print(\"using cpu\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specify the directory and file path\n",
    "benc_directory = \"benc_result/00-indobart/\"\n",
    "\n",
    "# Create the benc_directory if it doesn't exist\n",
    "os.makedirs(benc_directory, exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>document</th>\n",
       "      <th>id</th>\n",
       "      <th>summary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Jakarta, CNN Indonesia - - Dokter Ryan Thamrin...</td>\n",
       "      <td>1501893029-lula-kamal-dokter-ryan-thamrin-saki...</td>\n",
       "      <td>Dokter Lula Kamal yang merupakan selebriti sek...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Selfie ialah salah satu tema terpanas di kalan...</td>\n",
       "      <td>1509072914-dua-smartphone-zenfone-baru-tawarka...</td>\n",
       "      <td>Asus memperkenalkan   ZenFone generasi keempat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Jakarta, CNN Indonesia - - Dinas Pariwisata Pr...</td>\n",
       "      <td>1510613677-songsong-visit-2020-bengkulu-perkua...</td>\n",
       "      <td>Dinas Pariwisata Provinsi Bengkulu kembali men...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Merdeka.com - Indonesia Corruption Watch (ICW)...</td>\n",
       "      <td>1502706803-icw-ada-kejanggalan-atas-tewasnya-s...</td>\n",
       "      <td>Indonesia Corruption Watch (ICW) meminta Komis...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Merdeka.com - Presiden Joko Widodo (Jokowi) me...</td>\n",
       "      <td>1503039338-pembagian-sepeda-usai-upacara-penur...</td>\n",
       "      <td>Jokowi memimpin upacara penurunan bendera. Usa...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            document  \\\n",
       "0  Jakarta, CNN Indonesia - - Dokter Ryan Thamrin...   \n",
       "1  Selfie ialah salah satu tema terpanas di kalan...   \n",
       "2  Jakarta, CNN Indonesia - - Dinas Pariwisata Pr...   \n",
       "3  Merdeka.com - Indonesia Corruption Watch (ICW)...   \n",
       "4  Merdeka.com - Presiden Joko Widodo (Jokowi) me...   \n",
       "\n",
       "                                                  id  \\\n",
       "0  1501893029-lula-kamal-dokter-ryan-thamrin-saki...   \n",
       "1  1509072914-dua-smartphone-zenfone-baru-tawarka...   \n",
       "2  1510613677-songsong-visit-2020-bengkulu-perkua...   \n",
       "3  1502706803-icw-ada-kejanggalan-atas-tewasnya-s...   \n",
       "4  1503039338-pembagian-sepeda-usai-upacara-penur...   \n",
       "\n",
       "                                             summary  \n",
       "0  Dokter Lula Kamal yang merupakan selebriti sek...  \n",
       "1  Asus memperkenalkan   ZenFone generasi keempat...  \n",
       "2  Dinas Pariwisata Provinsi Bengkulu kembali men...  \n",
       "3  Indonesia Corruption Watch (ICW) meminta Komis...  \n",
       "4  Jokowi memimpin upacara penurunan bendera. Usa...  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ds = load_dataset('maryantocinn/indosum')\n",
    "ds = load_dataset(\"./repo/SEACrowd/indosum/indosum.py\")\n",
    "\n",
    "# show first 5 data from the dataset in pandas like table\n",
    "pd.DataFrame(ds[\"train\"][:5]).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train dataset length:  14262\n",
      "Validation dataset length:  750\n",
      "Test dataset length:  3762\n"
     ]
    }
   ],
   "source": [
    "# check the length of the dataset\n",
    "print(\"Train dataset length: \", len(ds[\"train\"]))\n",
    "print(\"Validation dataset length: \", len(ds[\"validation\"]))\n",
    "print(\"Test dataset length: \", len(ds[\"test\"]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# bart_model = MBartForConditionalGeneration.from_pretrained('indobenchmark/indobart-v2')\n",
    "\n",
    "bart_model = AutoModelForSeq2SeqLM.from_pretrained(\"indobenchmark/indobart-v2\")\n",
    "indonlg_tokenizer = IndoNLGTokenizer.from_pretrained(\"indobenchmark/indobart-v2\")\n",
    "\n",
    "model = bart_model\n",
    "tokenizer = indonlg_tokenizer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using auto half precision backend\n"
     ]
    }
   ],
   "source": [
    "# Prepare and tokenize dataset\n",
    "def preprocess_function(examples):\n",
    "    model_inputs = tokenizer(examples[\"document\"], max_length=768, truncation=True)\n",
    "\n",
    "    labels = tokenizer(text_target=examples[\"summary\"], max_length=128, truncation=True)\n",
    "\n",
    "    model_inputs[\"labels\"] = labels[\"input_ids\"]\n",
    "    return model_inputs\n",
    "\n",
    "\n",
    "# Setup evaluation\n",
    "nltk.download(\"punkt_tab\", quiet=True)\n",
    "metric = evaluate.load(\"rouge\")\n",
    "\n",
    "\n",
    "def compute_metrics(eval_preds):\n",
    "    preds, labels = eval_preds\n",
    "\n",
    "    # decode preds and labels\n",
    "    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n",
    "    decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)\n",
    "    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n",
    "\n",
    "    # rougeLSum expects newline after each sentence\n",
    "    decoded_preds = [\n",
    "        \"\\n\".join(nltk.sent_tokenize(pred.strip())) for pred in decoded_preds\n",
    "    ]\n",
    "    decoded_labels = [\n",
    "        \"\\n\".join(nltk.sent_tokenize(label.strip())) for label in decoded_labels\n",
    "    ]\n",
    "\n",
    "    result = metric.compute(\n",
    "        predictions=decoded_preds, references=decoded_labels, use_stemmer=True\n",
    "    )\n",
    "    return result\n",
    "\n",
    "\n",
    "tokenized_ds = ds.map(preprocess_function, batched=True)\n",
    "\n",
    "# Load pretrained model and evaluate model after each epoch\n",
    "data_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=model)\n",
    "\n",
    "per_device_batch_size = 8  # 8 for low hardware spec\n",
    "output_dir = \"./results/00-indobart\"\n",
    "\n",
    "training_args = Seq2SeqTrainingArguments(\n",
    "    output_dir=output_dir,\n",
    "    # overwrite_output_dir=True,\n",
    "    eval_strategy=\"epoch\",\n",
    "    learning_rate=3.75e-5,  # hf example: 2e-5\n",
    "    per_device_train_batch_size=per_device_batch_size,\n",
    "    per_device_eval_batch_size=per_device_batch_size,\n",
    "    weight_decay=0.01,\n",
    "    save_steps=1000,\n",
    "    save_total_limit=2,\n",
    "    num_train_epochs=3,  # hf example: 2\n",
    "    fp16=True,  # comment this if using mps/apple sillicon chip (not supported)\n",
    "    predict_with_generate=True,\n",
    "    generation_max_length=80,\n",
    "    log_level=\"info\",\n",
    "    logging_first_step=True,\n",
    "    resume_from_checkpoint=True,\n",
    ")\n",
    "\n",
    "trainer = Seq2SeqTrainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=tokenized_ds[\"train\"],\n",
    "    eval_dataset=tokenized_ds[\"validation\"],\n",
    "    processing_class=tokenizer,  # FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Seq2SeqTrainer.__init__`.\n",
    "    data_collator=data_collator,\n",
    "    compute_metrics=compute_metrics,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the training set don't have a corresponding argument in `MBartForConditionalGeneration.forward` and have been ignored: id, document, summary. If id, document, summary are not expected by `MBartForConditionalGeneration.forward`,  you can safely ignore this message.\n",
      "***** Running training *****\n",
      "  Num examples = 14,262\n",
      "  Num Epochs = 3\n",
      "  Instantaneous batch size per device = 8\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 8\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 5,349\n",
      "  Number of trainable parameters = 131,543,040\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='5349' max='5349' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [5349/5349 21:57, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Rouge1</th>\n",
       "      <th>Rouge2</th>\n",
       "      <th>Rougel</th>\n",
       "      <th>Rougelsum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.554700</td>\n",
       "      <td>0.518261</td>\n",
       "      <td>0.693098</td>\n",
       "      <td>0.615000</td>\n",
       "      <td>0.659614</td>\n",
       "      <td>0.684784</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.422500</td>\n",
       "      <td>0.501937</td>\n",
       "      <td>0.686966</td>\n",
       "      <td>0.607920</td>\n",
       "      <td>0.654286</td>\n",
       "      <td>0.678254</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.334300</td>\n",
       "      <td>0.509772</td>\n",
       "      <td>0.687822</td>\n",
       "      <td>0.608429</td>\n",
       "      <td>0.654800</td>\n",
       "      <td>0.679702</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to ./results/00-indobart/checkpoint-1000\n",
      "/home/paperspace/thesis-binus/.venv/lib/python3.9/site-packages/transformers/modeling_utils.py:2817: UserWarning: Moving the following attributes in the config to the generation config: {'early_stopping': True, 'num_beams': 4, 'no_repeat_ngram_size': 3}. You are seeing this warning because you've set generation parameters in the model config, as opposed to in the generation config.\n",
      "  warnings.warn(\n",
      "Configuration saved in ./results/00-indobart/checkpoint-1000/config.json\n",
      "Configuration saved in ./results/00-indobart/checkpoint-1000/generation_config.json\n",
      "Model weights saved in ./results/00-indobart/checkpoint-1000/model.safetensors\n",
      "tokenizer config file saved in ./results/00-indobart/checkpoint-1000/tokenizer_config.json\n",
      "Special tokens file saved in ./results/00-indobart/checkpoint-1000/special_tokens_map.json\n",
      "The following columns in the evaluation set don't have a corresponding argument in `MBartForConditionalGeneration.forward` and have been ignored: id, document, summary. If id, document, summary are not expected by `MBartForConditionalGeneration.forward`,  you can safely ignore this message.\n",
      "\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 750\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./results/00-indobart/checkpoint-2000\n",
      "Configuration saved in ./results/00-indobart/checkpoint-2000/config.json\n",
      "Configuration saved in ./results/00-indobart/checkpoint-2000/generation_config.json\n",
      "Model weights saved in ./results/00-indobart/checkpoint-2000/model.safetensors\n",
      "tokenizer config file saved in ./results/00-indobart/checkpoint-2000/tokenizer_config.json\n",
      "Special tokens file saved in ./results/00-indobart/checkpoint-2000/special_tokens_map.json\n",
      "Saving model checkpoint to ./results/00-indobart/checkpoint-3000\n",
      "Configuration saved in ./results/00-indobart/checkpoint-3000/config.json\n",
      "Configuration saved in ./results/00-indobart/checkpoint-3000/generation_config.json\n",
      "Model weights saved in ./results/00-indobart/checkpoint-3000/model.safetensors\n",
      "tokenizer config file saved in ./results/00-indobart/checkpoint-3000/tokenizer_config.json\n",
      "Special tokens file saved in ./results/00-indobart/checkpoint-3000/special_tokens_map.json\n",
      "Deleting older checkpoint [results/00-indobart/checkpoint-1000] due to args.save_total_limit\n",
      "The following columns in the evaluation set don't have a corresponding argument in `MBartForConditionalGeneration.forward` and have been ignored: id, document, summary. If id, document, summary are not expected by `MBartForConditionalGeneration.forward`,  you can safely ignore this message.\n",
      "\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 750\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to ./results/00-indobart/checkpoint-4000\n",
      "Configuration saved in ./results/00-indobart/checkpoint-4000/config.json\n",
      "Configuration saved in ./results/00-indobart/checkpoint-4000/generation_config.json\n",
      "Model weights saved in ./results/00-indobart/checkpoint-4000/model.safetensors\n",
      "tokenizer config file saved in ./results/00-indobart/checkpoint-4000/tokenizer_config.json\n",
      "Special tokens file saved in ./results/00-indobart/checkpoint-4000/special_tokens_map.json\n",
      "Deleting older checkpoint [results/00-indobart/checkpoint-2000] due to args.save_total_limit\n",
      "Saving model checkpoint to ./results/00-indobart/checkpoint-5000\n",
      "Configuration saved in ./results/00-indobart/checkpoint-5000/config.json\n",
      "Configuration saved in ./results/00-indobart/checkpoint-5000/generation_config.json\n",
      "Model weights saved in ./results/00-indobart/checkpoint-5000/model.safetensors\n",
      "tokenizer config file saved in ./results/00-indobart/checkpoint-5000/tokenizer_config.json\n",
      "Special tokens file saved in ./results/00-indobart/checkpoint-5000/special_tokens_map.json\n",
      "Deleting older checkpoint [results/00-indobart/checkpoint-3000] due to args.save_total_limit\n",
      "Saving model checkpoint to ./results/00-indobart/checkpoint-5349\n",
      "Configuration saved in ./results/00-indobart/checkpoint-5349/config.json\n",
      "Configuration saved in ./results/00-indobart/checkpoint-5349/generation_config.json\n",
      "Model weights saved in ./results/00-indobart/checkpoint-5349/model.safetensors\n",
      "tokenizer config file saved in ./results/00-indobart/checkpoint-5349/tokenizer_config.json\n",
      "Special tokens file saved in ./results/00-indobart/checkpoint-5349/special_tokens_map.json\n",
      "Deleting older checkpoint [results/00-indobart/checkpoint-4000] due to args.save_total_limit\n",
      "The following columns in the evaluation set don't have a corresponding argument in `MBartForConditionalGeneration.forward` and have been ignored: id, document, summary. If id, document, summary are not expected by `MBartForConditionalGeneration.forward`,  you can safely ignore this message.\n",
      "\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 750\n",
      "  Batch size = 8\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=5349, training_loss=0.4577951965030721, metrics={'train_runtime': 1317.9973, 'train_samples_per_second': 32.463, 'train_steps_per_second': 4.058, 'total_flos': 1.5555268214194176e+16, 'train_loss': 0.4577951965030721, 'epoch': 3.0})"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predict Test Data and evaluate the score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the test set don't have a corresponding argument in `MBartForConditionalGeneration.forward` and have been ignored: id, document, summary. If id, document, summary are not expected by `MBartForConditionalGeneration.forward`,  you can safely ignore this message.\n",
      "\n",
      "***** Running Prediction *****\n",
      "  Num examples = 3762\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ROUGE scores on the test set: {'rouge1': np.float64(0.6808638082877687), 'rouge2': np.float64(0.6014193813641668), 'rougeL': np.float64(0.6458487620938582), 'rougeLsum': np.float64(0.6719528217041734)}\n"
     ]
    }
   ],
   "source": [
    "# Generate predictions\n",
    "test_predictions = trainer.predict(tokenized_ds[\"test\"])\n",
    "\n",
    "# Get the predictions and labels from the result\n",
    "preds = test_predictions.predictions\n",
    "labels = test_predictions.label_ids\n",
    "\n",
    "# Evaluate using the compute_metrics function\n",
    "rouge_scores = compute_metrics((preds, labels))\n",
    "\n",
    "# Print the ROUGE scores\n",
    "print(\"ROUGE scores on the test set:\", rouge_scores)\n",
    "\n",
    "# save the results to a file 'rouge_scores.txt'\n",
    "with open(os.path.join(benc_directory, \"rouge_scores.txt\"), \"w\") as f:\n",
    "    f.write(str(rouge_scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pipeline summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "vscode": {
     "languageId": "ruby"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You seem to be using the pipelines sequentially on GPU. In order to maximize efficiency please use a dataset\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>document</th>\n",
       "      <th>summary</th>\n",
       "      <th>generated_summary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Jakarta, CNN Indonesia - - Dilansir AFP, seora...</td>\n",
       "      <td>Eman Ahmed Abd El Aty memiliki berat badan men...</td>\n",
       "      <td>seorang warga mesir yang dipercaya sebagai wa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Menteri Pertahanan Ryamizard Ryacudu menyambut...</td>\n",
       "      <td>Menteri Pertahanan Ryamizard Ryacudu menyambut...</td>\n",
       "      <td>menteri pertahanan ryamizard ryacudu menyambu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Jakarta, CNN Indonesia - - Meski sudah hampir ...</td>\n",
       "      <td>Rumah produksi film yang dibintangi Lindsay Lo...</td>\n",
       "      <td>meski sudah hampir 12 tahun berlalu, film mea...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Usai melaksanakan ibadah haji, Eggi Sudjana ak...</td>\n",
       "      <td>Eggi Sudjana akhirnya mendatangi kantor Baresk...</td>\n",
       "      <td>setelah melaksanakan ibadah haji, eggi sudjan...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Banyak cara untuk memberikan pengajaran kepada...</td>\n",
       "      <td>Game permainan Kartu Muslim. Menggunakan basis...</td>\n",
       "      <td>terdapat cara untuk memberikan pengajaran kep...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            document  \\\n",
       "0  Jakarta, CNN Indonesia - - Dilansir AFP, seora...   \n",
       "1  Menteri Pertahanan Ryamizard Ryacudu menyambut...   \n",
       "2  Jakarta, CNN Indonesia - - Meski sudah hampir ...   \n",
       "3  Usai melaksanakan ibadah haji, Eggi Sudjana ak...   \n",
       "4  Banyak cara untuk memberikan pengajaran kepada...   \n",
       "\n",
       "                                             summary  \\\n",
       "0  Eman Ahmed Abd El Aty memiliki berat badan men...   \n",
       "1  Menteri Pertahanan Ryamizard Ryacudu menyambut...   \n",
       "2  Rumah produksi film yang dibintangi Lindsay Lo...   \n",
       "3  Eggi Sudjana akhirnya mendatangi kantor Baresk...   \n",
       "4  Game permainan Kartu Muslim. Menggunakan basis...   \n",
       "\n",
       "                                   generated_summary  \n",
       "0   seorang warga mesir yang dipercaya sebagai wa...  \n",
       "1   menteri pertahanan ryamizard ryacudu menyambu...  \n",
       "2   meski sudah hampir 12 tahun berlalu, film mea...  \n",
       "3   setelah melaksanakan ibadah haji, eggi sudjan...  \n",
       "4   terdapat cara untuk memberikan pengajaran kep...  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get device\n",
    "summarizer = pipeline(\"summarization\", model=model, tokenizer=tokenizer, device=device)\n",
    "\n",
    "# create table to show the result: document, summary, generated_summary\n",
    "df = pd.DataFrame(columns=['document', 'summary', 'generated_summary'])\n",
    "for i in range(100):\n",
    "    document = ds['test'][i]['document']\n",
    "    summary = ds['test'][i]['summary']\n",
    "    generated_summary = summarizer(document, min_length=5, max_length=80)\n",
    "    df = pd.concat([df, pd.DataFrame([[document, summary, generated_summary[0]['summary_text']]], columns=['document', 'summary', 'generated_summary'])], ignore_index=True)\n",
    "\n",
    "\n",
    "# Save the DataFrame to a CSV file\n",
    "df.to_csv(f'{benc_directory}/summarization_result.csv')\n",
    "df.to_json(f'{benc_directory}/summarization_result.json')\n",
    "\n",
    "df.head()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}