chinese_alpaca_2_13b_finetuning_100k.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ca98b0a9-08fe-40cd-92e5-7368bf16a421",
   "metadata": {},
   "source": [
    "## Finetune chinese-llama/alpaca for couplet writing\n",
    "\n",
    "In this notebook, we are going to use 100k Chinese couplet data to finetune the Chinese llama/alpaca LLM for couplet writing!"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3bb79063-5891-4e09-8ba1-9237a960225c",
   "metadata": {},
   "source": [
    "### 1. Load Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10c97803-fd94-472b-9ba7-877843df4d9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prerequisites\n",
    "#!pip install -q accelerate==0.21.0 peft==0.4.0 bitsandbytes==0.40.2 trl==0.4.7 transformers[sentencepiece]==4.31.0 scipy tensorboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2876d039-584f-4de6-88cf-560c48c1da84",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import torch\n",
    "from datasets import load_dataset\n",
    "from transformers import (\n",
    "    AutoModelForCausalLM,\n",
    "    AutoTokenizer,\n",
    "    BitsAndBytesConfig,\n",
    "    GenerationConfig,\n",
    "    HfArgumentParser,\n",
    "    LlamaTokenizer,\n",
    "    TrainingArguments,\n",
    "    pipeline,\n",
    "    logging,\n",
    ")\n",
    "from peft import LoraConfig, PeftModel\n",
    "from trl import SFTTrainer\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa442783-2f6a-42a3-800f-ce4c7e0f8068",
   "metadata": {},
   "outputs": [],
   "source": [
    "if torch.cuda.is_available():\n",
    "    device = torch.device(0)\n",
    "else:\n",
    "    device = torch.device('cpu')\n",
    "\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "caad414f-477b-41da-82fe-cd259e739557",
   "metadata": {},
   "source": [
    "### 2. Set up model and finetuning parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dfa038a0-b028-4688-b6bf-921712581b11",
   "metadata": {},
   "outputs": [],
   "source": [
    "# The model that you want to train from the Hugging Face hub\n",
    "model_name = \"ziqingyang/chinese-alpaca-2-13b\"\n",
    "\n",
    "# The instruction dataset to use\n",
    "dataset_name = \"chenqile09/llama2-chinese-couplet-100k\"\n",
    "\n",
    "# Fine-tuned model name\n",
    "new_model = \"chinese-alpaca-2-LoRA-13B-couplet-100k\"\n",
    "\n",
    "################################################################################\n",
    "# QLoRA parameters\n",
    "################################################################################\n",
    "\n",
    "# LoRA attention dimension\n",
    "lora_r = 64\n",
    "\n",
    "# Alpha parameter for LoRA scaling\n",
    "lora_alpha = 16\n",
    "\n",
    "# Dropout probability for LoRA layers\n",
    "lora_dropout = 0.1\n",
    "\n",
    "################################################################################\n",
    "# bitsandbytes parameters\n",
    "################################################################################\n",
    "\n",
    "# Activate 4-bit precision base model loading\n",
    "use_4bit = True\n",
    "\n",
    "# Compute dtype for 4-bit base models\n",
    "bnb_4bit_compute_dtype = \"float16\"\n",
    "\n",
    "# Quantization type (fp4 or nf4)\n",
    "bnb_4bit_quant_type = \"nf4\"\n",
    "\n",
    "# Activate nested quantization for 4-bit base models (double quantization)\n",
    "use_nested_quant = False\n",
    "\n",
    "################################################################################\n",
    "# TrainingArguments parameters\n",
    "################################################################################\n",
    "\n",
    "# Output directory where the model predictions and checkpoints will be stored\n",
    "output_dir = \"./results\"\n",
    "\n",
    "# Number of training epochs\n",
    "num_train_epochs = 1\n",
    "\n",
    "# Enable fp16/bf16 training (set bf16 to True with an A100)\n",
    "fp16 = False\n",
    "bf16 = True\n",
    "\n",
    "# Batch size per GPU for training\n",
    "per_device_train_batch_size = 32\n",
    "\n",
    "# Batch size per GPU for evaluation\n",
    "per_device_eval_batch_size = 32\n",
    "\n",
    "# Number of update steps to accumulate the gradients for\n",
    "gradient_accumulation_steps = 1\n",
    "\n",
    "# Enable gradient checkpointing\n",
    "gradient_checkpointing = True\n",
    "\n",
    "# Maximum gradient normal (gradient clipping)\n",
    "max_grad_norm = 0.3\n",
    "\n",
    "# Initial learning rate (AdamW optimizer)\n",
    "learning_rate = 2e-4\n",
    "\n",
    "# Weight decay to apply to all layers except bias/LayerNorm weights\n",
    "weight_decay = 0.001\n",
    "\n",
    "# Optimizer to use\n",
    "optim = \"paged_adamw_32bit\"\n",
    "\n",
    "# Learning rate schedule (constant a bit better than cosine)\n",
    "lr_scheduler_type = \"constant\"\n",
    "\n",
    "# Number of training steps (overrides num_train_epochs)\n",
    "max_steps = -1\n",
    "\n",
    "# Ratio of steps for a linear warmup (from 0 to learning rate)\n",
    "warmup_ratio = 0.03\n",
    "\n",
    "# Group sequences into batches with same length\n",
    "# Saves memory and speeds up training considerably\n",
    "group_by_length = True\n",
    "\n",
    "# Save checkpoint every X updates steps\n",
    "save_steps = 1000\n",
    "\n",
    "# Log every X updates steps\n",
    "logging_steps = 50\n",
    "\n",
    "################################################################################\n",
    "# SFT parameters\n",
    "################################################################################\n",
    "\n",
    "# Maximum sequence length to use\n",
    "max_seq_length = None\n",
    "\n",
    "# Pack multiple short examples in the same input sequence to increase efficiency\n",
    "packing = False\n",
    "\n",
    "# Load the entire model on the GPU 0\n",
    "device_map = {\"\": 0}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "abd39608-c95e-46e2-8963-72cbc7f9dce9",
   "metadata": {},
   "source": [
    "### 3. Load dataset, tokenizer, and base model from Hugging Face"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9475e29-3a27-42f5-9903-3dfc4704c9de",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load dataset (you can process it here)\n",
    "dataset = load_dataset(dataset_name, split=\"train\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e3b669e-a776-443b-b443-4dab04c301f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load tokenizer\n",
    "\n",
    "tokenizer = LlamaTokenizer.from_pretrained(model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0ab286f6-befb-4467-9771-18e36887f40b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load tokenizer and model with QLoRA configuration\n",
    "compute_dtype = getattr(torch, bnb_4bit_compute_dtype)\n",
    "\n",
    "print(compute_dtype)\n",
    "\n",
    "bnb_config = BitsAndBytesConfig(\n",
    "    load_in_4bit=use_4bit,\n",
    "    bnb_4bit_quant_type=bnb_4bit_quant_type,\n",
    "    bnb_4bit_compute_dtype=compute_dtype,\n",
    "    bnb_4bit_use_double_quant=use_nested_quant,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8e44a49c-a427-4cf9-a746-e0222297e71b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check GPU compatibility with bfloat16\n",
    "if compute_dtype == torch.float16 and use_4bit:\n",
    "    major, _ = torch.cuda.get_device_capability()\n",
    "    if major >= 8:\n",
    "        print(\"=\" * 80)\n",
    "        print(\"Your GPU supports bfloat16: accelerate training with bf16=True\")\n",
    "        print(\"=\" * 80)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ab8dc6d5-22c3-4e4b-882d-d96335c8f012",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load base model\n",
    "model = AutoModelForCausalLM.from_pretrained(\n",
    "    model_name,\n",
    "    quantization_config=bnb_config,\n",
    "    device_map=device_map\n",
    ")\n",
    "model.config.use_cache = False\n",
    "model.config.pretraining_tp = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a7fcb8ee-99f7-4458-a71c-260f8d583ae4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# check GPU memory usage (optional)\n",
    "\n",
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f20b7a1a-f3ae-4baa-866b-c023c3c190a3",
   "metadata": {},
   "source": [
    "### 4. Load finetuning parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5492413c-be50-4e21-8fb1-d2051849dc87",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load LoRA configuration\n",
    "peft_config = LoraConfig(\n",
    "    lora_alpha=lora_alpha,\n",
    "    lora_dropout=lora_dropout,\n",
    "    r=lora_r,\n",
    "    bias=\"none\",\n",
    "    task_type=\"CAUSAL_LM\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "91c52e59-860d-4152-b7f6-db017582fcfe",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set training parameters\n",
    "training_arguments = TrainingArguments(\n",
    "    output_dir=output_dir,\n",
    "    num_train_epochs=num_train_epochs,\n",
    "    per_device_train_batch_size=per_device_train_batch_size,\n",
    "    gradient_accumulation_steps=gradient_accumulation_steps,\n",
    "    optim=optim,\n",
    "    save_steps=save_steps,\n",
    "    logging_steps=logging_steps,\n",
    "    learning_rate=learning_rate,\n",
    "    weight_decay=weight_decay,\n",
    "    fp16=fp16,\n",
    "    bf16=bf16,\n",
    "    max_grad_norm=max_grad_norm,\n",
    "    max_steps=max_steps,\n",
    "    warmup_ratio=warmup_ratio,\n",
    "    group_by_length=group_by_length,\n",
    "    lr_scheduler_type=lr_scheduler_type,\n",
    "    report_to=\"tensorboard\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a518c79c-e016-419a-bcd8-fd10c6cc376a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set supervised fine-tuning parameters\n",
    "trainer = SFTTrainer(\n",
    "    model=model,\n",
    "    train_dataset=dataset,\n",
    "    peft_config=peft_config,\n",
    "    dataset_text_field=\"text\",\n",
    "    max_seq_length=max_seq_length,\n",
    "    tokenizer=tokenizer,\n",
    "    args=training_arguments,\n",
    "    packing=packing,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "696e2f05-af28-47fb-95e0-d498ff1dbb2e",
   "metadata": {},
   "source": [
    "### 5. Finetune the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2849d0c4-e6ce-404c-b8cd-162a7003c67e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Train model\n",
    "trainer.train()\n",
    "\n",
    "# if the model training stops in the middle, use the following to resume training from the checkpoint\n",
    "# trainer.train(resume_from_checkpoint=\"./results/checkpoint-3525\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e9f28bee-b436-4a5f-8616-b785ded0f1bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save trained model\n",
    "trainer.model.save_pretrained(new_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6a8b260f-b73b-483f-bade-04ce9a9666d1",
   "metadata": {},
   "source": [
    "### 6. Merge model in FP16 and merge it with LoRA weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c57a03b2-7d57-4af3-be8a-db199e64d0fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reload model in FP16 and merge it with LoRA weights\n",
    "base_model = AutoModelForCausalLM.from_pretrained(\n",
    "    model_name,\n",
    "    low_cpu_mem_usage=True,\n",
    "    return_dict=True,\n",
    "    torch_dtype=torch.float16,\n",
    "    device_map=device_map,\n",
    ")\n",
    "model = PeftModel.from_pretrained(base_model, new_model)\n",
    "model = model.merge_and_unload()\n",
    "\n",
    "# Reload tokenizer to save it\n",
    "tokenizer = LlamaTokenizer.from_pretrained(model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c293d60e-3f27-475b-841b-0fc9ffe2c6d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# need huggingface-cli login\n",
    "!huggingface-cli login"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a8c05f2-d95b-430c-8d59-827beb410e21",
   "metadata": {},
   "outputs": [],
   "source": [
    "# push the model and tokenizer to hugging face\n",
    "model.push_to_hub(new_model, use_temp_dir=False)\n",
    "tokenizer.push_to_hub(new_model, use_temp_dir=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "62913465-71af-45cc-a99b-24fef5762a77",
   "metadata": {},
   "source": [
    "### 7. Set up eval configs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a59157a-1268-4cbb-9bf5-72f72adda2e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "import evaluate\n",
    "import numpy as np\n",
    "\n",
    "# Ignore warnings\n",
    "logging.set_verbosity(logging.CRITICAL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d9004972-0c1b-4602-a651-8b4a9bbf3ab2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load tokenizer and model with QLoRA configuration\n",
    "compute_dtype = getattr(torch, bnb_4bit_compute_dtype)\n",
    "\n",
    "print(compute_dtype)\n",
    "\n",
    "bnb_config = BitsAndBytesConfig(\n",
    "    load_in_4bit=use_4bit,\n",
    "    bnb_4bit_quant_type=bnb_4bit_quant_type,\n",
    "    bnb_4bit_compute_dtype=compute_dtype,\n",
    "    bnb_4bit_use_double_quant=use_nested_quant,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7e4db664-bce7-41c5-8331-5119e3a69c57",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load base model\n",
    "model = AutoModelForCausalLM.from_pretrained(\n",
    "    'chenqile09/chinese-alpaca-2-LoRA-13B-couplet-100k',\n",
    "    quantization_config=bnb_config,\n",
    "    device_map=device_map\n",
    ")\n",
    "model.config.use_cache = False\n",
    "model.config.pretraining_tp = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "63235fa9-2477-453f-a795-9046ad9a3976",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load base model\n",
    "model = AutoModelForCausalLM.from_pretrained(\n",
    "    new_model,\n",
    "    quantization_config=bnb_config,\n",
    "    device_map=device_map\n",
    ")\n",
    "model.config.use_cache = False\n",
    "model.config.pretraining_tp = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4493421-f162-4cd5-8aad-d5be86580778",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load tokenizer\n",
    "tokenizer = LlamaTokenizer.from_pretrained('chenqile09/chinese-alpaca-2-LoRA-13B-couplet-100k')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c1658370-9a9a-4eef-a8e4-e994ed6e135f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set up the text generation config\n",
    "\n",
    "generation_config = GenerationConfig(\n",
    "    temperature=0.2,\n",
    "    top_k=40,\n",
    "    top_p=0.9,\n",
    "    do_sample=True,\n",
    "    num_beams=1,\n",
    "    repetition_penalty=1.1,\n",
    "    max_new_tokens=400\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "54c40248-7db0-438a-a6d4-4eb77727488b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# prompt generation and helper function to generate output\n",
    "COUPLET_SYSTEM_PROMPT = \"\"\"你是一个写对联的助手，你会根据给出的上联来写下联。上联和下联的字数要一致，并且平仄要对应。\"\"\"\n",
    "\n",
    "TEMPLATE = (\n",
    "    \"[INST] <<SYS>> {system_prompt} <</SYS>> {instruction} [/INST]\"\n",
    ")\n",
    "\n",
    "def generate_prompt(instructions, system_prompt=COUPLET_SYSTEM_PROMPT):\n",
    "    return [TEMPLATE.format_map({'instruction': instruction,'system_prompt': system_prompt}) for instruction in instructions]\n",
    "\n",
    "def generate_output_from_llm(model, tokenizer, generation_config, instructions, system_prompt=COUPLET_SYSTEM_PROMPT):\n",
    "\n",
    "    input_texts = generate_prompt(instructions, system_prompt)\n",
    "    inputs = tokenizer(input_texts, return_tensors=\"pt\", padding=True)\n",
    "    generation_outputs = model.generate(\n",
    "        input_ids = inputs[\"input_ids\"].to(device),\n",
    "        attention_mask = inputs['attention_mask'].to(device),\n",
    "        eos_token_id=tokenizer.eos_token_id,\n",
    "        pad_token_id=tokenizer.pad_token_id,\n",
    "        generation_config = generation_config\n",
    "    )\n",
    "\n",
    "    outputs = [tokenizer.decode(s, skip_special_tokens=True) for s in generation_outputs]\n",
    "    responses = [output.split(\"[/INST]\")[-1].strip() for output in outputs]\n",
    "    return responses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2aecc0aa-e3d4-4168-ad98-19d72ca6b7f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# some example inputs\n",
    "# https://github.com/minaxixi/ai-couplet\n",
    "instructions = [\n",
    "    \"欲把心事付瑶琴\",\n",
    "    \"半衾幽梦香初散\",\n",
    "    \"应是飞鸿踏泥雪\",\n",
    "    \"云破月来花弄影\",\n",
    "    \"别后相思空一水\",\n",
    "    \"书中自有黄金屋\",\n",
    "    \"花谢花飞花满天\",\n",
    "    \"杨柳岸晓风残月\",\n",
    "    \"上海自来水来自海上\",\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b35a537-df3a-45af-8577-66649885f6c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate responses in batch, and then print out\n",
    "\n",
    "responses = generate_output_from_llm(\n",
    "    model,\n",
    "    tokenizer,\n",
    "    generation_config,\n",
    "    instructions,\n",
    "    COUPLET_SYSTEM_PROMPT\n",
    ")\n",
    "\n",
    "for instruction, response in zip(instructions, responses):\n",
    "    print(instruction+';'+response)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9a3fc0f0-ede9-46b3-ab42-1962ab6b9ffd",
   "metadata": {},
   "source": [
    "### 8. Run eval on validation dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "28205542-dd02-4218-a846-9f05534020be",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load dataset\n",
    "dataset = load_dataset(dataset_name, split=\"validation\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67a386b2-029e-4bfa-aa84-46d783f34156",
   "metadata": {},
   "outputs": [],
   "source": [
    "# batch the dataset into the dataloader\n",
    "ds_loader = torch.utils.data.DataLoader(\n",
    "    dataset,\n",
    "    batch_size=8,\n",
    "    num_workers=4,\n",
    "    pin_memory=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f822d8c-67a8-40a8-aa9d-a4537db9e153",
   "metadata": {},
   "outputs": [],
   "source": [
    "# eval mode\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3f3aadb-670b-4f8a-b6e7-b5692aee64ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "# run eval on bert-score\n",
    "bertscore = evaluate.load(\"bertscore\")\n",
    "\n",
    "for i, batch in tqdm(list(enumerate(ds_loader))):\n",
    "    with torch.no_grad():\n",
    "        predictions = generate_output_from_llm(\n",
    "            model,\n",
    "            tokenizer,\n",
    "            generation_config,\n",
    "            batch['input'],\n",
    "            COUPLET_SYSTEM_PROMPT\n",
    "        )\n",
    "\n",
    "    bertscore.add_batch(references=batch['output'], predictions=predictions)\n",
    "\n",
    "result = bertscore.compute(lang='zh')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eeda345c-da72-4a2d-99d0-030fa6e49732",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print the result\n",
    "results = dict()\n",
    "\n",
    "for key, val in result.items():\n",
    "    if key in {'precision', 'recall', 'f1'}:\n",
    "        results[key] = np.mean(val)\n",
    "\n",
    "print(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ae0486c2-0012-47a5-93ab-681d50b5653d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}