diff --git a/docs/docs/tutorials/entity_extraction/index.ipynb b/docs/docs/tutorials/entity_extraction/index.ipynb index 8ec298196..f298add18 100644 --- a/docs/docs/tutorials/entity_extraction/index.ipynb +++ b/docs/docs/tutorials/entity_extraction/index.ipynb @@ -239,7 +239,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Average Metric: 172.00 / 200 (86.0%): 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 200/200 [00:16<00:00, 11.94it/s]" + "Average Metric: 172.00 / 200 (86.0%): 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████| 200/200 [00:16<00:00, 11.94it/s]" ] }, { @@ -299,7 +299,7 @@ " [Nadim, Ladki]\n", " We extracted the tokens \"Nadim\" and \"Ladki\" as they refer to speci...\n", " [Nadim, Ladki]\n", - " \u2714\ufe0f [True]\n", + " ✔️ [True]\n", " \n", " \n", " 2\n", @@ -307,7 +307,7 @@ " []\n", " There are no tokens referring to specific people in the provided l...\n", " []\n", - " \u2714\ufe0f [True]\n", + " ✔️ [True]\n", " \n", " \n", " 3\n", @@ -315,7 +315,7 @@ " []\n", " We did not find any tokens referring to specific people in the pro...\n", " []\n", - " \u2714\ufe0f [True]\n", + " ✔️ [True]\n", " \n", " \n", " 4\n", @@ -339,7 +339,7 @@ " [David, Campese]\n", " The extracted_people includes \"David Campese\" as it refers to a sp...\n", " [David, Campese]\n", - " \u2714\ufe0f [True]\n", + " ✔️ [True]\n", " \n", " \n", " 196\n", @@ -347,7 +347,7 @@ " []\n", " The extracted_people includes \"Wallabies\" as it refers to a specif...\n", " []\n", - " \u2714\ufe0f [True]\n", + " ✔️ [True]\n", " \n", " \n", " 197\n", @@ -355,7 +355,7 @@ " [Campese, Rob, Andrew]\n", " The extracted tokens refer to specific people mentioned in the tex...\n", " [Campese, Rob, Andrew]\n", - " \u2714\ufe0f [True]\n", + " ✔️ [True]\n", " \n", " \n", " 198\n", @@ -363,7 +363,7 @@ " [Campo, Andrew]\n", " The extracted tokens referring to specific people include \"Campo\" ...\n", " [Campo, Andrew]\n", - " \u2714\ufe0f [True]\n", + " ✔️ [True]\n", " \n", " \n", " 199\n", @@ -371,11 +371,11 @@ " []\n", " We extracted the names of specific people from the tokenized text....\n", " []\n", - " \u2714\ufe0f [True]\n", + " ✔️ [True]\n", " \n", " \n", "\n", - "

200 rows \u00d7 5 columns

\n", + "

200 rows × 5 columns

\n", "" ], "text/plain": [ @@ -420,16 +420,16 @@ "\n", " extracted_people extraction_correctness_metric \n", "0 [JAPAN, CHINA] \n", - "1 [Nadim, Ladki] \u2714\ufe0f [True] \n", - "2 [] \u2714\ufe0f [True] \n", - "3 [] \u2714\ufe0f [True] \n", + "1 [Nadim, Ladki] ✔️ [True] \n", + "2 [] ✔️ [True] \n", + "3 [] ✔️ [True] \n", "4 [China, Uzbekistan] \n", ".. ... ... \n", - "195 [David, Campese] \u2714\ufe0f [True] \n", - "196 [] \u2714\ufe0f [True] \n", - "197 [Campese, Rob, Andrew] \u2714\ufe0f [True] \n", - "198 [Campo, Andrew] \u2714\ufe0f [True] \n", - "199 [] \u2714\ufe0f [True] \n", + "195 [David, Campese] ✔️ [True] \n", + "196 [] ✔️ [True] \n", + "197 [Campese, Rob, Andrew] ✔️ [True] \n", + "198 [Campo, Andrew] ✔️ [True] \n", + "199 [] ✔️ [True] \n", "\n", "[200 rows x 5 columns]" ] @@ -469,252 +469,9 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2024/11/18 21:08:04 INFO dspy.teleprompt.mipro_optimizer_v2: \n", - "RUNNING WITH THE FOLLOWING MEDIUM AUTO RUN SETTINGS:\n", - "num_trials: 25\n", - "minibatch: False\n", - "num_candidates: 19\n", - "valset size: 40\n", - "\n", - "2024/11/18 21:08:04 INFO dspy.teleprompt.mipro_optimizer_v2: \n", - "==> STEP 1: BOOTSTRAP FEWSHOT EXAMPLES <==\n", - "2024/11/18 21:08:04 INFO dspy.teleprompt.mipro_optimizer_v2: These will be used as few-shot example candidates for our program and for creating instructions.\n", - "\n", - "2024/11/18 21:08:04 INFO dspy.teleprompt.mipro_optimizer_v2: Bootstrapping N=19 sets of demonstrations...\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Bootstrapping set 1/19\n", - "Bootstrapping set 2/19\n", - "Bootstrapping set 3/19\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n...\n", - "...\n", - "...\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Bootstrapped 2 full traces after 3 examples for up to 1 rounds, amounting to 3 attempts.\n", - "Bootstrapping set 19/19\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " 40%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u258a | 4/10 [00:00<00:00, 995.21it/s]\n", - "2024/11/18 21:08:17 INFO dspy.teleprompt.mipro_optimizer_v2: \n", - "==> STEP 2: PROPOSE INSTRUCTION CANDIDATES <==\n", - "2024/11/18 21:08:17 INFO dspy.teleprompt.mipro_optimizer_v2: We will use the few-shot examples from the previous step, a generated dataset summary, a summary of the program code, and a randomly selected prompting tip to propose instructions.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Bootstrapped 4 full traces after 4 examples for up to 1 rounds, amounting to 4 attempts.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2024/11/18 21:08:21 INFO dspy.teleprompt.mipro_optimizer_v2: \n", - "Proposing instructions...\n", - "\n", - "2024/11/18 21:10:06 INFO dspy.teleprompt.mipro_optimizer_v2: Proposed Instructions for Predictor 0:\n", - "\n", - "2024/11/18 21:10:06 INFO dspy.teleprompt.mipro_optimizer_v2: 0: Extract contiguous tokens referring to specific people, if any, from a list of string tokens.\n", - "Output a list of tokens. In other words, do not combine multiple tokens into a single value.\n", - "\n", - "2024/11/18 21:10:06 INFO dspy.teleprompt.mipro_optimizer_v2: 1: Given a list of tokenized text, identify and extract all contiguous tokens that refer to specific individuals. Ensure that the output is a list of these tokens without combining them into single values. Provide a clear rationale explaining the reasoning behind each extraction.\n", - "\n", - "2024/11/18 21:10:06 INFO dspy.teleprompt.mipro_optimizer_v2: 2: In a high-stakes scenario where accurate identification of EU officials is crucial for compliance with new health regulations affecting livestock, extract contiguous tokens from the provided list that refer to specific individuals. Ensure that your output is a comprehensive list of these tokens, as any oversight could lead to significant regulatory implications. Remember, do not combine multiple tokens into a single value; each name must be clearly delineated.\n", - "\n", - "2024/11/18 21:10:06 INFO dspy.teleprompt.mipro_optimizer_v2: 3: Given a list of tokenized text strings, identify and extract any contiguous tokens that refer to specific individuals. Provide a rationale for your extraction process, explaining the reasoning step by step. Output the extracted names as a list of tokens, ensuring that multiple tokens are not combined into a single value.\n", - "\n", - "2024/11/18 21:10:06 INFO dspy.teleprompt.mipro_optimizer_v2: 4: You are a Named Entity Recognition expert. Your task is to extract contiguous tokens that refer to specific people from the provided list of string tokens. If there are no specific individuals mentioned, return an empty list. Ensure that you do not combine multiple tokens into a single value; output them as a list.\n", - "\n", - "2024/11/18 21:10:06 INFO dspy.teleprompt.mipro_optimizer_v2: 5: Given the tokenized text, extract contiguous tokens that refer to specific individuals. If there are no references to identifiable people, indicate that no people have been extracted. Provide a rationale for your reasoning process along with the list of extracted names.\n", - "\n", - "2024/11/18 21:10:06 INFO dspy.teleprompt.mipro_optimizer_v2: 6: In a critical situation where accurate identification of EU officials is essential for compliance with new regulations, extract contiguous tokens from the provided list of string tokens that specifically refer to individuals. Ensure that your output is a list of distinct tokens without combining them into single values. This task is vital for ensuring clear communication in health communications regarding livestock, particularly in the context of sheep and mad cow disease.\n", - "\n", - "2024/11/18 21:10:06 INFO dspy.teleprompt.mipro_optimizer_v2: 7: In a high-stakes situation where accurate identification of individuals is critical for regulatory compliance and public health communication, extract contiguous tokens referring to specific people from the provided list of string tokens. Ensure that you output each identified individual as separate tokens without combining multiple tokens into a single value. This task is essential for ensuring clarity and accountability in communications pertaining to EU regulations and health matters.\n", - "\n", - "2024/11/18 21:10:06 INFO dspy.teleprompt.mipro_optimizer_v2: 8: Given a list of tokenized text, identify and extract any contiguous sequences of tokens that refer specifically to individuals. Ensure that the output is a list of tokens representing those names, and do not merge multiple tokens into a single value. Provide reasoning for your extraction process, clearly stating if specific individuals were found or if the tokens did not contain any references to people.\n", - "\n", - "2024/11/18 21:10:06 INFO dspy.teleprompt.mipro_optimizer_v2: 9: In a high-stakes scenario where accurate identification of EU officials is critical for public health communications regarding livestock diseases, extract contiguous tokens that refer to specific people from the provided list of string tokens. Ensure that the output is a list of tokens, without combining multiple tokens into a single value. Provide a clear rationale explaining the reasoning behind the identification of these tokens as referring to specific individuals.\n", - "\n", - "2024/11/18 21:10:06 INFO dspy.teleprompt.mipro_optimizer_v2: 10: Identify and extract contiguous tokens from the provided list that specifically refer to individuals. Ensure that the output consists of distinct tokens representing the names, without merging them into single values.\n", - "\n", - "2024/11/18 21:10:06 INFO dspy.teleprompt.mipro_optimizer_v2: 11: In a critical situation where accurate identification of key individuals is essential for effective communication regarding EU regulations and health communications, extract contiguous tokens referring to specific people from the provided list of string tokens. Ensure that the output is a list of tokens without combining them into a single value. This task is crucial for clarity in reporting and decision-making processes.\n", - "\n", - "2024/11/18 21:10:06 INFO dspy.teleprompt.mipro_optimizer_v2: 12: In a critical situation where accurate identification of key individuals is essential for public health communications regarding EU regulations on livestock, extract contiguous tokens referring to specific people from the provided list of string tokens. Ensure that the output is a list of individual tokens, maintaining their separation to facilitate precise recognition of each person mentioned in the context.\n", - "\n", - "2024/11/18 21:10:06 INFO dspy.teleprompt.mipro_optimizer_v2: 13: In a high-stakes situation where accurate identification of individuals is critical for regulatory compliance and public health communication, extract contiguous tokens referring to specific people from the provided list of string tokens. Ensure that you output each identified individual as separate tokens without combining multiple tokens into a single value. This task is essential for ensuring clarity and accountability in communications pertaining to EU regulations and health matters.\n", - "\n", - "2024/11/18 21:10:06 INFO dspy.teleprompt.mipro_optimizer_v2: 14: You are a Named Entity Recognition expert. Your task is to extract contiguous tokens referring to specific people from a list of string tokens. Please ensure that you output a list of tokens without combining them into a single value. Provide a rationale for your extraction, explaining why the identified tokens refer to a specific person.\n", - "\n", - "2024/11/18 21:10:06 INFO dspy.teleprompt.mipro_optimizer_v2: 15: Given a list of tokenized words, identify and extract contiguous tokens that refer to specific individuals. Provide a rationale explaining the reasoning behind the extraction process, and output a list of the identified tokens without combining them into single values.\n", - "\n", - "2024/11/18 21:10:06 INFO dspy.teleprompt.mipro_optimizer_v2: 16: You are an AI text analyzer. Your task is to extract contiguous tokens that refer to specific individuals from a list of string tokens. Carefully examine the tokens and output a list of those that represent people. If no tokens refer to individuals, return an empty list. Remember to provide a rationale explaining your extraction process.\n", - "\n", - "2024/11/18 21:10:06 INFO dspy.teleprompt.mipro_optimizer_v2: 17: In a critical situation where EU regulations regarding livestock health are being discussed, it is essential to accurately identify and extract the names of officials involved in these discussions. Given a list of tokenized text, extract contiguous tokens that refer to specific individuals. Ensure that each name is output as separate tokens, as combining them could lead to confusion. This information is vital for understanding the key players in the regulatory landscape and their statements on issues like mad cow disease and sheep health.\n", - "\n", - "2024/11/18 21:10:06 INFO dspy.teleprompt.mipro_optimizer_v2: 18: In a high-stakes situation where accurate identification of key individuals is crucial for regulatory compliance and public health communication, extract contiguous tokens referring to specific people from the provided list of string tokens. Ensure that your output is a list of tokens representing individuals, without combining multiple tokens into a single value. This extraction is vital for understanding the roles and actions of officials in EU regulations related to livestock health.\n", - "\n", - "2024/11/18 21:10:06 INFO dspy.teleprompt.mipro_optimizer_v2: \n", - "\n", - "2024/11/18 21:10:06 INFO dspy.teleprompt.mipro_optimizer_v2: Evaluating the default program...\n", - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Average Metric: 34.00 / 40 (85.0%): 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 40/40 [00:10<00:00, 3.69it/s]" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2024/11/18 21:10:16 INFO dspy.evaluate.evaluate: Average Metric: 34 / 40 (85.0%)\n", - "2024/11/18 21:10:16 INFO dspy.teleprompt.mipro_optimizer_v2: Default program score: 85.0\n", - "\n", - "2024/11/18 21:10:16 INFO dspy.teleprompt.mipro_optimizer_v2: ==> STEP 3: FINDING OPTIMAL PROMPT PARAMETERS <==\n", - "2024/11/18 21:10:16 INFO dspy.teleprompt.mipro_optimizer_v2: We will evaluate the program over a series of trials with different combinations of instructions and few-shot examples to find the optimal combination using Bayesian Optimization.\n", - "\n", - "/Users/corey.zumar/miniconda3/envs/default/lib/python3.10/site-packages/optuna/samplers/_tpe/sampler.py:319: ExperimentalWarning: ``multivariate`` option is an experimental feature. The interface can change in the future.\n", - " warnings.warn(\n", - "2024/11/18 21:10:16 INFO dspy.teleprompt.mipro_optimizer_v2: ===== Trial 1 / 25 =====\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Average Metric: 34.00 / 40 (85.0%): 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 40/40 [00:17<00:00, 2.31it/s]" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2024/11/18 21:10:34 INFO dspy.evaluate.evaluate: Average Metric: 34 / 40 (85.0%)\n", - "2024/11/18 21:10:34 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 85.0 with parameters ['Predictor 0: Instruction 12', 'Predictor 0: Few-Shot Set 7'].\n", - "2024/11/18 21:10:34 INFO dspy.teleprompt.mipro_optimizer_v2: Scores so far: [85.0, 85.0]\n", - "2024/11/18 21:10:34 INFO dspy.teleprompt.mipro_optimizer_v2: Best score so far: 85.0\n", - "2024/11/18 21:10:34 INFO dspy.teleprompt.mipro_optimizer_v2: ========================\n", - "\n", - "\n", - "2024/11/18 21:10:34 INFO dspy.teleprompt.mipro_optimizer_v2: ===== Trial 2 / 25 =====\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Average Metric: 36.00 / 40 (90.0%): 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 40/40 [00:09<00:00, 4.16it/s]" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2024/11/18 21:10:43 INFO dspy.evaluate.evaluate: Average Metric: 36 / 40 (90.0%)\n", - "2024/11/18 21:10:43 INFO dspy.teleprompt.mipro_optimizer_v2: \u001b[92mBest full score so far!\u001b[0m Score: 90.0\n", - "2024/11/18 21:10:43 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 90.0 with parameters ['Predictor 0: Instruction 10', 'Predictor 0: Few-Shot Set 7'].\n", - "2024/11/18 21:10:43 INFO dspy.teleprompt.mipro_optimizer_v2: Scores so far: [85.0, 85.0, 90.0]\n", - "2024/11/18 21:10:43 INFO dspy.teleprompt.mipro_optimizer_v2: Best score so far: 90.0\n", - "2024/11/18 21:10:43 INFO dspy.teleprompt.mipro_optimizer_v2: ========================\n", - "\n", - "\n", - "2024/11/18 21:10:43 INFO dspy.teleprompt.mipro_optimizer_v2: ===== Trial 3 / 25 =====\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Average Metric: 39.00 / 40 (97.5%): 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 40/40 [00:10<00:00, 3.68it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n...\n", - "...\n", - "...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2024/11/18 21:14:37 INFO dspy.evaluate.evaluate: Average Metric: 34 / 40 (85.0%)\n", - "2024/11/18 21:14:37 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 85.0 with parameters ['Predictor 0: Instruction 2', 'Predictor 0: Few-Shot Set 0'].\n", - "2024/11/18 21:14:37 INFO dspy.teleprompt.mipro_optimizer_v2: Scores so far: [85.0, 85.0, 90.0, 97.5, 95.0, 97.5, 82.5, 92.5, 85.0, 77.5, 85.0, 97.5, 97.5, 97.5, 95.0, 95.0, 97.5, 85.0, 90.0, 97.5, 92.5, 95.0, 95.0, 95.0, 85.0]\n", - "2024/11/18 21:14:37 INFO dspy.teleprompt.mipro_optimizer_v2: Best score so far: 97.5\n", - "2024/11/18 21:14:37 INFO dspy.teleprompt.mipro_optimizer_v2: =========================\n", - "\n", - "\n", - "2024/11/18 21:14:37 INFO dspy.teleprompt.mipro_optimizer_v2: ===== Trial 25 / 25 =====\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Average Metric: 39.00 / 40 (97.5%): 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 40/40 [00:00<00:00, 2609.25it/s]" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2024/11/18 21:14:37 INFO dspy.evaluate.evaluate: Average Metric: 39 / 40 (97.5%)\n", - "2024/11/18 21:14:37 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 97.5 with parameters ['Predictor 0: Instruction 7', 'Predictor 0: Few-Shot Set 18'].\n", - "2024/11/18 21:14:37 INFO dspy.teleprompt.mipro_optimizer_v2: Scores so far: [85.0, 85.0, 90.0, 97.5, 95.0, 97.5, 82.5, 92.5, 85.0, 77.5, 85.0, 97.5, 97.5, 97.5, 95.0, 95.0, 97.5, 85.0, 90.0, 97.5, 92.5, 95.0, 95.0, 95.0, 85.0, 97.5]\n", - "2024/11/18 21:14:37 INFO dspy.teleprompt.mipro_optimizer_v2: Best score so far: 97.5\n", - "2024/11/18 21:14:37 INFO dspy.teleprompt.mipro_optimizer_v2: =========================\n", - "\n", - "\n", - "2024/11/18 21:14:37 INFO dspy.teleprompt.mipro_optimizer_v2: Returning best identified program with score 97.5!\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], + "outputs": [], "source": [ "mipro_optimizer = dspy.MIPROv2(\n", " metric=extraction_correctness_metric,\n", @@ -751,7 +508,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Average Metric: 186.00 / 200 (93.0%): 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 200/200 [00:23<00:00, 8.58it/s]" + "Average Metric: 186.00 / 200 (93.0%): 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████| 200/200 [00:23<00:00, 8.58it/s]" ] }, { @@ -811,7 +568,7 @@ " [Nadim, Ladki]\n", " The tokens \"Nadim Ladki\" refer to a specific individual. Both toke...\n", " [Nadim, Ladki]\n", - " \u2714\ufe0f [True]\n", + " ✔️ [True]\n", " \n", " \n", " 2\n", @@ -819,7 +576,7 @@ " []\n", " There are no tokens referring to specific people in the provided l...\n", " []\n", - " \u2714\ufe0f [True]\n", + " ✔️ [True]\n", " \n", " \n", " 3\n", @@ -827,7 +584,7 @@ " []\n", " There are no specific people mentioned in the provided tokens. The...\n", " []\n", - " \u2714\ufe0f [True]\n", + " ✔️ [True]\n", " \n", " \n", " 4\n", @@ -835,7 +592,7 @@ " []\n", " There are no tokens referring to specific people in the provided l...\n", " []\n", - " \u2714\ufe0f [True]\n", + " ✔️ [True]\n", " \n", " \n", " ...\n", @@ -851,7 +608,7 @@ " [David, Campese]\n", " The extracted tokens refer to a specific person mentioned in the t...\n", " [David, Campese]\n", - " \u2714\ufe0f [True]\n", + " ✔️ [True]\n", " \n", " \n", " 196\n", @@ -859,7 +616,7 @@ " []\n", " There are no specific individuals mentioned in the provided tokens...\n", " []\n", - " \u2714\ufe0f [True]\n", + " ✔️ [True]\n", " \n", " \n", " 197\n", @@ -867,7 +624,7 @@ " [Campese, Rob, Andrew]\n", " The tokens include the names \"Campese\" and \"Rob Andrew,\" both of w...\n", " [Campese, Rob, Andrew]\n", - " \u2714\ufe0f [True]\n", + " ✔️ [True]\n", " \n", " \n", " 198\n", @@ -875,7 +632,7 @@ " [Campo, Andrew]\n", " The extracted tokens refer to specific people mentioned in the tex...\n", " [Campo, Andrew]\n", - " \u2714\ufe0f [True]\n", + " ✔️ [True]\n", " \n", " \n", " 199\n", @@ -883,11 +640,11 @@ " []\n", " There are no specific people mentioned in the provided tokens. The...\n", " []\n", - " \u2714\ufe0f [True]\n", + " ✔️ [True]\n", " \n", " \n", "\n", - "

200 rows \u00d7 5 columns

\n", + "

200 rows × 5 columns

\n", "" ], "text/plain": [ @@ -932,16 +689,16 @@ "\n", " extracted_people extraction_correctness_metric \n", "0 [] \n", - "1 [Nadim, Ladki] \u2714\ufe0f [True] \n", - "2 [] \u2714\ufe0f [True] \n", - "3 [] \u2714\ufe0f [True] \n", - "4 [] \u2714\ufe0f [True] \n", + "1 [Nadim, Ladki] ✔️ [True] \n", + "2 [] ✔️ [True] \n", + "3 [] ✔️ [True] \n", + "4 [] ✔️ [True] \n", ".. ... ... \n", - "195 [David, Campese] \u2714\ufe0f [True] \n", - "196 [] \u2714\ufe0f [True] \n", - "197 [Campese, Rob, Andrew] \u2714\ufe0f [True] \n", - "198 [Campo, Andrew] \u2714\ufe0f [True] \n", - "199 [] \u2714\ufe0f [True] \n", + "195 [David, Campese] ✔️ [True] \n", + "196 [] ✔️ [True] \n", + "197 [Campese, Rob, Andrew] ✔️ [True] \n", + "198 [Campo, Andrew] ✔️ [True] \n", + "199 [] ✔️ [True] \n", "\n", "[200 rows x 5 columns]" ]