diff --git a/dummy-evaluation.py b/dummy-evaluation.py index 318b80d..53ec812 100644 --- a/dummy-evaluation.py +++ b/dummy-evaluation.py @@ -3,7 +3,12 @@ import plotly.graph_objects as go import plotly.io as pio -metrics = {"answer_relevancy", "answer_correctness", "context_precision", "context_recall"} +metrics = { + "answer_relevancy", + "answer_correctness", + "context_precision", + "context_recall", +} dummy_data = {metric: np.random.rand(100) for metric in metrics} df = pd.DataFrame(dummy_data) @@ -13,8 +18,20 @@ pio.templates.default = "gridon" fig = go.Figure() -metrics = [metric for metric in df.columns.to_list() if metric not in ["question", "ground_truth", "answer", "contexts"]] +metrics = [ + metric + for metric in df.columns.to_list() + if metric not in ["question", "ground_truth", "answer", "contexts"] +] for metric in metrics: - fig.add_trace(go.Violin(y=df[metric], name=metric, points="all", box_visible=True, meanline_visible=True)) -fig.update_yaxes(range=[-0.02,1.02]) + fig.add_trace( + go.Violin( + y=df[metric], + name=metric, + points="all", + box_visible=True, + meanline_visible=True, + ) + ) +fig.update_yaxes(range=[-0.02, 1.02]) fig.write_image("metrics.png") diff --git a/notebooks/ragas_eval.ipynb b/notebooks/ragas_eval.ipynb index 53d862b..56bcb43 100644 --- a/notebooks/ragas_eval.ipynb +++ b/notebooks/ragas_eval.ipynb @@ -54,7 +54,7 @@ "metadata": {}, "outputs": [], "source": [ - "nest_asyncio.apply() # apply the event loop async fix" + "nest_asyncio.apply() # apply the event loop async fix" ] }, { @@ -70,7 +70,9 @@ "metadata": {}, "outputs": [], "source": [ - "df = pd.read_csv(\"../data/evaluation-sets/eidc-eval-sample.csv\", converters={\"contexts\": pd.eval})\n", + "df = pd.read_csv(\n", + " \"../data/evaluation-sets/eidc-eval-sample.csv\", converters={\"contexts\": pd.eval}\n", + ")\n", "eval_dataset = Dataset.from_pandas(df)" ] }, @@ -235,8 +237,8 @@ "metadata": {}, "outputs": [], "source": [ - "llm = ChatOllama(model='mistral-nemo', num_ctx=16384)\n", - "embeddings = OllamaEmbeddings(model='mistral-nemo', num_ctx=16384)" + "llm = ChatOllama(model=\"mistral-nemo\", num_ctx=16384)\n", + "embeddings = OllamaEmbeddings(model=\"mistral-nemo\", num_ctx=16384)" ] }, { @@ -316,10 +318,22 @@ "result_df = result.to_pandas()\n", "pio.templates.default = \"gridon\"\n", "fig = go.Figure()\n", - "metrics = [metric for metric in result_df.columns.to_list() if metric not in [\"question\", \"ground_truth\", \"answer\", \"contexts\"]]\n", + "metrics = [\n", + " metric\n", + " for metric in result_df.columns.to_list()\n", + " if metric not in [\"question\", \"ground_truth\", \"answer\", \"contexts\"]\n", + "]\n", "for metric in metrics:\n", - " fig.add_trace(go.Violin(y=result_df[metric], name=metric, points=\"all\", box_visible=True, meanline_visible=True))\n", - "fig.update_yaxes(range=[-0.02,1.02])\n", + " fig.add_trace(\n", + " go.Violin(\n", + " y=result_df[metric],\n", + " name=metric,\n", + " points=\"all\",\n", + " box_visible=True,\n", + " meanline_visible=True,\n", + " )\n", + " )\n", + "fig.update_yaxes(range=[-0.02, 1.02])\n", "with open(\"eval.png\", \"wb\") as f:\n", " f.write(fig.to_image(format=\"png\"))" ] @@ -331,6 +345,7 @@ "outputs": [], "source": [ "import json\n", + "\n", "with open(\"metrics.json\", \"w\") as f:\n", " json.dump(result, f)" ] diff --git a/notebooks/ragas_synth.ipynb b/notebooks/ragas_synth.ipynb index b0c4371..cf39b04 100644 --- a/notebooks/ragas_synth.ipynb +++ b/notebooks/ragas_synth.ipynb @@ -19,8 +19,6 @@ "from ragas.testset.generator import TestsetGenerator\n", "from ragas.testset.evolutions import simple, reasoning, multi_context\n", "from ragas.run_config import RunConfig\n", - "from langchain.docstore.document import Document\n", - "import pandas as pd\n", "import nest_asyncio" ] }, @@ -30,7 +28,7 @@ "metadata": {}, "outputs": [], "source": [ - "nest_asyncio.apply() # apply the event loop async fix" + "nest_asyncio.apply() # apply the event loop async fix" ] }, { @@ -46,9 +44,11 @@ "metadata": {}, "outputs": [], "source": [ - "llm = ChatOllama(model='mistral-nemo', num_ctx=16384)\n", - "embeddings = OllamaEmbeddings(model='mistral-nemo', num_ctx=16384)\n", - "gen = TestsetGenerator.from_langchain(llm, llm, embeddings, run_config=RunConfig(max_workers=1, max_retries=1))\n", + "llm = ChatOllama(model=\"mistral-nemo\", num_ctx=16384)\n", + "embeddings = OllamaEmbeddings(model=\"mistral-nemo\", num_ctx=16384)\n", + "gen = TestsetGenerator.from_langchain(\n", + " llm, llm, embeddings, run_config=RunConfig(max_workers=1, max_retries=1)\n", + ")\n", "dist = {simple: 0.6, multi_context: 0.2, reasoning: 0.2}" ] }, @@ -65,7 +65,7 @@ "metadata": {}, "outputs": [], "source": [ - "docs = [] # load a set of langchain documents to base the synthetic test set generation on" + "docs = [] # load a set of langchain documents to base the synthetic test set generation on" ] }, { @@ -81,7 +81,6 @@ "metadata": {}, "outputs": [], "source": [ - "\n", "testset = gen.generate_with_langchain_docs(docs, 5, dist, is_async=False)" ] }, diff --git a/notebooks/vllm_test.ipynb b/notebooks/vllm_test.ipynb index 755c34c..86701c7 100644 --- a/notebooks/vllm_test.ipynb +++ b/notebooks/vllm_test.ipynb @@ -15,9 +15,7 @@ "metadata": {}, "outputs": [], "source": [ - "prompts = [\n", - " \"Tell me a joke.\"\n", - "]\n", + "prompts = [\"Tell me a joke.\"]\n", "params = SamplingParams(temperature=0.8, top_p=0.95, max_tokens=1024)" ] }, @@ -26,9 +24,7 @@ "execution_count": 3, "metadata": {}, "outputs": [], - "source": [ - "import os" - ] + "source": [] }, { "cell_type": "code", @@ -87,7 +83,12 @@ } ], "source": [ - "llm = LLM(model=\"unsloth/Mistral-Nemo-Instruct-2407-bnb-4bit\", quantization=\"bitsandbytes\", load_format=\"bitsandbytes\", max_model_len=4096)" + "llm = LLM(\n", + " model=\"unsloth/Mistral-Nemo-Instruct-2407-bnb-4bit\",\n", + " quantization=\"bitsandbytes\",\n", + " load_format=\"bitsandbytes\",\n", + " max_model_len=4096,\n", + ")" ] }, {