From f37ecfcbfcde5733b58ceea24941c184d1d5e5cf Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Wed, 3 Jul 2024 11:27:38 -0400 Subject: [PATCH] Fix mismatch in full pipeline outputs The full knowledge pipeline had `question` and `response` as output columns, while the skills pipelines used `question` and `answer`. `generate_data.py` currently expects `response` instead of `answer`. Instead of having to deal with both, just standardize on `response`, since that seems to be used more frequently. For example, various prompt filenames have "response" in their names. Signed-off-by: Russell Bryant --- .../sdg/configs/skills/evaluate_freeform_pair.yaml | 2 +- .../sdg/configs/skills/evaluate_grounded_pair.yaml | 4 ++-- src/instructlab/sdg/default_flows.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/instructlab/sdg/configs/skills/evaluate_freeform_pair.yaml b/src/instructlab/sdg/configs/skills/evaluate_freeform_pair.yaml index cd2921cb..1dd3e38d 100644 --- a/src/instructlab/sdg/configs/skills/evaluate_freeform_pair.yaml +++ b/src/instructlab/sdg/configs/skills/evaluate_freeform_pair.yaml @@ -33,7 +33,7 @@ generation: | [End of Question] [Start of Answer] - {answer} + {response} [End of Answer] Begin your evaluation by providing a short explanation. Be as objective as possible. After providing your explanation, you must rate the answer on a scale of 1 to 3 as mentioned above. diff --git a/src/instructlab/sdg/configs/skills/evaluate_grounded_pair.yaml b/src/instructlab/sdg/configs/skills/evaluate_grounded_pair.yaml index 45580d3b..15132c88 100644 --- a/src/instructlab/sdg/configs/skills/evaluate_grounded_pair.yaml +++ b/src/instructlab/sdg/configs/skills/evaluate_grounded_pair.yaml @@ -43,7 +43,7 @@ generation: | [End of Question] [Start of Answer] - {answer} + {response} [End of Answer] * Return the evaluation between [Start of Evaluation] and [End of Evaluation] tags. @@ -51,4 +51,4 @@ generation: | start_tags: ["[Start of Evaluation]", "[Start of Score]"] -end_tags: ["[End of Evaluation]", "[End of Score]"] \ No newline at end of file +end_tags: ["[End of Evaluation]", "[End of Score]"] diff --git a/src/instructlab/sdg/default_flows.py b/src/instructlab/sdg/default_flows.py index 31edd3d6..3a44f127 100644 --- a/src/instructlab/sdg/default_flows.py +++ b/src/instructlab/sdg/default_flows.py @@ -329,7 +329,7 @@ def get_flow(self) -> list: "client": self.client, "model_id": self.model_id, "model_prompt": _get_model_prompt(self.model_family), - "output_cols": ["answer"], + "output_cols": ["response"], "batch_kwargs": { "num_procs": 8, "batched": self.batched, @@ -449,7 +449,7 @@ def get_flow(self) -> list: "client": self.client, "model_id": self.model_id, "model_prompt": _get_model_prompt(self.model_family), - "output_cols": ["answer"], + "output_cols": ["response"], "batch_kwargs": { "num_procs": 8, "batched": self.batched,