Merge branch 'main' into yifanmai/fix-indexed-row-major-none

IBM · Jan 22, 2025 · 1ae3583 · 1ae3583
2 parents d48d1e7 + 4ba0ce2
commit 1ae3583
Show file tree

Hide file tree

Showing 10 changed files with 63 additions and 6 deletions.
diff --git a/prepare/cards/open_australian_legal_qa.py b/prepare/cards/open_australian_legal_qa.py
@@ -13,6 +13,7 @@
 card = TaskCard(
     loader=LoadHF(
         path="umarbutler/open-australian-legal-qa",
+        name="default",
     ),
     preprocess_steps=[
         SplitRandomMix(
@@ -51,6 +52,7 @@
 card = TaskCard(
     loader=LoadHF(
         path="umarbutler/open-australian-legal-qa",
+        name="default",
     ),
     preprocess_steps=[
         SplitRandomMix(

diff --git a/src/unitxt/catalog/cards/open_australian_legal_qa.json b/src/unitxt/catalog/cards/open_australian_legal_qa.json
@@ -2,7 +2,8 @@
     "__type__": "task_card",
     "loader": {
         "__type__": "load_hf",
-        "path": "umarbutler/open-australian-legal-qa"
+        "path": "umarbutler/open-australian-legal-qa",
+        "name": "default"
     },
     "preprocess_steps": [
         {

diff --git a/src/unitxt/catalog/cards/rag/response_generation/train/open_australian_legal_qa.json b/src/unitxt/catalog/cards/rag/response_generation/train/open_australian_legal_qa.json
@@ -2,7 +2,8 @@
     "__type__": "task_card",
     "loader": {
         "__type__": "load_hf",
-        "path": "umarbutler/open-australian-legal-qa"
+        "path": "umarbutler/open-australian-legal-qa",
+        "name": "default"
     },
     "preprocess_steps": [
         {

diff --git a/src/unitxt/catalog/metrics/llm_as_judge/direct/rits/granite3_0_8b.json b/src/unitxt/catalog/metrics/llm_as_judge/direct/rits/granite3_0_8b.json
@@ -0,0 +1,11 @@
+{
+    "__type__": "llm_judge_direct",
+    "inference_engine": {
+        "__type__": "rits_inference_engine",
+        "model_name": "ibm-granite/granite-3.0-8b-instruct",
+        "max_tokens": 1024,
+        "seed": 42
+    },
+    "evaluator_name": "GRANITE3_8B",
+    "generate_summaries": false
+}
diff --git a/src/unitxt/catalog/metrics/llm_as_judge/direct/rits/granite3_1_8b.json b/src/unitxt/catalog/metrics/llm_as_judge/direct/rits/granite3_1_8b.json
@@ -0,0 +1,11 @@
+{
+    "__type__": "llm_judge_direct",
+    "inference_engine": {
+        "__type__": "rits_inference_engine",
+        "model_name": "ibm-granite/granite-3.1-8b-instruct",
+        "max_tokens": 1024,
+        "seed": 42
+    },
+    "evaluator_name": "GRANITE3_1_8B",
+    "generate_summaries": false
+}
diff --git a/..._as_judge/direct/watsonx/granite3_8b.json → ...s_judge/direct/watsonx/granite3_0_8b.json b/..._as_judge/direct/watsonx/granite3_8b.json → ...s_judge/direct/watsonx/granite3_0_8b.json
diff --git a/src/unitxt/catalog/metrics/llm_as_judge/pairwise/rits/granite3_0_8b.json b/src/unitxt/catalog/metrics/llm_as_judge/pairwise/rits/granite3_0_8b.json
@@ -0,0 +1,11 @@
+{
+    "__type__": "llm_judge_pairwise",
+    "inference_engine": {
+        "__type__": "rits_inference_engine",
+        "model_name": "ibm-granite/granite-3.0-8b-instruct",
+        "max_tokens": 1024,
+        "seed": 42
+    },
+    "evaluator_name": "GRANITE3_8B",
+    "generate_summaries": false
+}
diff --git a/src/unitxt/catalog/metrics/llm_as_judge/pairwise/rits/granite3_1_8b.json b/src/unitxt/catalog/metrics/llm_as_judge/pairwise/rits/granite3_1_8b.json
@@ -0,0 +1,11 @@
+{
+    "__type__": "llm_judge_pairwise",
+    "inference_engine": {
+        "__type__": "rits_inference_engine",
+        "model_name": "ibm-granite/granite-3.1-8b-instruct",
+        "max_tokens": 1024,
+        "seed": 42
+    },
+    "evaluator_name": "GRANITE3_1_8B",
+    "generate_summaries": false
+}
diff --git a/...s_judge/pairwise/watsonx/granite3_8b.json → ...judge/pairwise/watsonx/granite3_0_8b.json b/...s_judge/pairwise/watsonx/granite3_8b.json → ...judge/pairwise/watsonx/granite3_0_8b.json
diff --git a/src/unitxt/llm_as_judge_constants.py b/src/unitxt/llm_as_judge_constants.py
@@ -80,8 +80,10 @@ class EvaluatorNameEnum(str, Enum):
     O1_PREVIEW = "o1-Preview"
     O1_MINI = "o1-Mini"
     GRANITE_13B = "Granite-13b"
-    GRANITE3_2B = "Granite3-2b"
-    GRANITE3_8B = "Granite3-8b"
+    GRANITE3_2B = "Granite3.0-2b"
+    GRANITE3_8B = "Granite3.0-8b"
+    GRANITE3_1_2B = "Granite3.1-2b"
+    GRANITE3_1_8B = "Granite3.1-8b"
     GRANITE_GUARDIAN_2B = "Granite Guardian 3.0 2B"
     GRANITE_GUARDIAN_8B = "Granite Guardian 3.0 8B"
 
@@ -108,6 +110,8 @@ class ModelProviderEnum(str, Enum):
     EvaluatorNameEnum.GRANITE_13B: "ibm/granite-13b-instruct-v2",
     EvaluatorNameEnum.GRANITE3_2B: "ibm/granite-3-2b-instruct",
     EvaluatorNameEnum.GRANITE3_8B: "ibm/granite-3-8b-instruct",
+    EvaluatorNameEnum.GRANITE3_1_2B: "ibm/granite-3.1-2b-instruct",
+    EvaluatorNameEnum.GRANITE3_1_8B: "ibm/granite-3.1-8b-instruct",
     EvaluatorNameEnum.GRANITE_GUARDIAN_2B: "ibm/granite-guardian-3-2b",
     EvaluatorNameEnum.GRANITE_GUARDIAN_8B: "ibm/granite-guardian-3-8b",
 }
@@ -116,7 +120,8 @@ class ModelProviderEnum(str, Enum):
     ModelProviderEnum.RITS: {
         "meta-llama/llama-3-1-8b-instruct": "meta-llama/Llama-3.1-8B-Instruct",
         "mistralai/mixtral-8x7b-instruct-v01": "mistralai/mixtral-8x7B-instruct-v0.1",
-        "ibm/granite-guardian-3-2b": "ibm-granite/granite-3.0-8b-instruct",
+        "ibm/granite-3-8b-instruct": "ibm-granite/granite-3.0-8b-instruct",
+        "ibm/granite-3.1-8b-instruct": "ibm-granite/granite-3.1-8b-instruct",
         "meta-llama/llama-3-405b-instruct": "meta-llama/llama-3-1-405b-instruct-fp8",
         "mistralai/mistral-large": "mistralai/mistral-large-instruct-2407",
     },
@@ -154,7 +159,11 @@ def __init__(self, name, providers):
     ),
     EvaluatorMetadata(
         EvaluatorNameEnum.GRANITE3_8B,
-        [ModelProviderEnum.WATSONX],
+        [ModelProviderEnum.WATSONX, ModelProviderEnum.RITS],
+    ),
+    EvaluatorMetadata(
+        EvaluatorNameEnum.GRANITE3_1_8B,
+        [ModelProviderEnum.RITS],
     ),
     EvaluatorMetadata(
         EvaluatorNameEnum.GPT4,