From b0e596dd0368491d89858fec9585ccacc6793551 Mon Sep 17 00:00:00 2001
From: Firoj Alam <firojalam@gmail.com>
Date: Mon, 29 Jan 2024 16:14:24 +0300
Subject: [PATCH 1/2] exp completed for ArabGend_Jais13b_ZeroShot

---
 .../gender/ArabGend_Jais13b_ZeroShot.py       | 48 +++++++++++++++
 .../gender/ArapTweet_Jais13b_ZeroShot.py      | 58 +++++++++++++++++++
 2 files changed, 106 insertions(+)
 create mode 100644 assets/ar/demographic_attributes/gender/ArabGend_Jais13b_ZeroShot.py
 create mode 100644 assets/ar/demographic_attributes/gender/ArapTweet_Jais13b_ZeroShot.py

diff --git a/assets/ar/demographic_attributes/gender/ArabGend_Jais13b_ZeroShot.py b/assets/ar/demographic_attributes/gender/ArabGend_Jais13b_ZeroShot.py
new file mode 100644
index 00000000..11ff52c7
--- /dev/null
+++ b/assets/ar/demographic_attributes/gender/ArabGend_Jais13b_ZeroShot.py
@@ -0,0 +1,48 @@
+from llmebench.datasets import ArabGendDataset
+from llmebench.models import FastChatModel
+from llmebench.tasks import ClassificationTask
+
+
+def metadata():
+    return {
+        "author": "Arabic Language Technologies, QCRI, HBKU",
+        "model": "Jais-13b",
+        "description": "Locally hosted Jais-13b-chat model using FastChat.",
+        "scores": {"Macro-F1": ""},
+    }
+
+
+def config():
+    return {
+        "dataset": ArabGendDataset,
+        "task": ClassificationTask,
+        "model": FastChatModel,
+        "model_args": {
+            "class_labels": ["m", "f"],
+            "max_tries": 3,
+        },
+    }
+
+
+def prompt(input_sample):
+    base_prompt = (
+        f"Identify the gender from the following name as 'female' or 'male'.\n\n"
+        f"name: {input_sample}"
+        f"gender: \n"
+    )
+    return [
+        {
+            "role": "user",
+            "content": base_prompt,
+        },
+    ]
+
+
+def post_process(response):
+    label = response["choices"][0]["message"]["content"]
+    if label.lower() == "male":
+        return "m"
+    elif "female" in label.lower():
+        return "f"
+    else:
+        return None
diff --git a/assets/ar/demographic_attributes/gender/ArapTweet_Jais13b_ZeroShot.py b/assets/ar/demographic_attributes/gender/ArapTweet_Jais13b_ZeroShot.py
new file mode 100644
index 00000000..2e49be90
--- /dev/null
+++ b/assets/ar/demographic_attributes/gender/ArapTweet_Jais13b_ZeroShot.py
@@ -0,0 +1,58 @@
+from llmebench.datasets import ArapTweetDataset
+from llmebench.models import FastChatModel
+from llmebench.tasks import ClassificationTask
+
+
+def metadata():
+    return {
+        "author": "Arabic Language Technologies, QCRI, HBKU",
+        "model": "Jais-13b",
+        "description": "Locally hosted Jais-13b-chat model using FastChat.",
+        "scores": {"Macro-F1": ""},
+    }
+
+
+def config():
+    return {
+        "dataset": ArapTweetDataset,
+        "task": ClassificationTask,
+        "model": FastChatModel,
+        "model_args": {
+            "class_labels": ["Female", "Male"],
+            "max_tries": 30,
+        },
+    }
+
+
+def prompt(input_sample):
+    base_prompt = (
+        f"Identify the gender from the following name as 'Female' or 'Male'.\n\n"
+        f"name: {input_sample}"
+        f"gender: \n"
+    )
+
+    return [
+        {
+            "role": "user",
+            "content": base_prompt,
+        },
+    ]
+
+
+def post_process(response):
+    label = response["choices"][0]["message"]["content"]
+    # label = label.replace("gender:", "").strip()
+    if "gender: Female" in label or "\nFemale" in label or label == "Female":
+        label = "Female"
+    elif (
+        "gender: Male" in label
+        or "\nMale" in label
+        or "likely to be 'Male'" in label
+        or label == "Male"
+        or "typically a 'Male' name" in label
+    ):
+        label = "Male"
+    else:
+        label = None
+
+    return label

From afe404d4859c283b8866f6aba138c9d2297b74db Mon Sep 17 00:00:00 2001
From: Firoj Alam <firojalam@gmail.com>
Date: Mon, 29 Jan 2024 16:18:02 +0300
Subject: [PATCH 2/2] updated results

---
 .../demographic_attributes/gender/ArabGend_Jais13b_ZeroShot.py  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/assets/ar/demographic_attributes/gender/ArabGend_Jais13b_ZeroShot.py b/assets/ar/demographic_attributes/gender/ArabGend_Jais13b_ZeroShot.py
index 11ff52c7..56a74db5 100644
--- a/assets/ar/demographic_attributes/gender/ArabGend_Jais13b_ZeroShot.py
+++ b/assets/ar/demographic_attributes/gender/ArabGend_Jais13b_ZeroShot.py
@@ -8,7 +8,7 @@ def metadata():
         "author": "Arabic Language Technologies, QCRI, HBKU",
         "model": "Jais-13b",
         "description": "Locally hosted Jais-13b-chat model using FastChat.",
-        "scores": {"Macro-F1": ""},
+        "scores": {"Macro-F1": "0.674"},
     }