diff --git a/README.md b/README.md index 84e0454..e70a429 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -![TITLE](assets/title.png) +![TITLE](assets/haico_banner_title.png) # HAICOSYSTEM [![Python 3.11](https://img.shields.io/badge/python-3.11-blue.svg)](https://www.python.org/downloads/release/python-3109/) [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://pre-commit.com/) @@ -6,6 +6,19 @@ [![Checked with mypy](https://www.mypy-lang.org/static/mypy_badge.svg)](https://mypy-lang.org/) [![bear-ified](https://raw.githubusercontent.com/beartype/beartype-assets/main/badge/bear-ified.svg)](https://beartype.readthedocs.io) +This is the open-source codebase for the [HAICOSYSTEM](https://arxiv.org/abs/2409.16427) project. +```bibtex +@misc{zhou2024haicosystemecosystemsandboxingsafety, + title={HAICOSYSTEM: An Ecosystem for Sandboxing Safety Risks in Human-AI Interactions}, + author={Xuhui Zhou and Hyunwoo Kim and Faeze Brahman and Liwei Jiang and Hao Zhu and Ximing Lu and Frank Xu and Bill Yuchen Lin and Yejin Choi and Niloofar Mireshghallah and Ronan Le Bras and Maarten Sap}, + year={2024}, + eprint={2409.16427}, + archivePrefix={arXiv}, + primaryClass={cs.AI}, + url={https://arxiv.org/abs/2409.16427}, +} +``` + ## Get started diff --git a/assets/haico_banner_title.png b/assets/haico_banner_title.png new file mode 100644 index 0000000..48b390b Binary files /dev/null and b/assets/haico_banner_title.png differ diff --git a/examples/notebooks/database.ipynb b/examples/notebooks/database.ipynb index 7c90419..df9b272 100644 --- a/examples/notebooks/database.ipynb +++ b/examples/notebooks/database.ipynb @@ -33,12 +33,90 @@ "# print(len(episodes))" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Serialize the episodes and scenarios" + ] + }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "# export all the episodes\n", + "import json\n", + "\n", + "tags = [\n", + " \"benchmark_gpt-4-turbo_gpt-4o-2024-08-06_gpt-4o-2024-08-06_haicosystem_trial2\",\n", + " \"benchmark_gpt-3.5-turbo_gpt-4o-2024-08-06_gpt-4o-2024-08-06_haicosystem_trial2\",\n", + " \"benchmark_together_ai/meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo_gpt-4o-2024-08-06_gpt-4o-2024-08-06_haicosystem_trial2\",\n", + " \"benchmark_together_ai/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo_gpt-4o-2024-08-06_gpt-4o-2024-08-06_haicosystem_trial2\",\n", + " \"benchmark_together_ai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo_gpt-4o-2024-08-06_gpt-4o-2024-08-06_haicosystem_trial2\",\n", + " \"benchmark_together_ai/mistralai/Mixtral-8x22B-Instruct-v0.1_gpt-4o-2024-08-06_gpt-4o-2024-08-06_haicosystem_trial2\",\n", + " \"benchmark_together_ai/Qwen/Qwen1.5-72B-Chat_gpt-4o-2024-08-06_gpt-4o-2024-08-06_haicosystem_trial2\",\n", + " \"benchmark_together_ai/Qwen/Qwen2-72B-Instruct_gpt-4o-2024-08-06_gpt-4o-2024-08-06_haicosystem_trial2\",\n", + " \"benchmark_together_ai/Qwen/Qwen1.5-110B-Chat_gpt-4o-2024-08-06_gpt-4o-2024-08-06_haicosystem_trial2\",\n", + " \"benchmark_together_ai/meta-llama/Meta-Llama-3-70B-Instruct-Turbo_gpt-4o-2024-08-06_gpt-4o-2024-08-06_haicosystem_trial2\",\n", + " \"benchmark_together_ai/meta-llama/Meta-Llama-3-8B-Instruct-Turbo_gpt-4o-2024-08-06_gpt-4o-2024-08-06_haicosystem_trial2\",\n", + " \"benchmark_together_ai/deepseek-ai/deepseek-llm-67b-chat_gpt-4o-2024-08-06_gpt-4o-2024-08-06_haicosystem_trial2\",\n", + "]\n", + "all_episodes = []\n", + "for tag in tags:\n", + " episodes = EpisodeLog.find(EpisodeLog.tag == tag).all()\n", + " all_episodes.extend(episodes)\n", + "\n", + "with open(\"all_episodes.jsonl\", \"w\") as f:\n", + " for episode in all_episodes:\n", + " f.write(episode.json() + \"\\n\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# serialize the environment\n", + "from haicosystem.protocols import HaiEnvironmentProfile\n", + "\n", + "with open(\"all_environments.jsonl\", \"w\") as f:\n", + " for environment in HaiEnvironmentProfile.all():\n", + " f.write(environment.json() + \"\\n\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# serialize the combos\n", + "from sotopia.database import EnvAgentComboStorage\n", + "\n", + "with open(\"all_combos.jsonl\", \"w\") as f:\n", + " for combo in EnvAgentComboStorage.all():\n", + " f.write(combo.json() + \"\\n\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# load the episodes\n", + "with open(\"all_episodes.jsonl\", \"r\") as f:\n", + " for line in f:\n", + " print(line)\n", + " episode_dict = json.loads(line)\n", + " print(episode_dict[\"pk\"])\n", + " episode = EpisodeLog(**episode_dict)\n", + " print(episode)\n", + " break" + ] } ], "metadata": {