Skip to content

Commit

Permalink
Remove token category in calculation function
Browse files Browse the repository at this point in the history
  • Loading branch information
@menamerai
Siwei Li authored and menamerai committed Apr 11, 2024
1 parent 2d1ba75 commit 367d0d8
Show file tree
Hide file tree
Showing 2 changed files with 111 additions and 65 deletions.
124 changes: 90 additions & 34 deletions notebooks/eval_notebook.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,12 +59,12 @@
"data": {
"application/vnd.holoviews_exec.v0+json": "",
"text/html": [
"<div id='f8973921-5af7-4118-8b9f-306702128fe6'>\n",
" <div id=\"f4c246ae-3b93-47d9-90ae-c79a6016c8bb\" data-root-id=\"f8973921-5af7-4118-8b9f-306702128fe6\" style=\"display: contents;\"></div>\n",
"<div id='766cef6d-5edf-4a28-802f-c35e5727f258'>\n",
" <div id=\"b7f8c7ed-8bfe-4cd8-8cc1-c5ac1dbb13b4\" data-root-id=\"766cef6d-5edf-4a28-802f-c35e5727f258\" style=\"display: contents;\"></div>\n",
"</div>\n",
"<script type=\"application/javascript\">(function(root) {\n",
" var docs_json = {\"976a94d0-345d-4dcd-92c3-8c698508808f\":{\"version\":\"3.4.0\",\"title\":\"Bokeh Application\",\"roots\":[{\"type\":\"object\",\"name\":\"panel.models.browser.BrowserInfo\",\"id\":\"f8973921-5af7-4118-8b9f-306702128fe6\"},{\"type\":\"object\",\"name\":\"panel.models.comm_manager.CommManager\",\"id\":\"501751dc-214f-4453-a8f5-e01f570a9d42\",\"attributes\":{\"plot_id\":\"f8973921-5af7-4118-8b9f-306702128fe6\",\"comm_id\":\"3ccefab99ce94fdcab2f9ee2d3baf038\",\"client_comm_id\":\"79b1be4e12b8451d8cacbe142a7ee33b\"}}],\"defs\":[{\"type\":\"model\",\"name\":\"ReactiveHTML1\"},{\"type\":\"model\",\"name\":\"FlexBox1\",\"properties\":[{\"name\":\"align_content\",\"kind\":\"Any\",\"default\":\"flex-start\"},{\"name\":\"align_items\",\"kind\":\"Any\",\"default\":\"flex-start\"},{\"name\":\"flex_direction\",\"kind\":\"Any\",\"default\":\"row\"},{\"name\":\"flex_wrap\",\"kind\":\"Any\",\"default\":\"wrap\"},{\"name\":\"justify_content\",\"kind\":\"Any\",\"default\":\"flex-start\"},{\"name\":\"gap\",\"kind\":\"Any\",\"default\":\"\"}]},{\"type\":\"model\",\"name\":\"FloatPanel1\",\"properties\":[{\"name\":\"config\",\"kind\":\"Any\",\"default\":{\"type\":\"map\"}},{\"name\":\"contained\",\"kind\":\"Any\",\"default\":true},{\"name\":\"position\",\"kind\":\"Any\",\"default\":\"right-top\"},{\"name\":\"offsetx\",\"kind\":\"Any\",\"default\":null},{\"name\":\"offsety\",\"kind\":\"Any\",\"default\":null},{\"name\":\"theme\",\"kind\":\"Any\",\"default\":\"primary\"},{\"name\":\"status\",\"kind\":\"Any\",\"default\":\"normalized\"}]},{\"type\":\"model\",\"name\":\"GridStack1\",\"properties\":[{\"name\":\"mode\",\"kind\":\"Any\",\"default\":\"warn\"},{\"name\":\"ncols\",\"kind\":\"Any\",\"default\":null},{\"name\":\"nrows\",\"kind\":\"Any\",\"default\":null},{\"name\":\"allow_resize\",\"kind\":\"Any\",\"default\":true},{\"name\":\"allow_drag\",\"kind\":\"Any\",\"default\":true},{\"name\":\"state\",\"kind\":\"Any\",\"default\":[]}]},{\"type\":\"model\",\"name\":\"drag1\",\"properties\":[{\"name\":\"slider_width\",\"kind\":\"Any\",\"default\":5},{\"name\":\"slider_color\",\"kind\":\"Any\",\"default\":\"black\"},{\"name\":\"value\",\"kind\":\"Any\",\"default\":50}]},{\"type\":\"model\",\"name\":\"click1\",\"properties\":[{\"name\":\"terminal_output\",\"kind\":\"Any\",\"default\":\"\"},{\"name\":\"debug_name\",\"kind\":\"Any\",\"default\":\"\"},{\"name\":\"clears\",\"kind\":\"Any\",\"default\":0}]},{\"type\":\"model\",\"name\":\"FastWrapper1\",\"properties\":[{\"name\":\"object\",\"kind\":\"Any\",\"default\":null},{\"name\":\"style\",\"kind\":\"Any\",\"default\":null}]},{\"type\":\"model\",\"name\":\"NotificationAreaBase1\",\"properties\":[{\"name\":\"js_events\",\"kind\":\"Any\",\"default\":{\"type\":\"map\"}},{\"name\":\"position\",\"kind\":\"Any\",\"default\":\"bottom-right\"},{\"name\":\"_clear\",\"kind\":\"Any\",\"default\":0}]},{\"type\":\"model\",\"name\":\"NotificationArea1\",\"properties\":[{\"name\":\"js_events\",\"kind\":\"Any\",\"default\":{\"type\":\"map\"}},{\"name\":\"notifications\",\"kind\":\"Any\",\"default\":[]},{\"name\":\"position\",\"kind\":\"Any\",\"default\":\"bottom-right\"},{\"name\":\"_clear\",\"kind\":\"Any\",\"default\":0},{\"name\":\"types\",\"kind\":\"Any\",\"default\":[{\"type\":\"map\",\"entries\":[[\"type\",\"warning\"],[\"background\",\"#ffc107\"],[\"icon\",{\"type\":\"map\",\"entries\":[[\"className\",\"fas fa-exclamation-triangle\"],[\"tagName\",\"i\"],[\"color\",\"white\"]]}]]},{\"type\":\"map\",\"entries\":[[\"type\",\"info\"],[\"background\",\"#007bff\"],[\"icon\",{\"type\":\"map\",\"entries\":[[\"className\",\"fas fa-info-circle\"],[\"tagName\",\"i\"],[\"color\",\"white\"]]}]]}]}]},{\"type\":\"model\",\"name\":\"Notification\",\"properties\":[{\"name\":\"background\",\"kind\":\"Any\",\"default\":null},{\"name\":\"duration\",\"kind\":\"Any\",\"default\":3000},{\"name\":\"icon\",\"kind\":\"Any\",\"default\":null},{\"name\":\"message\",\"kind\":\"Any\",\"default\":\"\"},{\"name\":\"notification_type\",\"kind\":\"Any\",\"default\":null},{\"name\":\"_destroyed\",\"kind\":\"Any\",\"default\":false}]},{\"type\":\"model\",\"name\":\"TemplateActions1\",\"properties\":[{\"name\":\"open_modal\",\"kind\":\"Any\",\"default\":0},{\"name\":\"close_modal\",\"kind\":\"Any\",\"default\":0}]},{\"type\":\"model\",\"name\":\"BootstrapTemplateActions1\",\"properties\":[{\"name\":\"open_modal\",\"kind\":\"Any\",\"default\":0},{\"name\":\"close_modal\",\"kind\":\"Any\",\"default\":0}]},{\"type\":\"model\",\"name\":\"TemplateEditor1\",\"properties\":[{\"name\":\"layout\",\"kind\":\"Any\",\"default\":[]}]},{\"type\":\"model\",\"name\":\"MaterialTemplateActions1\",\"properties\":[{\"name\":\"open_modal\",\"kind\":\"Any\",\"default\":0},{\"name\":\"close_modal\",\"kind\":\"Any\",\"default\":0}]},{\"type\":\"model\",\"name\":\"copy_to_clipboard1\",\"properties\":[{\"name\":\"fill\",\"kind\":\"Any\",\"default\":\"none\"},{\"name\":\"value\",\"kind\":\"Any\",\"default\":null}]}]}};\n",
" var render_items = [{\"docid\":\"976a94d0-345d-4dcd-92c3-8c698508808f\",\"roots\":{\"f8973921-5af7-4118-8b9f-306702128fe6\":\"f4c246ae-3b93-47d9-90ae-c79a6016c8bb\"},\"root_ids\":[\"f8973921-5af7-4118-8b9f-306702128fe6\"]}];\n",
" var docs_json = {\"01e43340-2e4a-4e82-b3de-797c2c99037c\":{\"version\":\"3.4.0\",\"title\":\"Bokeh Application\",\"roots\":[{\"type\":\"object\",\"name\":\"panel.models.browser.BrowserInfo\",\"id\":\"766cef6d-5edf-4a28-802f-c35e5727f258\"},{\"type\":\"object\",\"name\":\"panel.models.comm_manager.CommManager\",\"id\":\"3a4d12c5-2afd-4ea9-93a2-4773d42944fc\",\"attributes\":{\"plot_id\":\"766cef6d-5edf-4a28-802f-c35e5727f258\",\"comm_id\":\"b4aeb58f3a864703977e0ccf1163bd23\",\"client_comm_id\":\"515633130bee4747af190854d0c65b95\"}}],\"defs\":[{\"type\":\"model\",\"name\":\"ReactiveHTML1\"},{\"type\":\"model\",\"name\":\"FlexBox1\",\"properties\":[{\"name\":\"align_content\",\"kind\":\"Any\",\"default\":\"flex-start\"},{\"name\":\"align_items\",\"kind\":\"Any\",\"default\":\"flex-start\"},{\"name\":\"flex_direction\",\"kind\":\"Any\",\"default\":\"row\"},{\"name\":\"flex_wrap\",\"kind\":\"Any\",\"default\":\"wrap\"},{\"name\":\"justify_content\",\"kind\":\"Any\",\"default\":\"flex-start\"},{\"name\":\"gap\",\"kind\":\"Any\",\"default\":\"\"}]},{\"type\":\"model\",\"name\":\"FloatPanel1\",\"properties\":[{\"name\":\"config\",\"kind\":\"Any\",\"default\":{\"type\":\"map\"}},{\"name\":\"contained\",\"kind\":\"Any\",\"default\":true},{\"name\":\"position\",\"kind\":\"Any\",\"default\":\"right-top\"},{\"name\":\"offsetx\",\"kind\":\"Any\",\"default\":null},{\"name\":\"offsety\",\"kind\":\"Any\",\"default\":null},{\"name\":\"theme\",\"kind\":\"Any\",\"default\":\"primary\"},{\"name\":\"status\",\"kind\":\"Any\",\"default\":\"normalized\"}]},{\"type\":\"model\",\"name\":\"GridStack1\",\"properties\":[{\"name\":\"mode\",\"kind\":\"Any\",\"default\":\"warn\"},{\"name\":\"ncols\",\"kind\":\"Any\",\"default\":null},{\"name\":\"nrows\",\"kind\":\"Any\",\"default\":null},{\"name\":\"allow_resize\",\"kind\":\"Any\",\"default\":true},{\"name\":\"allow_drag\",\"kind\":\"Any\",\"default\":true},{\"name\":\"state\",\"kind\":\"Any\",\"default\":[]}]},{\"type\":\"model\",\"name\":\"drag1\",\"properties\":[{\"name\":\"slider_width\",\"kind\":\"Any\",\"default\":5},{\"name\":\"slider_color\",\"kind\":\"Any\",\"default\":\"black\"},{\"name\":\"value\",\"kind\":\"Any\",\"default\":50}]},{\"type\":\"model\",\"name\":\"click1\",\"properties\":[{\"name\":\"terminal_output\",\"kind\":\"Any\",\"default\":\"\"},{\"name\":\"debug_name\",\"kind\":\"Any\",\"default\":\"\"},{\"name\":\"clears\",\"kind\":\"Any\",\"default\":0}]},{\"type\":\"model\",\"name\":\"FastWrapper1\",\"properties\":[{\"name\":\"object\",\"kind\":\"Any\",\"default\":null},{\"name\":\"style\",\"kind\":\"Any\",\"default\":null}]},{\"type\":\"model\",\"name\":\"NotificationAreaBase1\",\"properties\":[{\"name\":\"js_events\",\"kind\":\"Any\",\"default\":{\"type\":\"map\"}},{\"name\":\"position\",\"kind\":\"Any\",\"default\":\"bottom-right\"},{\"name\":\"_clear\",\"kind\":\"Any\",\"default\":0}]},{\"type\":\"model\",\"name\":\"NotificationArea1\",\"properties\":[{\"name\":\"js_events\",\"kind\":\"Any\",\"default\":{\"type\":\"map\"}},{\"name\":\"notifications\",\"kind\":\"Any\",\"default\":[]},{\"name\":\"position\",\"kind\":\"Any\",\"default\":\"bottom-right\"},{\"name\":\"_clear\",\"kind\":\"Any\",\"default\":0},{\"name\":\"types\",\"kind\":\"Any\",\"default\":[{\"type\":\"map\",\"entries\":[[\"type\",\"warning\"],[\"background\",\"#ffc107\"],[\"icon\",{\"type\":\"map\",\"entries\":[[\"className\",\"fas fa-exclamation-triangle\"],[\"tagName\",\"i\"],[\"color\",\"white\"]]}]]},{\"type\":\"map\",\"entries\":[[\"type\",\"info\"],[\"background\",\"#007bff\"],[\"icon\",{\"type\":\"map\",\"entries\":[[\"className\",\"fas fa-info-circle\"],[\"tagName\",\"i\"],[\"color\",\"white\"]]}]]}]}]},{\"type\":\"model\",\"name\":\"Notification\",\"properties\":[{\"name\":\"background\",\"kind\":\"Any\",\"default\":null},{\"name\":\"duration\",\"kind\":\"Any\",\"default\":3000},{\"name\":\"icon\",\"kind\":\"Any\",\"default\":null},{\"name\":\"message\",\"kind\":\"Any\",\"default\":\"\"},{\"name\":\"notification_type\",\"kind\":\"Any\",\"default\":null},{\"name\":\"_destroyed\",\"kind\":\"Any\",\"default\":false}]},{\"type\":\"model\",\"name\":\"TemplateActions1\",\"properties\":[{\"name\":\"open_modal\",\"kind\":\"Any\",\"default\":0},{\"name\":\"close_modal\",\"kind\":\"Any\",\"default\":0}]},{\"type\":\"model\",\"name\":\"BootstrapTemplateActions1\",\"properties\":[{\"name\":\"open_modal\",\"kind\":\"Any\",\"default\":0},{\"name\":\"close_modal\",\"kind\":\"Any\",\"default\":0}]},{\"type\":\"model\",\"name\":\"TemplateEditor1\",\"properties\":[{\"name\":\"layout\",\"kind\":\"Any\",\"default\":[]}]},{\"type\":\"model\",\"name\":\"MaterialTemplateActions1\",\"properties\":[{\"name\":\"open_modal\",\"kind\":\"Any\",\"default\":0},{\"name\":\"close_modal\",\"kind\":\"Any\",\"default\":0}]},{\"type\":\"model\",\"name\":\"copy_to_clipboard1\",\"properties\":[{\"name\":\"fill\",\"kind\":\"Any\",\"default\":\"none\"},{\"name\":\"value\",\"kind\":\"Any\",\"default\":null}]}]}};\n",
" var render_items = [{\"docid\":\"01e43340-2e4a-4e82-b3de-797c2c99037c\",\"roots\":{\"766cef6d-5edf-4a28-802f-c35e5727f258\":\"b7f8c7ed-8bfe-4cd8-8cc1-c5ac1dbb13b4\"},\"root_ids\":[\"766cef6d-5edf-4a28-802f-c35e5727f258\"]}];\n",
" var docs = Object.values(docs_json)\n",
" if (!docs) {\n",
" return\n",
Expand Down Expand Up @@ -128,7 +128,7 @@
},
"metadata": {
"application/vnd.holoviews_exec.v0+json": {
"id": "f8973921-5af7-4118-8b9f-306702128fe6"
"id": "766cef6d-5edf-4a28-802f-c35e5727f258"
}
},
"output_type": "display_data"
Expand Down Expand Up @@ -158,7 +158,7 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -169,7 +169,7 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -200,21 +200,21 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "5338ba1fd81542dfa60a9e4d7da9fdbf",
"model_id": "3a5ec083483b4637afb5e90f7da83de0",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"BokehModel(combine_events=True, render_bundle={'docs_json': {'aafb62c3-8afe-4377-9ef6-b72eb84daa71': {'version…"
"BokehModel(combine_events=True, render_bundle={'docs_json': {'f86ee677-8c32-47f1-90ec-71a89a2c7bec': {'version…"
]
},
"execution_count": 4,
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -233,14 +233,14 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Selected IDs: [268, 261, 264, 266, 282]\n"
"Selected IDs: [260, 264]\n"
]
}
],
Expand All @@ -250,36 +250,92 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Processing model 100k\n",
"Processing model 200k\n",
"Processing model 400k\n",
"Processing model 800k\n",
"Processing model 1.6m\n",
"Processing model 3.2m\n",
"Processing model 6.4m\n",
"Processing model 12.8m\n",
"Processing model 25.6m\n"
]
}
],
"outputs": [],
"source": [
"# TODO: remove token_groups\n",
"token_groups = {id: {\"selected\": id in selected_ids} for id in vocab.values()}\n",
"model_group_stats = calc_model_group_stats(\n",
" list(tokenized_corpus_dataset), next_logprobs_plot, token_groups, [\"selected\"]\n",
" list(tokenized_corpus_dataset), next_logprobs_plot, selected_ids\n",
")\n",
"\n",
"assert (\n",
" model_group_stats != {}\n",
") # sometimes tokens can't be found in the dataset and the stats will be empty"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'100k': {'mean': -1.070133,\n",
" 'median': -0.6912808,\n",
" 'min': -7.122874,\n",
" 'max': -0.017355476,\n",
" '25th': -1.3876090049743652,\n",
" '75th': -0.31879591941833496},\n",
" '200k': {'mean': -1.0078776,\n",
" 'median': -0.6108246,\n",
" 'min': -7.1288815,\n",
" 'max': -0.006140296,\n",
" '25th': -1.3651126325130463,\n",
" '75th': -0.21433717757463455},\n",
" '400k': {'mean': -0.8517932,\n",
" 'median': -0.5542941,\n",
" 'min': -6.2654996,\n",
" 'max': -0.0039506494,\n",
" '25th': -1.0751871466636658,\n",
" '75th': -0.13063477724790573},\n",
" '800k': {'mean': -0.78640485,\n",
" 'median': -0.31092834,\n",
" 'min': -6.6738915,\n",
" 'max': -0.0011469699,\n",
" '25th': -1.117132544517517,\n",
" '75th': -0.11057000048458576},\n",
" '1.6m': {'mean': -0.74975806,\n",
" 'median': -0.30155045,\n",
" 'min': -5.3355055,\n",
" 'max': -0.00043644916,\n",
" '25th': -1.0707703530788422,\n",
" '75th': -0.057139165699481964},\n",
" '3.2m': {'mean': -0.69542694,\n",
" 'median': -0.263493,\n",
" 'min': -4.481785,\n",
" 'max': -0.00014411364,\n",
" '25th': -1.0095961689949036,\n",
" '75th': -0.039097873494029045},\n",
" '6.4m': {'mean': -0.60625404,\n",
" 'median': -0.19129953,\n",
" 'min': -5.051317,\n",
" 'max': -7.00926e-05,\n",
" '25th': -0.804155021905899,\n",
" '75th': -0.028934753965586424},\n",
" '12.8m': {'mean': -0.56314814,\n",
" 'median': -0.13154678,\n",
" 'min': -4.793927,\n",
" 'max': -1.2159274e-05,\n",
" '25th': -0.8005392700433731,\n",
" '75th': -0.01866082102060318},\n",
" '25.6m': {'mean': -0.56998307,\n",
" 'median': -0.091308385,\n",
" 'min': -4.9958663,\n",
" 'max': -1.0967195e-05,\n",
" '25th': -0.577660083770752,\n",
" '75th': -0.006869094213470817}}"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model_group_stats"
]
},
{
"cell_type": "code",
"execution_count": 8,
Expand Down
Loading

0 comments on commit 367d0d8

Please sign in to comment.