notebooks/_old_train_catboost.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8f61e1a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import sys\n",
    "import polars as pl\n",
    "import json\n",
    "from joblib import Parallel, delayed\n",
    "import deepchem\n",
    "from tqdm import tqdm\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "# Configure Polars \n",
    "cfg = pl.Config()\n",
    "cfg.set_tbl_rows(20)\n",
    "cfg.set_tbl_cols(50)\n",
    "from sklearn.model_selection import StratifiedGroupKFold, GroupKFold, KFold, GroupShuffleSplit, ShuffleSplit\n",
    "import gc\n",
    "import random\n",
    "from functools import partial\n",
    "import catboost\n",
    "from catboost import FeaturesData, Pool, CatBoost, CatBoostClassifier, CatBoostRegressor, CatBoostRanker\n",
    "import time\n",
    "from sklearn.metrics import average_precision_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1f1602bb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "NUM_TRAIN = 400_000_000\n",
    "SEED = 42\n",
    "FROM_GLOBAL_POOL = False\n",
    "# POS_WEIGHT = 1 / 0.015779035993603314\n",
    "POS_WEIGHT = 10\n",
    "POS_WEIGHT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76fe1280",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2c3925be",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(66710159, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>_index</th><th>index</th><th>label</th></tr><tr><td>u32</td><td>i64</td><td>i64</td></tr></thead><tbody><tr><td>0</td><td>363110</td><td>0</td></tr><tr><td>1</td><td>363118</td><td>0</td></tr><tr><td>2</td><td>363119</td><td>0</td></tr><tr><td>3</td><td>363122</td><td>0</td></tr><tr><td>4</td><td>363123</td><td>0</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 3)\n",
       "┌────────┬────────┬───────┐\n",
       "│ _index ┆ index  ┆ label │\n",
       "│ ---    ┆ ---    ┆ ---   │\n",
       "│ u32    ┆ i64    ┆ i64   │\n",
       "╞════════╪════════╪═══════╡\n",
       "│ 0      ┆ 363110 ┆ 0     │\n",
       "│ 1      ┆ 363118 ┆ 0     │\n",
       "│ 2      ┆ 363119 ┆ 0     │\n",
       "│ 3      ┆ 363122 ┆ 0     │\n",
       "│ 4      ┆ 363123 ┆ 0     │\n",
       "└────────┴────────┴───────┘"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv_train_df = pl.scan_csv('/home/dangnh36/datasets/competitions/leash_belka/processed/cv/v1/train.csv').with_row_index('_index').collect()\n",
    "print(cv_train_df.shape)\n",
    "cv_train_df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7f1c7e4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8a4e6fb8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(31705451, 4)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 4)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>_index</th><th>index</th><th>label</th><th>subset</th></tr><tr><td>u32</td><td>i64</td><td>i64</td><td>i64</td></tr></thead><tbody><tr><td>0</td><td>0</td><td>0</td><td>1</td></tr><tr><td>1</td><td>1</td><td>0</td><td>1</td></tr><tr><td>2</td><td>2</td><td>0</td><td>1</td></tr><tr><td>3</td><td>3</td><td>0</td><td>1</td></tr><tr><td>4</td><td>4</td><td>0</td><td>1</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 4)\n",
       "┌────────┬───────┬───────┬────────┐\n",
       "│ _index ┆ index ┆ label ┆ subset │\n",
       "│ ---    ┆ ---   ┆ ---   ┆ ---    │\n",
       "│ u32    ┆ i64   ┆ i64   ┆ i64    │\n",
       "╞════════╪═══════╪═══════╪════════╡\n",
       "│ 0      ┆ 0     ┆ 0     ┆ 1      │\n",
       "│ 1      ┆ 1     ┆ 0     ┆ 1      │\n",
       "│ 2      ┆ 2     ┆ 0     ┆ 1      │\n",
       "│ 3      ┆ 3     ┆ 0     ┆ 1      │\n",
       "│ 4      ┆ 4     ┆ 0     ┆ 1      │\n",
       "└────────┴───────┴───────┴────────┘"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv_val_df = pl.scan_csv('/home/dangnh36/datasets/competitions/leash_belka/processed/cv/v1/val.csv').with_row_index('_index').collect()\n",
    "print(cv_val_df.shape)\n",
    "cv_val_df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c9d48b10",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((299837,), (299837,))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sampled_val_idxs = cv_val_df.filter(pl.col('subset').is_in([0,2,3]))['index'].to_numpy()\n",
    "sampled_val_labels = cv_val_df.filter(pl.col('subset').is_in([0,2,3]))['label'].to_numpy()\n",
    "sampled_val_idxs.shape, sampled_val_labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c0a4283",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b5f60cf2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(66710159,)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if NUM_TRAIN < cv_train_df.shape[0]:\n",
    "    pos_idxs = cv_train_df.filter(pl.col('label') > 0)['_index'].to_numpy()\n",
    "    neg_idxs = cv_train_df.filter(pl.col('label') == 0)['_index'].to_numpy()\n",
    "    print('Pos/Neg num:', pos_idxs.shape, neg_idxs.shape)\n",
    "    np.random.seed(SEED)\n",
    "    sampled_neg_idxs = np.random.choice(neg_idxs, NUM_TRAIN - len(pos_idxs), replace = False)\n",
    "    _sampled_idxs = np.concatenate([pos_idxs, sampled_neg_idxs], axis = 0)\n",
    "else:\n",
    "    _sampled_idxs = cv_train_df['_index'].to_numpy()\n",
    "_sampled_idxs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dbf2d727",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "986d8afe",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c02de98e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8f0e7188",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NUM SAMPLED TRAIN: (66710159,)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.015779035993603314"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if FROM_GLOBAL_POOL:\n",
    "    sampled_train_idxs = cv_train_df[_sampled_idxs, 'index'].to_numpy()\n",
    "    sampled_train_labels = cv_train_df[_sampled_idxs, 'label'].to_numpy()\n",
    "else:\n",
    "    sampled_train_idxs = _sampled_idxs\n",
    "    sampled_train_labels = cv_train_df['label'].to_numpy()\n",
    "    \n",
    "# get some statistic\n",
    "print('NUM SAMPLED TRAIN:', sampled_train_idxs.shape)\n",
    "(cv_train_df[_sampled_idxs, 'label']!=0).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8cafd0cb",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'cv_train_df' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[9], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mdel\u001b[39;00m cv_train_df, cv_val_df\n\u001b[1;32m      2\u001b[0m gc\u001b[38;5;241m.\u001b[39mcollect()\n",
      "\u001b[0;31mNameError\u001b[0m: name 'cv_train_df' is not defined"
     ]
    }
   ],
   "source": [
    "del cv_train_df, cv_val_df\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "605916e6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f959122",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "91c50a63",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9cd3cd1c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "416a8457",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(66710159, 256)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if FROM_GLOBAL_POOL:\n",
    "    train_ds = '/home/dangnh36/datasets/competitions/leash_belka/processed/datasets/catboost/ecfp6_packbits/train.tsv'\n",
    "    train_cd = '/home/dangnh36/datasets/competitions/leash_belka/processed/datasets/catboost/ecfp6_packbits/train_num.cd'\n",
    "else:\n",
    "    train_ds = '/home/dangnh36/datasets/competitions/leash_belka/processed/datasets/catboost/ecfp6_packbits/_train_66M.tsv'\n",
    "    train_cd = '/home/dangnh36/datasets/competitions/leash_belka/processed/datasets/catboost/ecfp6_packbits/train_num.cd'\n",
    "\n",
    "global_pool = Pool(\n",
    "    train_ds,\n",
    "    column_description=train_cd,\n",
    "    has_header = False,\n",
    "    thread_count = -1,\n",
    ")\n",
    "global_pool.num_row(), global_pool.num_col()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a81191e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "22f1f0aa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "66710159 256\n"
     ]
    }
   ],
   "source": [
    "train_pool = global_pool.slice(sampled_train_idxs)\n",
    "gc.collect()\n",
    "print(train_pool.num_row(), train_pool.num_col())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c8a40383",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cd026ee0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d46f3709",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e9433cfd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ab67fb3e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "367ae351",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "1bcc2df4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# val_pool = global_pool.slice(sampled_val_idxs)\n",
    "# gc.collect()\n",
    "# print(val_pool.num_row(), val_pool.num_col())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8742cb8a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd3e4843",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "6cc796a9",
   "metadata": {},
   "source": [
    "# 5M"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e1460cdc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_ds = '/home/dangnh36/datasets/competitions/leash_belka/processed/datasets/catboost/ecfp6/_train_5M.tsv'\n",
    "# train_cd = '/home/dangnh36/datasets/competitions/leash_belka/processed/datasets/catboost/ecfp6/train_ctr.cd'\n",
    "\n",
    "# train_pool = Pool(\n",
    "#     train_ds,\n",
    "#     column_description=train_cd,\n",
    "#     has_header = False,\n",
    "#     thread_count = -1,\n",
    "# )\n",
    "# print(train_pool.num_row(), train_pool.num_col())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0b577af5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "299837 256\n"
     ]
    }
   ],
   "source": [
    "val_ds = '/home/dangnh36/datasets/competitions/leash_belka/processed/datasets/catboost/ecfp6_packbits/_val_300k.tsv'\n",
    "val_cd = '/home/dangnh36/datasets/competitions/leash_belka/processed/datasets/catboost/ecfp6_packbits/train_num.cd'\n",
    "\n",
    "val_pool = Pool(\n",
    "    val_ds,\n",
    "    column_description=val_cd,\n",
    "    has_header = False,\n",
    "    thread_count = -1,\n",
    ")\n",
    "print(val_pool.num_row(), val_pool.num_col())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "77c35e07",
   "metadata": {},
   "source": [
    "# End 5M"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "496174f9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "514"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fc6e1500",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "6d74dc52",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # DEL GLOBAL POOL IF NEEDED\n",
    "# del global_pool\n",
    "# gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9361b0b4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73009984",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "84a9766f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "b53db8a0",
   "metadata": {},
   "source": [
    "- 3 gpus, 0.95 -> ok\n",
    "- 3 gpus, 0.8 -> ok\n",
    "- 3 gpus, 0.5 -> not enough\n",
    "- 3 gpus (1 with 10 GB already), 0.3 -> die\n",
    "- 4 gpus (1 with 10 GB), 0.95 -> not enough (Error: not enough memory for learning)\n",
    "- 4 gpus (1 with 10 GB already), 0.8 -> die"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b0940bc5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ori_train_labels = train_pool.get_label()\n",
    "# ori_val_labels = val_pool.get_label()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "16d80fff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.015779035993603314, 0.01364741509553524)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(sampled_train_labels > 0).mean(), (sampled_val_labels > 0).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "ad99d2a6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<catboost.core.Pool at 0x7fc25c3ee500>"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# train_pool.set_weight((sampled_train_labels > 0).astype(np.float32) * (POS_WEIGHT - 1) + 1) \n",
    "# val_pool.set_weight((sampled_val_labels > 0).astype(np.float32) * (POS_WEIGHT - 1) + 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "7c212206",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<catboost.core.Pool at 0x7fc25c3ee500>"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_pool.set_weight(np.ones_like(sampled_train_labels)) \n",
    "val_pool.set_weight(np.ones_like(sampled_val_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "b47aacc6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss_function': 'MultiLogloss', 'task_type': 'GPU', 'devices': '1:2:3', 'n_estimators': 4000, 'random_state': 42}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "17bedaaf9d49430798c76d82c54445b6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "MetricVisualizer(layout=Layout(align_self='stretch', height='500px'))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learning rate set to 0.015603\n",
      "0:\tlearn: 0.6490300\ttest: 0.6489748\tbest: 0.6489748 (0)\ttotal: 946ms\tremaining: 1h 3m 2s\n",
      "100:\tlearn: 0.0281296\ttest: 0.0290355\tbest: 0.0290355 (100)\ttotal: 1m 37s\tremaining: 1h 2m 41s\n",
      "200:\tlearn: 0.0234446\ttest: 0.0253229\tbest: 0.0253229 (200)\ttotal: 2m 56s\tremaining: 55m 37s\n",
      "300:\tlearn: 0.0220581\ttest: 0.0247110\tbest: 0.0247110 (300)\ttotal: 4m 23s\tremaining: 53m 56s\n",
      "400:\tlearn: 0.0212358\ttest: 0.0243995\tbest: 0.0243995 (400)\ttotal: 5m 46s\tremaining: 51m 49s\n",
      "500:\tlearn: 0.0206168\ttest: 0.0241788\tbest: 0.0241788 (500)\ttotal: 7m 9s\tremaining: 49m 59s\n",
      "600:\tlearn: 0.0200504\ttest: 0.0240080\tbest: 0.0240080 (600)\ttotal: 8m 32s\tremaining: 48m 17s\n",
      "700:\tlearn: 0.0196320\ttest: 0.0238578\tbest: 0.0238578 (700)\ttotal: 9m 51s\tremaining: 46m 24s\n",
      "800:\tlearn: 0.0192623\ttest: 0.0236834\tbest: 0.0236834 (800)\ttotal: 11m 12s\tremaining: 44m 45s\n",
      "900:\tlearn: 0.0189568\ttest: 0.0235566\tbest: 0.0235566 (900)\ttotal: 12m 30s\tremaining: 43m\n",
      "1000:\tlearn: 0.0187096\ttest: 0.0234779\tbest: 0.0234739 (994)\ttotal: 13m 50s\tremaining: 41m 27s\n",
      "1100:\tlearn: 0.0185030\ttest: 0.0234061\tbest: 0.0234061 (1100)\ttotal: 15m 9s\tremaining: 39m 53s\n",
      "1200:\tlearn: 0.0183160\ttest: 0.0233547\tbest: 0.0233547 (1200)\ttotal: 16m 29s\tremaining: 38m 25s\n",
      "1300:\tlearn: 0.0181290\ttest: 0.0232889\tbest: 0.0232881 (1297)\ttotal: 17m 50s\tremaining: 37m\n",
      "1400:\tlearn: 0.0179520\ttest: 0.0232288\tbest: 0.0232288 (1400)\ttotal: 19m 11s\tremaining: 35m 35s\n",
      "1500:\tlearn: 0.0177853\ttest: 0.0232175\tbest: 0.0232175 (1500)\ttotal: 20m 30s\tremaining: 34m 8s\n",
      "1600:\tlearn: 0.0176249\ttest: 0.0231995\tbest: 0.0231982 (1597)\ttotal: 21m 50s\tremaining: 32m 44s\n",
      "1700:\tlearn: 0.0174717\ttest: 0.0231786\tbest: 0.0231762 (1696)\ttotal: 23m 12s\tremaining: 31m 21s\n",
      "1800:\tlearn: 0.0173312\ttest: 0.0231692\tbest: 0.0231626 (1738)\ttotal: 24m 31s\tremaining: 29m 56s\n",
      "1900:\tlearn: 0.0171912\ttest: 0.0231564\tbest: 0.0231507 (1857)\ttotal: 25m 51s\tremaining: 28m 33s\n",
      "2000:\tlearn: 0.0170629\ttest: 0.0231560\tbest: 0.0231507 (1857)\ttotal: 27m 11s\tremaining: 27m 9s\n",
      "2100:\tlearn: 0.0169506\ttest: 0.0231499\tbest: 0.0231472 (2095)\ttotal: 28m 30s\tremaining: 25m 45s\n",
      "2200:\tlearn: 0.0168457\ttest: 0.0231367\tbest: 0.0231351 (2180)\ttotal: 29m 48s\tremaining: 24m 22s\n",
      "2300:\tlearn: 0.0167406\ttest: 0.0231284\tbest: 0.0231282 (2281)\ttotal: 31m 7s\tremaining: 22m 59s\n",
      "2400:\tlearn: 0.0166482\ttest: 0.0231127\tbest: 0.0231115 (2398)\ttotal: 32m 27s\tremaining: 21m 37s\n",
      "2500:\tlearn: 0.0165590\ttest: 0.0231098\tbest: 0.0231065 (2489)\ttotal: 33m 46s\tremaining: 20m 14s\n",
      "2600:\tlearn: 0.0164733\ttest: 0.0231120\tbest: 0.0231065 (2489)\ttotal: 35m 4s\tremaining: 18m 51s\n",
      "2700:\tlearn: 0.0163936\ttest: 0.0231184\tbest: 0.0231065 (2489)\ttotal: 36m 19s\tremaining: 17m 28s\n",
      "2800:\tlearn: 0.0163163\ttest: 0.0231061\tbest: 0.0231037 (2795)\ttotal: 37m 35s\tremaining: 16m 5s\n",
      "2900:\tlearn: 0.0162363\ttest: 0.0230807\tbest: 0.0230804 (2889)\ttotal: 38m 54s\tremaining: 14m 44s\n",
      "3000:\tlearn: 0.0161571\ttest: 0.0230843\tbest: 0.0230804 (2889)\ttotal: 40m 11s\tremaining: 13m 22s\n",
      "3100:\tlearn: 0.0160893\ttest: 0.0230841\tbest: 0.0230804 (2889)\ttotal: 41m 30s\tremaining: 12m 1s\n",
      "3200:\tlearn: 0.0160183\ttest: 0.0230814\tbest: 0.0230804 (2889)\ttotal: 42m 47s\tremaining: 10m 40s\n",
      "3300:\tlearn: 0.0159471\ttest: 0.0230712\tbest: 0.0230700 (3292)\ttotal: 44m 6s\tremaining: 9m 20s\n",
      "3400:\tlearn: 0.0158806\ttest: 0.0230788\tbest: 0.0230694 (3312)\ttotal: 45m 24s\tremaining: 7m 59s\n",
      "3500:\tlearn: 0.0158205\ttest: 0.0230772\tbest: 0.0230694 (3312)\ttotal: 46m 41s\tremaining: 6m 39s\n",
      "3600:\tlearn: 0.0157593\ttest: 0.0230793\tbest: 0.0230694 (3312)\ttotal: 48m\tremaining: 5m 19s\n",
      "3700:\tlearn: 0.0157031\ttest: 0.0230764\tbest: 0.0230694 (3312)\ttotal: 49m 16s\tremaining: 3m 58s\n",
      "3800:\tlearn: 0.0156435\ttest: 0.0230810\tbest: 0.0230694 (3312)\ttotal: 50m 36s\tremaining: 2m 38s\n",
      "3900:\tlearn: 0.0155915\ttest: 0.0230742\tbest: 0.0230694 (3312)\ttotal: 51m 52s\tremaining: 1m 18s\n",
      "3999:\tlearn: 0.0155374\ttest: 0.0230659\tbest: 0.0230648 (3967)\ttotal: 53m 9s\tremaining: 0us\n",
      "bestTest = 0.02306483007\n",
      "bestIteration = 3967\n",
      "Shrink model to first 3968 iterations.\n",
      "ALL PARAMS:\n",
      " {'nan_mode': 'Min', 'gpu_ram_part': 0.95, 'eval_metric': 'MultiLogloss', 'iterations': 4000, 'leaf_estimation_method': 'Newton', 'observations_to_bootstrap': 'TestOnly', 'random_score_type': 'NormalWithModelSizeDecrease', 'grow_policy': 'SymmetricTree', 'penalties_coefficient': 1, 'boosting_type': 'Plain', 'feature_border_type': 'GreedyLogSum', 'bayesian_matrix_reg': 0.10000000149011612, 'devices': '1:2:3', 'eval_fraction': 0, 'pinned_memory_bytes': '104857600', 'force_unit_auto_pair_weights': False, 'l2_leaf_reg': 3, 'random_strength': 1, 'rsm': 1, 'boost_from_average': False, 'gpu_cat_features_storage': 'GpuRam', 'fold_size_loss_normalization': False, 'model_size_reg': 0.5, 'pool_metainfo_options': {'tags': {}}, 'use_best_model': True, 'meta_l2_frequency': 0, 'class_names': [0, 1, 2], 'random_seed': 42, 'depth': 6, 'border_count': 128, 'min_fold_size': 100, 'data_partition': 'DocParallel', 'bagging_temperature': 1, 'classes_count': 0, 'auto_class_weights': 'None', 'leaf_estimation_backtracking': 'AnyImprovement', 'best_model_min_trees': 1, 'min_data_in_leaf': 1, 'add_ridge_penalty_to_loss_function': False, 'loss_function': 'MultiLogloss', 'learning_rate': 0.015603000298142431, 'meta_l2_exponent': 1, 'score_function': 'Cosine', 'task_type': 'GPU', 'leaf_estimation_iterations': 10, 'bootstrap_type': 'Bayesian', 'max_leaves': 64}\n",
      "CPU times: user 5h 33min 34s, sys: 37min 20s, total: 6h 10min 55s\n",
      "Wall time: 54min 59s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "lr = None\n",
    "model = CatBoostClassifier(task_type='GPU',\n",
    "                           devices='1:2:3',\n",
    "#                            loss_function = 'MultiCrossEntropy',\n",
    "                           loss_function = 'MultiLogloss',\n",
    "                           n_estimators = 4000,\n",
    "                           learning_rate = lr,\n",
    "                           random_state = 42,\n",
    "#                            depth = 12,\n",
    "#                            auto_class_weights='Balanced',\n",
    "#                            scale_pos_weight = 1.2,\n",
    "#                            class_weights=[2.0, 1.0],\n",
    "                           # max_ctr_complexity = None,\n",
    "#                            eval_metrics = \n",
    "#                            used_ram_limit = '180GB',\n",
    "#                            gpu_ram_part = 0.3,\n",
    "#                            max_ctr_complexity = 1,\n",
    "#                            gpu_cat_features_storage = 'CpuPinnedMemory',\n",
    "                          )\n",
    "print(model.get_params())\n",
    "\n",
    "model.fit(\n",
    "    train_pool,\n",
    "    verbose=100,\n",
    "    eval_set=val_pool,\n",
    "    plot = True\n",
    ")\n",
    "\n",
    "print('ALL PARAMS:\\n', model.get_all_params())\n",
    "\n",
    "SAVE_PATH = f'../runs/ckpts/model_256_full_weight=63_lr={lr}.cbm'\n",
    "model.save_model(SAVE_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "740f7c02",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6fbeefa7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "704ddbae",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2de6e699",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15d07c78",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "48577e11",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7f1edbc9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<catboost.core.CatBoostClassifier at 0x7fb79c278220>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = CatBoostClassifier()\n",
    "# model.load_model(SAVE_PATH)\n",
    "model.load_model(f'../runs/ckpts/model_256_full_weight=63_lr={None}.cbm')\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "1bfc8a64",
   "metadata": {},
   "outputs": [],
   "source": [
    "# metrics = model.eval_metrics(val_pool,\n",
    "#              metrics = ['F1', 'Accuracy'],\n",
    "#              ntree_start=0,\n",
    "#              ntree_end=0,\n",
    "#              eval_period=1,\n",
    "#              thread_count=-1,\n",
    "#              )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "a5fff33f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pd.Series(metrics['F1:class=2']).hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e9984c40",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(299837, 3)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds = model.predict_proba(val_pool, verbose = 100)\n",
    "preds.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b64e9f4e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(299837, 3)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels = val_pool.get_label().astype(np.uint8)\n",
    "labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ebb0ed0f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.24414941127764045\n",
      "0.11894557382816726\n",
      "0.36578464871741667\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.27468052236519097"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for i in range(3):\n",
    "    print(average_precision_score(labels[:, i], preds[:, i], pos_label=1, average='micro'))\n",
    "\n",
    "average_precision_score(labels.reshape(-1), preds.reshape(-1), pos_label=1, average='micro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "daeb0356",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6db02e61",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb5796e3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "394f4921",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8a419924",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "835c426d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "488c701c",
   "metadata": {},
   "source": [
    "# Result\n",
    "\n",
    "- 256 + CTR + LR=0.1: 0.2851591290053086, 0.03271806002, 3996 (12 min + 60 GB)\n",
    "- 256 + NUM + LR=0.1: 0.28140081176626164, 0.03188184721 (3min32s + 10 GB)\n",
    "- 256 + NUM + LR=0.2 + 40M: 0.2569072157627742, 0.0228263346\n",
    "- 2048 + CTR + LR=0.2: 0.31463215756548685 (5mins + 90 GB)\n",
    "- 2048 + NUM + LR=0.2: 0.31528349357809027 (4min7s + 53 GB)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b83aba57",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1bb048f4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5096cf8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "22f08910",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fc8c9b12",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "99868fec",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d8a576e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "b48d95f9",
   "metadata": {},
   "source": [
    "# Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "00860fdc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# test_pool = Pool('/home/dangnh36/datasets/competitions/leash_belka/processed/datasets/catboost/ecfp6_packbits/test.tsv',\n",
    "#                   column_description='/home/dangnh36/datasets/competitions/leash_belka/processed/datasets/catboost/ecfp6_packbits/test_num.cd',\n",
    "#                   has_header = False,\n",
    "#                   thread_count = -1,\n",
    "#                  )\n",
    "# test_pool.num_row(), test_pool.num_col()\n",
    "\n",
    "test_pool = Pool('/home/dangnh36/datasets/competitions/leash_belka/processed/datasets/catboost/ecfp6/test.tsv',\n",
    "                  column_description='/home/dangnh36/datasets/competitions/leash_belka/processed/datasets/catboost/ecfp6/test_ctr.cd',\n",
    "                  has_header = False,\n",
    "                  thread_count = -1,\n",
    "                 )\n",
    "test_pool.num_row(), test_pool.num_col()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42909815",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8eb877ee",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bfacbf92",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c11584d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_preds = model.predict_proba(test_pool, verbose = 100)\n",
    "print('Mean preds:', test_preds.mean(axis = 0))\n",
    "test_preds.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0af0e9f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = pl.scan_csv('/home/dangnh36/datasets/competitions/leash_belka/processed/test_v3.csv').collect()\n",
    "test_ids = test_df[['id_BRD4', 'id_HSA', 'id_sEH']].to_numpy()\n",
    "assert(test_preds.shape==test_ids.shape)\n",
    "\n",
    "test_ids = test_ids.reshape(-1)\n",
    "test_preds = test_preds.reshape(-1)\n",
    "test_ids.shape, test_preds.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a42a97c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "mask = test_ids != 0 \n",
    "assert(mask.sum()==1674896)\n",
    "\n",
    "submit_df = pd.DataFrame({\n",
    "    'id':test_ids[mask],\n",
    "    'binds':test_preds[mask],\n",
    "})\n",
    "display(submit_df)\n",
    "submit_df.to_csv(f'../runs/submission.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "adffcd58",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "85e526bd",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}