categorical-catboost-pool-cv-bayes-opt.ipynb

{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Categorical Catboost Pool CV\n_By Nick Brooks, 2019-10-30_\n\n**Goal:** <br>\nSince Catboost has fancy methods to handle categorical data known as [Quantization](https://catboost.ai/docs/concepts/quantization.html). Lets see how it does and checkout some new catboost features while were at it."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport catboost\nfrom sklearn import preprocessing\nfrom contextlib import contextmanager\nimport time\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport gc\n\nfrom bayes_opt import BayesianOptimization\nimport warnings\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        \nfrom catboost.utils import get_gpu_device_count\nprint('\\n%i GPU devices available' % get_gpu_device_count())\n\n# Any results you write to the current directory are saved as output.\nprint(catboost.__version__)\n\nnotebookstart = time.time()\nseed = 25\n\ntop_boosting_rounds = 9000\nearly_stopping_rounds = 100","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@contextmanager\ndef timer(name):\n    \"\"\"\n    Time Each Process\n    \"\"\"\n    t0 = time.time()\n    yield\n    print('\\n[{}] done in {} Minutes'.format(name, round((time.time() - t0)/60,2)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with timer(\"Load\"):\n    nrow = None\n\n    train = pd.read_csv(\"/kaggle/input/cat-in-the-dat/train.csv\", index_col = 'id', nrows = nrow)\n    test = pd.read_csv(\"/kaggle/input/cat-in-the-dat/test.csv\", index_col = 'id')\n    submission_df = pd.read_csv(\"/kaggle/input/cat-in-the-dat/sample_submission.csv\")\n    [print(x.shape) for x in [train, test, submission_df]]\n\n    traindex = train.index\n    testdex = test.index\n\n    y = train.target.copy()\n\n    df = pd.concat([train.drop('target',axis = 1), test], axis = 0)\n    del train, test, submission_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with timer(\"Categorical Processing\"):\n    categorical = df.columns\n    # Encoder:\n    for col in categorical:\n        diff = list(set(df.loc[testdex, col].unique()) - set(df.loc[traindex,col].unique()))\n        if diff:\n            print(\"Column {} has {} unseen categories in test set\".format(col, len(diff)))\n            df.loc[df[col].isin(diff),col] = 999\n        if df[col].dtypes == object:\n            df[col] = df[col].astype(str)\n        lbl = preprocessing.LabelEncoder()\n        df[col] = pd.Series(lbl.fit_transform(df[col].values)).astype('category')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## CV Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Prepare Data Object\ncategorical_index = list(range(0, len(categorical)))\nfeatures_names = df.columns\n\ncatboost_pool = catboost.Pool(df.loc[traindex,:],\n    label=y,\n    cat_features=categorical_index)\n\ntest_pool = catboost.Pool(data=df.loc[testdex,:],\n    cat_features = categorical_index)\n\ndel df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def catboost_blackbox(max_depth, reg_lambda):\n    # num_leaves removed\n    param = {\n        'learning_rate': 0.2,\n        'bagging_temperature': 0.1, \n        'l2_leaf_reg': reg_lambda,\n        'depth': int(max_depth), \n#         'max_leaves': int(num_leaves),\n#         'max_bin':255,\n        'iterations' : top_boosting_rounds,\n        'task_type':'GPU',\n#         'grow_policy': 'Lossguide '\n        'loss_function' : \"Logloss\",\n        'objective':'Logloss',\n        'eval_metric' : \"AUC\",\n        'bootstrap_type' : 'Bayesian',\n        'random_seed': seed,\n        'early_stopping_rounds' : early_stopping_rounds,\n        'use_best_model': False,\n        \"verbose\": False\n    }\n    \n    modelstart= time.time()\n    scores = catboost.cv(catboost_pool,\n                param,\n                fold_count = 2,\n                stratified = True,\n                shuffle = True,\n                partition_random_seed = seed,\n                plot = False\n                )\n    runtime = (time.time() - modelstart)/60\n    \n    optimise = scores.loc[scores['test-AUC-mean'].idxmax(),'test-AUC-mean'] - scores.loc[scores['test-AUC-mean'].idxmax(),'test-AUC-std']\n    optimisation_info.append([scores['test-AUC-mean'].idxmax(), optimise, runtime, param, scores['test-AUC-mean'].idxmax()])\n    \n    \n    return optimise","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"parameter_bounds = {\n#     'num_leaves': (31, 500), \n    'reg_lambda': (0.1, 10),\n    'max_depth':(3,16)\n}\n\ninit_points = 2\nn_iter = 8\n\noptimisation_info = []\nCATBOOST_BO = BayesianOptimization(catboost_blackbox,\n                                   parameter_bounds,\n                                   random_state=seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with timer(\"Bayesian Optimisation - {} Iterations\".format(init_points + n_iter)):\n    with warnings.catch_warnings():\n        warnings.filterwarnings('ignore')\n        CATBOOST_BO.maximize(init_points = init_points,\n                             n_iter = n_iter,\n                             acq = 'ucb',\n                             xi = 0.0,\n                             alpha = 1e-6)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Best Score: {}\".format(CATBOOST_BO.max['target']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CATBOOST_BO.max['params']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Lower Learning Rate with Best Parameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"optimisation_pd = pd.DataFrame(optimisation_info, columns = ['Best Round', 'Score', 'Runtime','Param', 'Iterations'])\noptimisation_pd.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"optimisation_pd.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_param = optimisation_pd.loc[optimisation_pd['Score'].idxmax(),'Param']\nbest_param['iterations'] = top_boosting_rounds*3\nbest_param['learning_rate'] = 0.04\nbest_param['early_stopping_rounds'] = early_stopping_rounds\n\nbest_param","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with timer(\"Catboost CV\"):\n    scores = catboost.cv(catboost_pool,\n                best_param,\n                fold_count = 3,\n                stratified = True,\n                partition_random_seed = seed,\n                plot = True,\n                shuffle = True,\n                )\n\ndisplay(scores.tail())\n\nfig, axes = plt.subplots(nrows=1, ncols=2, figsize=(12, 4))\naxes[0].plot(scores['iterations'],scores['test-Logloss-mean'], label='test-Logloss-mean')\naxes[0].plot(scores['iterations'],scores['train-Logloss-mean'], label='train-Logloss-mean')\naxes[0].legend()\n\naxes[1].plot(scores['iterations'],scores['test-AUC-mean'], label='validation_rocauc')\naxes[1].legend()\nplt.show()\n\nbest_iteration = scores['test-AUC-mean'].idxmax()\nprint(\"Best Iteration: {}\".format(best_iteration))\n\ndisplay(scores.loc[best_iteration,:])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## Single Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"with timer(\"Catboost Single Model\"):\n    best_param['iterations'] = best_iteration\n    model = catboost.CatBoostClassifier(**best_param)\n    model.fit(catboost_pool)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feat_imp = pd.DataFrame()\nfeat_imp['importance'] = model.get_feature_importance()\nfeat_imp['features'] = features_names\nfeat_imp.sort_values(by = 'importance', inplace = True, ascending = False)\n\nsns.barplot(y = feat_imp['features'], x = feat_imp['importance'])\nplt.title(\"Feature Importance\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cm = catboost.utils.get_confusion_matrix(model, catboost_pool)\nprint(cm)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"roc_curve_values = catboost.utils.get_roc_curve(model, catboost_pool, plot=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"(thresholds, fnr) = catboost.utils.get_fnr_curve(curve=roc_curve_values, plot=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"(thresholds, fpr) = catboost.utils.get_fpr_curve(curve=roc_curve_values, plot=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results = model.predict_proba(test_pool)[:, 1]\nsubmission = pd.DataFrame({'id': testdex, 'target': results})\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!head submission.csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Notebook Runtime: %0.2f Hours\"%((time.time() - notebookstart)/60/60))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}