pca-and-catboost-pool-parameter-search.ipynb

{"cells":[{"metadata":{"_uuid":"ea82ef3478dd222dfc9f9cbcab966017f9fa4b80"},"cell_type":"markdown","source":"#### Catboost Pool Parameter Search\n_By Nick Brooks, June 2018_"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\nimport time\nnotebookstart= time.time()\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport gc\n\n# Models Packages\nfrom sklearn import metrics\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import preprocessing\nfrom IPython.display import display\n\n#CatBoost\nimport hyperopt \nfrom catboost import Pool, CatBoostRegressor\nfrom sklearn.metrics import mean_squared_error\nfrom catboost import cv as catcv\n\n# Unsupervised Models\nfrom sklearn.decomposition import PCA, TruncatedSVD, FastICA\nfrom sklearn.random_projection import GaussianRandomProjection, SparseRandomProjection\n\n# Viz\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport re\n\n# Specify index/ target name\nid_col = \"ID\"\ntarget_var = \"target\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2a1ab1869fd84d67b49f07090ad3b1f2395b1ec7"},"cell_type":"code","source":"def get_data(Debug = False):\n    print(\"Load Data\")\n    nrows = None\n    if Debug is True: nrows= 500\n    train = pd.read_csv(\"../input/santander-value-prediction-challenge/train.csv\", index_col = id_col, nrows=nrows)\n    train[\"log_compiled_leak\"] = np.log1p(pd.read_csv(\"../input/breaking-lb-fresh-start-with-lag-selection/train_leak.csv\", nrows=nrows)[\"compiled_leak\"].values)\n    traindex = train.index\n    test_df = pd.read_csv(\"../input/santander-value-prediction-challenge/test.csv\", index_col = id_col, nrows=nrows)\n    test_df[\"log_compiled_leak\"] = np.log1p(pd.read_csv(\"../input/breaking-lb-fresh-start-with-lag-selection/test_leak.csv\", nrows=nrows)[\"compiled_leak\"].values)\n    testdex = test_df.index\n    y = np.log1p(train[target_var]).copy()\n    train.drop(target_var,axis=1,inplace=True)\n    print('Train shape: {} Rows, {} Columns'.format(*train.shape))\n    print('Test shape: {} Rows, {} Columns'.format(*test_df.shape))\n    \n    return train, traindex, test_df, testdex, y\ntrain, traindex, test_df, testdex, y = get_data(Debug = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b5d379fde25f8c36a56acec0bf0dd1be440c11a2"},"cell_type":"code","source":"print(\"Combine Train and Test\")\ndf = pd.concat([train,test_df],axis=0)\ndel train,test_df\ngc.collect()\nprint('\\nAll Data shape: {} Rows, {} Columns'.format(*df.shape))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9da1f70580c79fef3b45fe91596450f479286f99"},"cell_type":"markdown","source":"## Modeling Stage"},{"metadata":{"trusted":true,"_uuid":"d44917218679d320e76b93fd3d52307b68c91e08"},"cell_type":"code","source":"# Feature Names\nfeat_names = df.columns\n\n# Modeling Datasets\ntest_df = df.loc[testdex,:]\nX = df.loc[traindex,:]\nprint(\"Starting Catboost. Train shape: {}, Test shape: {}\".format(X.shape,test_df.shape))\nprint(\"Feature Num: \",len(feat_names))\n\n# Create a dataframe to save results from CV tuning\nresults = pd.DataFrame(columns = [\"Rounds\",\"Score\",\"STDV\", \"LB\", \"Parameters\"])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"211c2ab57c074bafb88d4aecbf97da2f0664cc74"},"cell_type":"markdown","source":"## Dimensionality Reduction\nNumber of components and method determined by my [notebook](https://www.kaggle.com/nicapotato/comparing-dimensional-reduction-methods-lr) comparing dimensionality methods."},{"metadata":{"trusted":true,"_uuid":"f6c166ae5f1f804ae7c6e5b5749e3973910c119f"},"cell_type":"code","source":"# pca = PCA(random_state=23, n_components = 186)\n# X = pca.fit_transform(X)\n# test_df = pca.transform(test_df)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3e3c86b9e6e189b0cc453c850e580489c1642009"},"cell_type":"markdown","source":"## Caboost CV"},{"metadata":{"_uuid":"698b4b1d1d344eed339b21b28bd8a1a1083d19f0"},"cell_type":"markdown","source":"The goal here is to use cross-validation to ensure that I have parameters that generalize well."},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"270d85aac04647a91ae36aa6b379904fc268ea54","collapsed":true},"cell_type":"code","source":"print(\"Parameter Tuning / Ideal Boosting Rounds\")\ncat_params = {\"eval_metric\":'RMSE',\n               \"iterations\": 4000,\n               \"od_wait\": 150,\n               \"random_seed\": 42,\n               \"logging_level\": \"Verbose\",\n               \"metric_period\": 75,\n               \"od_type\": 'Iter',\n               \"od_wait\": 100\n             }\nmodel = CatBoostRegressor(**cat_params)\n\n# Cross-Validation\ncatpool = Pool(X,y)\ncv_data = catcv(catpool, model.get_params(), fold_count=5)\n\noptimal_round = cv_data['test-RMSE-mean'].idxmin()\nprint(\"Best Iteration: \",optimal_round + 1)\nprint(\"Best Score: {} +/- {}\".format(cv_data['test-RMSE-mean'][optimal_round],cv_data['test-RMSE-std'][optimal_round]))\n\n# Append Scores\nresults = results.append({\"Rounds\": optimal_round,\n                          \"Score\": cv_data['test-RMSE-mean'][optimal_round],\n                          \"STDV\": cv_data['test-RMSE-std'][optimal_round],\n                          \"LB\": None,\n                          \"Parameters\": cat_params}, ignore_index=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e34e2432304071b538f9adace368d8558c12bbb8"},"cell_type":"markdown","source":"Now that the ideal parameters are found, I will run them on the full data and submit. Now, I there is always more tuning to be done, so fork the notebook and turn some parametric nobes! The results will be stored and track your progress with the table below."},{"metadata":{"trusted":true,"_uuid":"4d6158137bb38135821384e04486e5d73ac8aa68","collapsed":true},"cell_type":"code","source":"pd.set_option('max_colwidth', 800)\ndisplay(results)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"64bd8c43528b774d808cc1ef588163199bde2184"},"cell_type":"markdown","source":"## Catboost Submission Model [Optimal Rounds]"},{"metadata":{"trusted":true,"_uuid":"a5c81da02b3d4c25b33b88fc1ce78947259108c3","collapsed":true},"cell_type":"code","source":"print(\"Train Submission Model\")\ncat_params[\"iterations\"] = optimal_round + 1\nmodel = CatBoostRegressor(**cat_params)\nmodel.fit(X,y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"079b6baf321a57a4ca85e17159a28b1c4e7fd700","collapsed":true},"cell_type":"code","source":"catpred = np.expm1(model.predict(test_df))\nsubmission = pd.DataFrame({'ID':testdex,'target':catpred})\nsubmission.to_csv('catboost.csv',index=False)\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"8c373fea5ce979d5c65a9516ee22152b897b4241"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}