gpyopt-hyperparameter-optimisation-gpu-lgbm.ipynb

{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Gpyopt Hyperparameter Optimisation - GPU LGBM\n_By Nick Brooks_\n\nV1 - 04/08/2019 - First Commit <br>\nV2 - 11/08/2019 - 0.4442 - Add Final Model Evaluation Plots / Hyperparameter Relationships / EDA on top features <br>\nV3 - 14/08/2019 - 0.4419 - Tried unlimited tree size with smaller leaf count. <br>\nV4 - 26/08/2019 - 0.4419 - Try higher jitter level. <br>\n\n**Aim:** <br>\nImplement a Bayesian Hyperparameter Optimisation Framework. Understand how the GPYOPT framework works, and understand its exploration vs. exploitation tradeoff.\n\n\n**Sources:** <br>\n[GPYOPT Documentation](https://buildmedia.readthedocs.org/media/pdf/gpyopt/latest/gpyopt.pdf) <br>\n[krasserm's Blog Post (Super Awesome)](http://krasserm.github.io/2018/03/21/bayesian-optimization/) <br>\n\n\n**Kaggle:** <br>\n[GPU Installation with Kirankunapuli](https://www.kaggle.com/kirankunapuli/ieee-fraud-lightgbm-with-gpu/comments) <br>\n[Vincent Model RoC/PR/Confusion Matrix Evaluation Plots](https://www.kaggle.com/vincentlugat/ieee-lgb-bayesian-opt) <br>\n[Feature Engineering](https://www.kaggle.com/davidcairuz/feature-engineering-lightgbm-w-gpu) <br>"},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"!rm -r /opt/conda/lib/python3.6/site-packages/lightgbm\n!git clone --recursive https://github.com/Microsoft/LightGBM\n!apt-get install -y -qq libboost-all-dev","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"%%bash\ncd LightGBM\nrm -r build\nmkdir build\ncd build\ncmake -DUSE_GPU=1 -DOpenCL_LIBRARY=/usr/local/cuda/lib64/libOpenCL.so -DOpenCL_INCLUDE_DIR=/usr/local/cuda/include/ ..\nmake -j$(nproc)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"!cd LightGBM/python-package/;python3 setup.py install --precompile","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"!mkdir -p /etc/OpenCL/vendors && echo \"libnvidia-opencl.so.1\" > /etc/OpenCL/vendors/nvidia.icd\n!rm -r LightGBM\n\n# Latest Pandas version\n!pip install -q 'pandas==0.25' --force-reinstall\n# Install Gpyopt\n!pip install GPyOpt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgb\nimport pandas as pd\nimport GPyOpt\nfrom GPyOpt.methods import BayesianOptimization\nprint(\"GPyOpt version:\", GPyOpt.__version__)\nprint(\"LGBM version:\", lgb.__version__)\nprint(\"Pandas version:\", pd.__version__)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import time\nnotebookstart = time.time()\n\nimport os\nfrom contextlib import contextmanager\nimport gc; gc.enable()\nimport pprint\n\nimport datetime\nimport csv\nimport random\n\nimport numpy as np\nfrom pandas.io.json import json_normalize\nfrom itertools import combinations\n\n# Viz\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Modeling\nfrom sklearn.model_selection import KFold, train_test_split, TimeSeriesSplit\nfrom sklearn import preprocessing\nfrom sklearn import metrics\nfrom sklearn.metrics import precision_score, recall_score, confusion_matrix, accuracy_score, roc_auc_score, f1_score, roc_curve, auc,precision_recall_curve\nfrom scipy import interp\nimport itertools\nimport warnings\n\nwarnings.filterwarnings(\"ignore\")\n\nseed = 24\nnp.random.seed(seed)\n\npd.set_option('display.max_columns', 500)\npd.options.display.max_rows = 999\npd.set_option('max_colwidth', 500)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"print(\"Define DF Schema..\")\n\ntarget_var = 'isFraud'\n\nschema = {\n    \"TransactionDT\":       \"int32\",\n    \"TransactionAmt\":    \"float32\",\n    \"ProductCD\":          \"object\",\n    \"card1\":               \"int16\",\n    \"card2\":             \"float32\",\n    \"card3\":             \"float32\",\n    \"card4\":              \"object\",\n    \"card5\":             \"float32\",\n    \"card6\":              \"object\",\n    \"addr1\":             \"float32\",\n    \"addr2\":             \"float32\",\n    \"dist1\":             \"float32\",\n    \"dist2\":             \"float32\",\n    \"P_emaildomain\":      \"object\",\n    \"R_emaildomain\":      \"object\",\n    \"C1\":                \"float32\",\n    \"C2\":                \"float32\",\n    \"C3\":                \"float32\",\n    \"C4\":                \"float32\",\n    \"C5\":                \"float32\",\n    \"C6\":                \"float32\",\n    \"C7\":                \"float32\",\n    \"C8\":                \"float32\",\n    \"C9\":                \"float32\",\n    \"C10\":               \"float32\",\n    \"C11\":               \"float32\",\n    \"C12\":               \"float32\",\n    \"C13\":               \"float32\",\n    \"C14\":               \"float32\",\n    \"D1\":                \"float32\",\n    \"D2\":                \"float32\",\n    \"D3\":                \"float32\",\n    \"D4\":                \"float32\",\n    \"D5\":                \"float32\",\n    \"D6\":                \"float32\",\n    \"D7\":                \"float32\",\n    \"D8\":                \"float32\",\n    \"D9\":                \"float32\",\n    \"D10\":               \"float32\",\n    \"D11\":               \"float32\",\n    \"D12\":               \"float32\",\n    \"D13\":               \"float32\",\n    \"D14\":               \"float32\",\n    \"D15\":               \"float32\",\n    \"M1\":                 \"object\",\n    \"M2\":                 \"object\",\n    \"M3\":                 \"object\",\n    \"M4\":                 \"object\",\n    \"M5\":                 \"object\",\n    \"M6\":                 \"object\",\n    \"M7\":                 \"object\",\n    \"M8\":                 \"object\",\n    \"M9\":                 \"object\",\n    \"V1\":                \"float32\",\n    \"V2\":                \"float32\",\n    \"V3\":                \"float32\",\n    \"V4\":                \"float32\",\n    \"V5\":                \"float32\",\n    \"V6\":                \"float32\",\n    \"V7\":                \"float32\",\n    \"V8\":                \"float32\",\n    \"V9\":                \"float32\",\n    \"V10\":               \"float32\",\n    \"V11\":               \"float32\",\n    \"V12\":               \"float32\",\n    \"V13\":               \"float32\",\n    \"V14\":               \"float32\",\n    \"V15\":               \"float32\",\n    \"V16\":               \"float32\",\n    \"V17\":               \"float32\",\n    \"V18\":               \"float32\",\n    \"V19\":               \"float32\",\n    \"V20\":               \"float32\",\n    \"V21\":               \"float32\",\n    \"V22\":               \"float32\",\n    \"V23\":               \"float32\",\n    \"V24\":               \"float32\",\n    \"V25\":               \"float32\",\n    \"V26\":               \"float32\",\n    \"V27\":               \"float32\",\n    \"V28\":               \"float32\",\n    \"V29\":               \"float32\",\n    \"V30\":               \"float32\",\n    \"V31\":               \"float32\",\n    \"V32\":               \"float32\",\n    \"V33\":               \"float32\",\n    \"V34\":               \"float32\",\n    \"V35\":               \"float32\",\n    \"V36\":               \"float32\",\n    \"V37\":               \"float32\",\n    \"V38\":               \"float32\",\n    \"V39\":               \"float32\",\n    \"V40\":               \"float32\",\n    \"V41\":               \"float32\",\n    \"V42\":               \"float32\",\n    \"V43\":               \"float32\",\n    \"V44\":               \"float32\",\n    \"V45\":               \"float32\",\n    \"V46\":               \"float32\",\n    \"V47\":               \"float32\",\n    \"V48\":               \"float32\",\n    \"V49\":               \"float32\",\n    \"V50\":               \"float32\",\n    \"V51\":               \"float32\",\n    \"V52\":               \"float32\",\n    \"V53\":               \"float32\",\n    \"V54\":               \"float32\",\n    \"V55\":               \"float32\",\n    \"V56\":               \"float32\",\n    \"V57\":               \"float32\",\n    \"V58\":               \"float32\",\n    \"V59\":               \"float32\",\n    \"V60\":               \"float32\",\n    \"V61\":               \"float32\",\n    \"V62\":               \"float32\",\n    \"V63\":               \"float32\",\n    \"V64\":               \"float32\",\n    \"V65\":               \"float32\",\n    \"V66\":               \"float32\",\n    \"V67\":               \"float32\",\n    \"V68\":               \"float32\",\n    \"V69\":               \"float32\",\n    \"V70\":               \"float32\",\n    \"V71\":               \"float32\",\n    \"V72\":               \"float32\",\n    \"V73\":               \"float32\",\n    \"V74\":               \"float32\",\n    \"V75\":               \"float32\",\n    \"V76\":               \"float32\",\n    \"V77\":               \"float32\",\n    \"V78\":               \"float32\",\n    \"V79\":               \"float32\",\n    \"V80\":               \"float32\",\n    \"V81\":               \"float32\",\n    \"V82\":               \"float32\",\n    \"V83\":               \"float32\",\n    \"V84\":               \"float32\",\n    \"V85\":               \"float32\",\n    \"V86\":               \"float32\",\n    \"V87\":               \"float32\",\n    \"V88\":               \"float32\",\n    \"V89\":               \"float32\",\n    \"V90\":               \"float32\",\n    \"V91\":               \"float32\",\n    \"V92\":               \"float32\",\n    \"V93\":               \"float32\",\n    \"V94\":               \"float32\",\n    \"V95\":               \"float32\",\n    \"V96\":               \"float32\",\n    \"V97\":               \"float32\",\n    \"V98\":               \"float32\",\n    \"V99\":               \"float32\",\n    \"V100\":              \"float32\",\n    \"V101\":              \"float32\",\n    \"V102\":              \"float32\",\n    \"V103\":              \"float32\",\n    \"V104\":              \"float32\",\n    \"V105\":              \"float32\",\n    \"V106\":              \"float32\",\n    \"V107\":              \"float32\",\n    \"V108\":              \"float32\",\n    \"V109\":              \"float32\",\n    \"V110\":              \"float32\",\n    \"V111\":              \"float32\",\n    \"V112\":              \"float32\",\n    \"V113\":              \"float32\",\n    \"V114\":              \"float32\",\n    \"V115\":              \"float32\",\n    \"V116\":              \"float32\",\n    \"V117\":              \"float32\",\n    \"V118\":              \"float32\",\n    \"V119\":              \"float32\",\n    \"V120\":              \"float32\",\n    \"V121\":              \"float32\",\n    \"V122\":              \"float32\",\n    \"V123\":              \"float32\",\n    \"V124\":              \"float32\",\n    \"V125\":              \"float32\",\n    \"V126\":              \"float32\",\n    \"V127\":              \"float32\",\n    \"V128\":              \"float32\",\n    \"V129\":              \"float32\",\n    \"V130\":              \"float32\",\n    \"V131\":              \"float32\",\n    \"V132\":              \"float32\",\n    \"V133\":              \"float32\",\n    \"V134\":              \"float32\",\n    \"V135\":              \"float32\",\n    \"V136\":              \"float32\",\n    \"V137\":              \"float32\",\n    \"V138\":              \"float32\",\n    \"V139\":              \"float32\",\n    \"V140\":              \"float32\",\n    \"V141\":              \"float32\",\n    \"V142\":              \"float32\",\n    \"V143\":              \"float32\",\n    \"V144\":              \"float32\",\n    \"V145\":              \"float32\",\n    \"V146\":              \"float32\",\n    \"V147\":              \"float32\",\n    \"V148\":              \"float32\",\n    \"V149\":              \"float32\",\n    \"V150\":              \"float32\",\n    \"V151\":              \"float32\",\n    \"V152\":              \"float32\",\n    \"V153\":              \"float32\",\n    \"V154\":              \"float32\",\n    \"V155\":              \"float32\",\n    \"V156\":              \"float32\",\n    \"V157\":              \"float32\",\n    \"V158\":              \"float32\",\n    \"V159\":              \"float32\",\n    \"V160\":              \"float32\",\n    \"V161\":              \"float32\",\n    \"V162\":              \"float32\",\n    \"V163\":              \"float32\",\n    \"V164\":              \"float32\",\n    \"V165\":              \"float32\",\n    \"V166\":              \"float32\",\n    \"V167\":              \"float32\",\n    \"V168\":              \"float32\",\n    \"V169\":              \"float32\",\n    \"V170\":              \"float32\",\n    \"V171\":              \"float32\",\n    \"V172\":              \"float32\",\n    \"V173\":              \"float32\",\n    \"V174\":              \"float32\",\n    \"V175\":              \"float32\",\n    \"V176\":              \"float32\",\n    \"V177\":              \"float32\",\n    \"V178\":              \"float32\",\n    \"V179\":              \"float32\",\n    \"V180\":              \"float32\",\n    \"V181\":              \"float32\",\n    \"V182\":              \"float32\",\n    \"V183\":              \"float32\",\n    \"V184\":              \"float32\",\n    \"V185\":              \"float32\",\n    \"V186\":              \"float32\",\n    \"V187\":              \"float32\",\n    \"V188\":              \"float32\",\n    \"V189\":              \"float32\",\n    \"V190\":              \"float32\",\n    \"V191\":              \"float32\",\n    \"V192\":              \"float32\",\n    \"V193\":              \"float32\",\n    \"V194\":              \"float32\",\n    \"V195\":              \"float32\",\n    \"V196\":              \"float32\",\n    \"V197\":              \"float32\",\n    \"V198\":              \"float32\",\n    \"V199\":              \"float32\",\n    \"V200\":              \"float32\",\n    \"V201\":              \"float32\",\n    \"V202\":              \"float32\",\n    \"V203\":              \"float32\",\n    \"V204\":              \"float32\",\n    \"V205\":              \"float32\",\n    \"V206\":              \"float32\",\n    \"V207\":              \"float32\",\n    \"V208\":              \"float32\",\n    \"V209\":              \"float32\",\n    \"V210\":              \"float32\",\n    \"V211\":              \"float32\",\n    \"V212\":              \"float32\",\n    \"V213\":              \"float32\",\n    \"V214\":              \"float32\",\n    \"V215\":              \"float32\",\n    \"V216\":              \"float32\",\n    \"V217\":              \"float32\",\n    \"V218\":              \"float32\",\n    \"V219\":              \"float32\",\n    \"V220\":              \"float32\",\n    \"V221\":              \"float32\",\n    \"V222\":              \"float32\",\n    \"V223\":              \"float32\",\n    \"V224\":              \"float32\",\n    \"V225\":              \"float32\",\n    \"V226\":              \"float32\",\n    \"V227\":              \"float32\",\n    \"V228\":              \"float32\",\n    \"V229\":              \"float32\",\n    \"V230\":              \"float32\",\n    \"V231\":              \"float32\",\n    \"V232\":              \"float32\",\n    \"V233\":              \"float32\",\n    \"V234\":              \"float32\",\n    \"V235\":              \"float32\",\n    \"V236\":              \"float32\",\n    \"V237\":              \"float32\",\n    \"V238\":              \"float32\",\n    \"V239\":              \"float32\",\n    \"V240\":              \"float32\",\n    \"V241\":              \"float32\",\n    \"V242\":              \"float32\",\n    \"V243\":              \"float32\",\n    \"V244\":              \"float32\",\n    \"V245\":              \"float32\",\n    \"V246\":              \"float32\",\n    \"V247\":              \"float32\",\n    \"V248\":              \"float32\",\n    \"V249\":              \"float32\",\n    \"V250\":              \"float32\",\n    \"V251\":              \"float32\",\n    \"V252\":              \"float32\",\n    \"V253\":              \"float32\",\n    \"V254\":              \"float32\",\n    \"V255\":              \"float32\",\n    \"V256\":              \"float32\",\n    \"V257\":              \"float32\",\n    \"V258\":              \"float32\",\n    \"V259\":              \"float32\",\n    \"V260\":              \"float32\",\n    \"V261\":              \"float32\",\n    \"V262\":              \"float32\",\n    \"V263\":              \"float32\",\n    \"V264\":              \"float32\",\n    \"V265\":              \"float32\",\n    \"V266\":              \"float32\",\n    \"V267\":              \"float32\",\n    \"V268\":              \"float32\",\n    \"V269\":              \"float32\",\n    \"V270\":              \"float32\",\n    \"V271\":              \"float32\",\n    \"V272\":              \"float32\",\n    \"V273\":              \"float32\",\n    \"V274\":              \"float32\",\n    \"V275\":              \"float32\",\n    \"V276\":              \"float32\",\n    \"V277\":              \"float32\",\n    \"V278\":              \"float32\",\n    \"V279\":              \"float32\",\n    \"V280\":              \"float32\",\n    \"V281\":              \"float32\",\n    \"V282\":              \"float32\",\n    \"V283\":              \"float32\",\n    \"V284\":              \"float32\",\n    \"V285\":              \"float32\",\n    \"V286\":              \"float32\",\n    \"V287\":              \"float32\",\n    \"V288\":              \"float32\",\n    \"V289\":              \"float32\",\n    \"V290\":              \"float32\",\n    \"V291\":              \"float32\",\n    \"V292\":              \"float32\",\n    \"V293\":              \"float32\",\n    \"V294\":              \"float32\",\n    \"V295\":              \"float32\",\n    \"V296\":              \"float32\",\n    \"V297\":              \"float32\",\n    \"V298\":              \"float32\",\n    \"V299\":              \"float32\",\n    \"V300\":              \"float32\",\n    \"V301\":              \"float32\",\n    \"V302\":              \"float32\",\n    \"V303\":              \"float32\",\n    \"V304\":              \"float32\",\n    \"V305\":              \"float32\",\n    \"V306\":              \"float32\",\n    \"V307\":              \"float32\",\n    \"V308\":              \"float32\",\n    \"V309\":              \"float32\",\n    \"V310\":              \"float32\",\n    \"V311\":              \"float32\",\n    \"V312\":              \"float32\",\n    \"V313\":              \"float32\",\n    \"V314\":              \"float32\",\n    \"V315\":              \"float32\",\n    \"V316\":              \"float32\",\n    \"V317\":              \"float32\",\n    \"V318\":              \"float32\",\n    \"V319\":              \"float32\",\n    \"V320\":              \"float32\",\n    \"V321\":              \"float32\",\n    \"V322\":              \"float32\",\n    \"V323\":              \"float32\",\n    \"V324\":              \"float32\",\n    \"V325\":              \"float32\",\n    \"V326\":              \"float32\",\n    \"V327\":              \"float32\",\n    \"V328\":              \"float32\",\n    \"V329\":              \"float32\",\n    \"V330\":              \"float32\",\n    \"V331\":              \"float32\",\n    \"V332\":              \"float32\",\n    \"V333\":              \"float32\",\n    \"V334\":              \"float32\",\n    \"V335\":              \"float32\",\n    \"V336\":              \"float32\",\n    \"V337\":              \"float32\",\n    \"V338\":              \"float32\",\n    \"V339\":              \"float32\",\n    \"id_01\":             \"float32\",\n    \"id_02\":             \"float32\",\n    \"id_03\":             \"float32\",\n    \"id_04\":             \"float32\",\n    \"id_05\":             \"float32\",\n    \"id_06\":             \"float32\",\n    \"id_07\":             \"float32\",\n    \"id_08\":             \"float32\",\n    \"id_09\":             \"float32\",\n    \"id_10\":             \"float32\",\n    \"id_11\":             \"float32\",\n    \"id_12\":              \"object\",\n    \"id_13\":             \"float32\",\n    \"id_14\":             \"float32\",\n    \"id_15\":              \"object\",\n    \"id_16\":              \"object\",\n    \"id_17\":             \"float32\",\n    \"id_18\":             \"float32\",\n    \"id_19\":             \"float32\",\n    \"id_20\":             \"float32\",\n    \"id_21\":             \"float32\",\n    \"id_22\":             \"float32\",\n    \"id_23\":              \"object\",\n    \"id_24\":             \"float32\",\n    \"id_25\":             \"float32\",\n    \"id_26\":             \"float32\",\n    \"id_27\":              \"object\",\n    \"id_28\":              \"object\",\n    \"id_29\":              \"object\",\n    \"id_30\":              \"object\",\n    \"id_31\":              \"object\",\n    \"id_32\":             \"float32\",\n    \"id_33\":              \"object\",\n    \"id_34\":              \"object\",\n    \"id_35\":              \"object\",\n    \"id_36\":              \"object\",\n    \"id_37\":              \"object\",\n    \"id_38\":              \"object\",\n    \"DeviceType\":         \"object\",\n    \"DeviceInfo\":         \"object\",\n    \"is_fraud\":\t\t\t  \"int8\"\n}\n\nemails = {'gmail': 'google',\n'att.net': 'att',\n'twc.com': 'spectrum',\n'scranton.edu': 'other',\n'optonline.net': 'other',\n'hotmail.co.uk': 'microsoft',\n'comcast.net': 'other',\n'yahoo.com.mx': 'yahoo',\n'yahoo.fr': 'yahoo',\n'yahoo.es': 'yahoo',\n'charter.net': 'spectrum',\n'live.com': 'microsoft',\n'aim.com': 'aol',\n'hotmail.de': 'microsoft',\n'centurylink.net': 'centurylink',\n'gmail.com': 'google',\n'me.com': 'apple',\n'earthlink.net': 'other',\n'gmx.de': 'other',\n'web.de': 'other',\n'cfl.rr.com': 'other',\n'hotmail.com': 'microsoft',\n'protonmail.com': 'other',\n'hotmail.fr': 'microsoft',\n'windstream.net': 'other',\n'outlook.es': 'microsoft',\n'yahoo.co.jp': 'yahoo',\n'yahoo.de': 'yahoo',\n'servicios-ta.com': 'other',\n'netzero.net': 'other',\n'suddenlink.net': 'other',\n'roadrunner.com': 'other',\n'sc.rr.com': 'other',\n'live.fr': 'microsoft',\n'verizon.net': 'yahoo',\n'msn.com': 'microsoft',\n'q.com': 'centurylink',\n'prodigy.net.mx': 'att',\n'frontier.com': 'yahoo',\n'anonymous.com': 'other',\n'rocketmail.com': 'yahoo',\n'sbcglobal.net': 'att',\n'frontiernet.net': 'yahoo',\n'ymail.com': 'yahoo',\n'outlook.com': 'microsoft',\n'mail.com': 'other',\n'bellsouth.net': 'other',\n'embarqmail.com': 'centurylink',\n'cableone.net': 'other',\n'hotmail.es': 'microsoft',\n'mac.com': 'apple',\n'yahoo.co.uk': 'yahoo',\n'netzero.com': 'other',\n'yahoo.com': 'yahoo',\n'live.com.mx': 'microsoft',\n'ptd.net': 'other',\n'cox.net': 'other',\n'aol.com': 'aol',\n'juno.com': 'other',\n'icloud.com': 'apple'}\n\n\nus_emails = ['gmail', 'net', 'edu']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@contextmanager\ndef timer(name):\n    \"\"\"\n    Time Each Process\n    \"\"\"\n    t0 = time.time()\n    yield\n    print('\\n[{}] done in {} Minutes\\n'.format(name, round((time.time() - t0)/60,2)))\n\n# Device Features\ndef id_split(dataframe):\n    # https://www.kaggle.com/davidcairuz/feature-engineering-lightgbm-w-gpu\n    dataframe['device_name'] = dataframe['DeviceInfo'].str.split('/', expand=True)[0]\n    dataframe['device_version'] = dataframe['DeviceInfo'].str.split('/', expand=True)[1]\n\n    dataframe['OS_id_30'] = dataframe['id_30'].str.split(' ', expand=True)[0]\n    dataframe['version_id_30'] = dataframe['id_30'].str.split(' ', expand=True)[1]\n\n    dataframe['browser_id_31'] = dataframe['id_31'].str.split(' ', expand=True)[0]\n    dataframe['version_id_31'] = dataframe['id_31'].str.split(' ', expand=True)[1]\n\n    dataframe['screen_width'] = dataframe['id_33'].str.split('x', expand=True)[0]\n    dataframe['screen_height'] = dataframe['id_33'].str.split('x', expand=True)[1]\n\n    dataframe['id_34'] = dataframe['id_34'].str.split(':', expand=True)[1]\n    dataframe['id_23'] = dataframe['id_23'].str.split(':', expand=True)[1]\n\n    dataframe.loc[dataframe['device_name'].str.contains('SM', na=False), 'device_name'] = 'Samsung'\n    dataframe.loc[dataframe['device_name'].str.contains('SAMSUNG', na=False), 'device_name'] = 'Samsung'\n    dataframe.loc[dataframe['device_name'].str.contains('GT-', na=False), 'device_name'] = 'Samsung'\n    dataframe.loc[dataframe['device_name'].str.contains('Moto G', na=False), 'device_name'] = 'Motorola'\n    dataframe.loc[dataframe['device_name'].str.contains('Moto', na=False), 'device_name'] = 'Motorola'\n    dataframe.loc[dataframe['device_name'].str.contains('moto', na=False), 'device_name'] = 'Motorola'\n    dataframe.loc[dataframe['device_name'].str.contains('LG-', na=False), 'device_name'] = 'LG'\n    dataframe.loc[dataframe['device_name'].str.contains('rv:', na=False), 'device_name'] = 'RV'\n    dataframe.loc[dataframe['device_name'].str.contains('HUAWEI', na=False), 'device_name'] = 'Huawei'\n    dataframe.loc[dataframe['device_name'].str.contains('ALE-', na=False), 'device_name'] = 'Huawei'\n    dataframe.loc[dataframe['device_name'].str.contains('-L', na=False), 'device_name'] = 'Huawei'\n    dataframe.loc[dataframe['device_name'].str.contains('Blade', na=False), 'device_name'] = 'ZTE'\n    dataframe.loc[dataframe['device_name'].str.contains('BLADE', na=False), 'device_name'] = 'ZTE'\n    dataframe.loc[dataframe['device_name'].str.contains('Linux', na=False), 'device_name'] = 'Linux'\n    dataframe.loc[dataframe['device_name'].str.contains('XT', na=False), 'device_name'] = 'Sony'\n    dataframe.loc[dataframe['device_name'].str.contains('HTC', na=False), 'device_name'] = 'HTC'\n    dataframe.loc[dataframe['device_name'].str.contains('ASUS', na=False), 'device_name'] = 'Asus'\n\n    dataframe.loc[dataframe.device_name.isin(dataframe.device_name.value_counts()[dataframe.device_name.value_counts() < 200].index), 'device_name'] = \"Others\"\n    dataframe['had_id'] = 1\n    gc.collect()\n    \n    return dataframe\n    \ndef fraud_preprocessing(debug = None):\n    print(\"Starting Pre-Processing..\")\n    with timer(\"Load Tables\"):\n        train_transaction = pd.read_csv('../input/train_transaction.csv',\n                                        index_col='TransactionID', nrows= debug, dtype = schema)\n        test_transaction = pd.read_csv('../input/test_transaction.csv',\n                                       index_col='TransactionID', nrows= debug, dtype = schema)\n\n        train_identity = pd.read_csv('../input/train_identity.csv', index_col='TransactionID')\n        test_identity = pd.read_csv('../input/test_identity.csv', index_col='TransactionID')\n\n        sample_submission = pd.read_csv('../input/sample_submission.csv',\n                                        index_col='TransactionID',\n                                        nrows= debug)\n\n    with timer(\"Merge Tables\"):\n        train = train_transaction.merge(train_identity, how='left', left_index=True, right_index=True)\n        test = test_transaction.merge(test_identity, how='left', left_index=True, right_index=True)\n\n        print(\"Train Shape: {} Rows, {} Columns\".format(*train.shape))\n        print(\"Test Shape: {} Rows, {} Columns\".format(*test.shape))\n\n        y = train[target_var].copy()\n        del train_transaction, train_identity, test_transaction, test_identity\n\n        traindex = train.index\n        testdex = test.index\n        \n    with timer(\"Train/Test Split Feature Engineering\"):\n        # Credit https://www.kaggle.com/davidcairuz/feature-engineering-lightgbm-w-gpu\n        train['TransactionAmt_to_mean_card1'] = train['TransactionAmt'] / train.groupby(['card1'])['TransactionAmt'].transform('mean')\n        train['TransactionAmt_to_mean_card4'] = train['TransactionAmt'] / train.groupby(['card4'])['TransactionAmt'].transform('mean')\n        train['TransactionAmt_to_std_card1'] = train['TransactionAmt'] / train.groupby(['card1'])['TransactionAmt'].transform('std')\n        train['TransactionAmt_to_std_card4'] = train['TransactionAmt'] / train.groupby(['card4'])['TransactionAmt'].transform('std')\n\n        test['TransactionAmt_to_mean_card1'] = test['TransactionAmt'] / test.groupby(['card1'])['TransactionAmt'].transform('mean')\n        test['TransactionAmt_to_mean_card4'] = test['TransactionAmt'] / test.groupby(['card4'])['TransactionAmt'].transform('mean')\n        test['TransactionAmt_to_std_card1'] = test['TransactionAmt'] / test.groupby(['card1'])['TransactionAmt'].transform('std')\n        test['TransactionAmt_to_std_card4'] = test['TransactionAmt'] / test.groupby(['card4'])['TransactionAmt'].transform('std')\n\n        train['id_02_to_mean_card1'] = train['id_02'] / train.groupby(['card1'])['id_02'].transform('mean')\n        train['id_02_to_mean_card4'] = train['id_02'] / train.groupby(['card4'])['id_02'].transform('mean')\n        train['id_02_to_std_card1'] = train['id_02'] / train.groupby(['card1'])['id_02'].transform('std')\n        train['id_02_to_std_card4'] = train['id_02'] / train.groupby(['card4'])['id_02'].transform('std')\n\n        test['id_02_to_mean_card1'] = test['id_02'] / test.groupby(['card1'])['id_02'].transform('mean')\n        test['id_02_to_mean_card4'] = test['id_02'] / test.groupby(['card4'])['id_02'].transform('mean')\n        test['id_02_to_std_card1'] = test['id_02'] / test.groupby(['card1'])['id_02'].transform('std')\n        test['id_02_to_std_card4'] = test['id_02'] / test.groupby(['card4'])['id_02'].transform('std')\n\n        train['D15_to_mean_card1'] = train['D15'] / train.groupby(['card1'])['D15'].transform('mean')\n        train['D15_to_mean_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('mean')\n        train['D15_to_std_card1'] = train['D15'] / train.groupby(['card1'])['D15'].transform('std')\n        train['D15_to_std_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('std')\n\n        test['D15_to_mean_card1'] = test['D15'] / test.groupby(['card1'])['D15'].transform('mean')\n        test['D15_to_mean_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('mean')\n        test['D15_to_std_card1'] = test['D15'] / test.groupby(['card1'])['D15'].transform('std')\n        test['D15_to_std_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('std')\n\n        train['D15_to_mean_addr1'] = train['D15'] / train.groupby(['addr1'])['D15'].transform('mean')\n        train['D15_to_mean_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('mean')\n        train['D15_to_std_addr1'] = train['D15'] / train.groupby(['addr1'])['D15'].transform('std')\n        train['D15_to_std_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('std')\n\n        test['D15_to_mean_addr1'] = test['D15'] / test.groupby(['addr1'])['D15'].transform('mean')\n        test['D15_to_mean_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('mean')\n        test['D15_to_std_addr1'] = test['D15'] / test.groupby(['addr1'])['D15'].transform('std')\n        test['D15_to_std_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('std')\n        \n        # New feature - log of transaction amount. ()\n        train['TransactionAmt_Log'] = np.log(train['TransactionAmt'])\n        test['TransactionAmt_Log'] = np.log(test['TransactionAmt'])\n        \n        # Encoding - count encoding for both train and test\n        for feature in ['card1', 'card2', 'card3', 'card4', 'card5', 'card6', 'id_36']:\n            train[feature + '_count_full'] = train[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True).value_counts(dropna=False))\n            test[feature + '_count_full'] = test[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True).value_counts(dropna=False))\n\n        # Encoding - count encoding separately for train and test\n        for feature in ['id_01', 'id_31', 'id_33', 'id_36']:\n            train[feature + '_count_dist'] = train[feature].map(train[feature].value_counts(dropna=False))\n            test[feature + '_count_dist'] = test[feature].map(test[feature].value_counts(dropna=False))\n            \n        # https://www.kaggle.com/c/ieee-fraud-detection/discussion/100499\n        for c in ['P_emaildomain', 'R_emaildomain']:\n            train[c + '_bin'] = train[c].map(emails)\n            test[c + '_bin'] = test[c].map(emails)\n\n            train[c + '_suffix'] = train[c].map(lambda x: str(x).split('.')[-1])\n            test[c + '_suffix'] = test[c].map(lambda x: str(x).split('.')[-1])\n\n            train[c + '_suffix'] = train[c + '_suffix'].map(lambda x: x if str(x) not in us_emails else 'us')\n            test[c + '_suffix'] = test[c + '_suffix'].map(lambda x: x if str(x) not in us_emails else 'us')\n            \n        # Extract Device Information\n        train = id_split(train)\n        test = id_split(test)\n        \n        # Combine\n        df = pd.concat([train.drop(target_var,axis=1),test],axis = 0)\n        del train, test\n        \n    with timer(\"Whole Feature Engineering\"):\n        START_DATE = '2017-12-01'\n        startdate = datetime.datetime.strptime(START_DATE, '%Y-%m-%d')    \n        df = df.assign(\n                # New feature - decimal part of the transaction amount\n                TransactionAmt_decimal = ((df['TransactionAmt'] - df['TransactionAmt'].astype(int)) * 1000).astype(int),\n\n                # Count encoding for card1 feature. \n                # Explained in this kernel: https://www.kaggle.com/nroman/eda-for-cis-fraud-detection\n                card1_count_full = df['card1'].map(df['card1'].value_counts(dropna=False)),\n\n                # https://www.kaggle.com/fchmiel/day-and-time-powerful-predictive-feature\n                Transaction_day_of_week = np.floor((df['TransactionDT'] / (3600 * 24) - 1) % 7),\n                Transaction_hour = np.floor(df['TransactionDT'] / 3600) % 24,\n\n                TransactionDT = df['TransactionDT'].apply(lambda x: (startdate + datetime.timedelta(seconds = x))),\n            )\n        df = df.assign(\n                # Time of Day\n                year = df['TransactionDT'].dt.year,\n                month = df['TransactionDT'].dt.month,\n                dow = df['TransactionDT'].dt.dayofweek,\n                quarter = df['TransactionDT'].dt.quarter,\n                hour = df['TransactionDT'].dt.hour,\n                day = df['TransactionDT'].dt.day,\n        \n                # All NaN\n                all_group_nan_sum = df.isnull().sum(axis=1) / df.shape[1],\n                all_group_0_count = (df == 0).astype(int).sum(axis=1) / (df.shape[1] - df.isnull().sum(axis=1))\n        )\n        \n        # Create Features based on anonymised prefix groups\n        prefix = ['C','D','Device','M','Transaction','V','addr','card','dist','id']\n        for i, p in enumerate(prefix):\n            column_set = [x for x in df.columns.tolist() if x.startswith(prefix[i])]\n\n            # Take NA count\n            df[p + \"group_nan_sum\"] = df[column_set].isnull().sum(axis=1) / df[column_set].shape[1]\n\n            # Take SUM/Mean if numeric\n            numeric_cols = [x for x in column_set if df[x].dtype != object]\n            if numeric_cols:\n                df[p + \"group_sum\"] = df[column_set].sum(axis=1)\n                df[p + \"group_mean\"] = df[column_set].mean(axis=1)\n                # Zero Count\n                df[p + \"group_0_count\"] = (df[column_set] == 0).astype(int).sum(axis=1) / (df[column_set].shape[1] - df[p + \"group_nan_sum\"])\n\n    with timer(\"Label Encode\"):\n        categorical_cols = []\n        # Label Encoding\n        for f in df.columns:\n            if df[f].dtype=='object': \n                categorical_cols += [f]\n                lbl = preprocessing.LabelEncoder()\n                df[f] = lbl.fit_transform(df[f].astype(str))\n    print(\"Total Shape: {} Rows, {} Columns\".format(*df.shape))\n    return df, y, traindex, testdex, categorical_cols, sample_submission","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Prepare Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Bayesian Parameters\nmax_iter = 35\ninitial_iter = 5\n\n# Load Data\nDEBUG = None # None for no debug, else number of rows\ndf, y, traindex, testdex, cat_cols, sample_submission = fraud_preprocessing(debug = DEBUG)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Features for EDA\ndf['yrmth'] = df.year.astype(str) + df.month.map(\"{:02}\".format)\ndf['Fraud'] = np.nan\ndf.loc[traindex,'Fraud'] = y[traindex]\ndf['traintest'] = 'Test'\ndf.loc[df.Fraud.notnull(),'traintest'] = 'Train'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Are there redundant Transaction IDs?\")\nprint(df.index.value_counts().value_counts())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, ax = plt.subplots(2,2, figsize = [12,10])\nfor tt in ['Train','Test']:\n    df.loc[df.traintest == tt,['all_group_nan_sum','TransactionDT']].set_index('TransactionDT')\\\n        .resample('1d').count().plot(label = tt, ax = ax[0,0])\n    df.loc[df.traintest == tt,['all_group_nan_sum','TransactionDT']].set_index('TransactionDT')\\\n        .resample('1d').mean().plot(label = tt, ax = ax[1,0])\n    df.loc[df.traintest == tt,['all_group_0_count','TransactionDT']].set_index('TransactionDT')\\\n        .resample('1d').mean().plot(label = tt, ax = ax[1,1])\n    df.loc[df.traintest == tt,['TransactionAmt','TransactionDT']].set_index('TransactionDT')\\\n        .resample('1d').mean().plot(label = tt, ax = ax[0,1])\nax[0,0].set_title(\"Observation Count: Train/ Test\")\nax[0,0].set_ylabel(\"Count\")\nax[1,0].set_title(\"Average Number of Missing Values in Rows: Train/ Test\")\nax[1,0].set_ylabel(\"Percent of Rows Is Null\")\nax[0,1].set_title(\"Average Number of Missing Values in Rows: Train/ Test\")\nax[0,1].set_ylabel(\"Percent of Rows Is Null\")\nax[1,1].set_title(\"Average Number of Zero Values in Rows: Train/ Test\")\nax[1,1].set_ylabel(\"Percent of Rows Is Zero\")\n\nplt.tight_layout(pad=0)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, ax = plt.subplots(1,2,figsize = [12,5])\n\nprefix = ['C','D','Device','M','Transaction','V','addr','card','dist','id']\nfor i, p in enumerate(prefix):\n    df.loc[df.traintest == tt,[p + \"group_nan_sum\",'TransactionDT']].set_index('TransactionDT')\\\n        .resample('1d').mean().plot(label = \"Missing\", ax = ax[0])\n    df.loc[df.traintest == tt,[p + \"group_0_count\",'TransactionDT']].set_index('TransactionDT')\\\n        .resample('1d').mean().plot(label = \"Zero\", ax = ax[1])\n\nax[0].get_legend().remove()\nax[1].legend(prefix,fontsize='large', loc='center left',bbox_to_anchor=(1, 0.5))\n\nax[0].set_title(\"Proportion of Data Missing by Column Group\")\nax[1].set_title(\"Proportion of Data Equal Zero by Column Group\")\nax[0].set_ylabel(\"Proportion Missing\")\nax[1].set_ylabel(\"Proportion Zero\")\n\nplt.tight_layout(pad=1)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Missing Values Pattern\n# Hourly Pattern\n# Individual's susepticality to fraud (explore ID)\n# Is there a way to see how close various fraud claims are?\n\n# Create a CPU kernel where I can experiment with features and LOFO..\n# Smash all data together.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Bayesian Hyper Parameter Search"},{"metadata":{"trusted":true},"cell_type":"code","source":"X = df.loc[traindex,:]\nfeature_subset = X.columns.tolist()\ntest = df.loc[testdex,:]\n\nsplit_size = 0.35\nn_estimators = 10000\nmetric = 'auc'\nESR = 150\n\nfeature_subset = [x for x in X.columns.tolist() if x not in ['TransactionDT','Fraud', 'traintest', 'yrmth']]\nX_train, X_valid, y_train, y_valid = train_test_split(\n    X.loc[:,feature_subset], y, test_size=split_size,\n    random_state=seed, shuffle=False)\n\ndel df ; gc.collect();\n\nprint(\"None Fraud: {}%, Fraud: {}%\".format(*y.value_counts(normalize=True)))\nprint(\"Randomness Score AUC: {}\".format(\n    metrics.roc_auc_score(y,np.array([y.value_counts(normalize=True)[0]]*y.shape[0]))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bds = [ {'name': 'min_data_in_leaf', 'type': 'continuous', 'domain': (2, 100)},\n        {'name': 'num_leaves', 'type': 'continuous', 'domain': (20, 1000)},\n        {'name': 'subsample_for_bin', 'type': 'continuous', 'domain': (1000, 5000)},\n        {'name': 'min_sum_hessian_in_leaf', 'type': 'continuous', 'domain': (0, 15)},\n        {'name': 'reg_alpha', 'type': 'continuous', 'domain': (0, 3)},\n        {'name': 'reg_lambda', 'type': 'continuous', 'domain': (0, 3)},\n        {'name': 'bagging_fraction', 'type': 'continuous', 'domain': (.01, 1)},\n        {'name': 'feature_fraction', 'type': 'continuous', 'domain': (.01, 1)}\n      ]\n\nsb_cat_cols = [x for x in cat_cols if x in feature_subset]\nlgtrain = lgb.Dataset(X_train, y_train, categorical_feature = sb_cat_cols, free_raw_data=False)\nlgvalid = lgb.Dataset(X_valid, y_valid, categorical_feature = sb_cat_cols, free_raw_data=False)\n\n# Optimization objective \ndef lgb_score(para):\n    parameters = para[0]\n#     num_leaves = 2**parameters[0] if 2**parameters[0] < 4095 else 4095\n#     parameters[0] = -1 if parameters[0] == 45 else parameters[0]\n#     num_leaves = int(num_leaves * parameters[1])\n    modelstart= time.time()\n    params = {\n        # Static Variables\n        'objective': 'binary',\n    #     'num_class': [3],\n        'metric': metric,\n        'learning_rate': 0.05, # Multiplication performed on each boosting iteration.\n        'device': 'gpu', # GPU usage.\n        'tree_learner': 'serial',\n        'boost_from_average': 'true',\n        'num_boost_round': n_estimators,\n\n    #     # Dynamic Variables\n    #     # https://sites.google.com/view/lauraepp/parameters\n        'boosting_type': 'gbdt',#, 'goss', 'dart'],\n\n        # Bushi-ness Parameters\n        'max_depth': -1,  # -1 means no tree depth limit\n        'num_leaves': int(parameters[1]), # we should let it be smaller than 2^(max_depth)\n\n        # Tree Depth Regularization\n        'subsample_for_bin': int(parameters[2]), # Number of samples for constructing bin\n        'min_data_in_leaf': int(parameters[0]), # Minimum number of data need in a child(min_data_in_leaf) - Must be motified when using a smaller dataset\n    #     'min_gain_to_split': [0], # Prune by minimum loss requirement.\n        'min_sum_hessian_in_leaf': parameters[3], # Prune by minimum hessian requirement - Minimum sum of instance weight(hessian) needed in a child(leaf)\n\n        # Regularization L1/L2\n        'reg_alpha': parameters[4], # L1 regularization term on weights (0 is no regular)\n        'reg_lambda': parameters[5], # L2 regularization term on weights\n    #     'max_bin': list(range(70, 300, 30)),  # Number of bucketed bin for feature values\n\n        # Row/Column Sampling\n    #     'colsample_bytree': list(np.linspace(0.2, 1, 10).round(2)), # Subsample ratio of columns when constructing each tree.\n#         'subsample': parameters[6], # Subsample ratio of the training instance.\n    #     'subsample_freq': 0, # frequence of subsample, <=0 means no enable\n        'bagging_fraction': parameters[6],# Percentage of rows used per iteration frequency.\n        'bagging_freq': 1,# Iteration frequency to update the selected rows.\n        'feature_fraction': parameters[7], # Percentage of columns used per iteration.\n    #     'colsample_bylevel': [1], # DANGER - Note Recommended Tuning - Percentage of columns used per split selection.\n\n        # Dart Specific\n    #     'max_drop': list(np.linspace(1, 70, 5).round(0).astype(int)), # Maximum number of dropped trees on one iteration.\n    #     'rate_drop': list(np.linspace(0, .8, 10).round(2)), # Dropout - Probability to to drop a tree on one iteration.\n    #     'skip_drop': list(np.linspace(.4, .6, 3).round(2)), # Probability of skipping any drop on one iteration. \n    #     'uniform_drop': [False], # Uniform weight application for trees.\n\n        # GOSS Specific\n    #     'top_rate': [.2], # Keep top gradients.\n    #     'other_rate': [.1], # Keep bottom gradients.\n        # When top_rate + other_rate <= 0.5, the first iteration is sampled by (top_rate + other_rate)%. Attempts to keep only the bottom other_rate% gradients per iteration.\n\n        # Imbalanced Dependent Variable\n        'is_unbalance': False, #True if int(parameters[8]) == 1 else False, # because training data is unbalance (replaced with scale_pos_weight)\n    #     'scale_pos_weight': []\n        'nthread': -1, # Multi-threading\n        'verbose': -1, # Logging Iteration Progression\n        'seed': seed # Seed for row sampling RNG.\n    }\n    \n    experiment_lgb = lgb.train(\n            params = params,\n            train_set = lgtrain,\n            valid_sets=[lgtrain, lgvalid],\n            valid_names=['train','valid'],\n            verbose_eval= 0,\n            early_stopping_rounds= ESR\n            )\n    runtime = (time.time() - modelstart)/60\n\n    val_pred = experiment_lgb.predict(X_valid)\n\n    # Get Metrics\n    score = experiment_lgb.best_score['valid'][metric]\n    loss = metrics.log_loss(y_valid, val_pred)\n    params['num_boost_round'] = experiment_lgb.best_iteration\n\n    gpyopt_output.append(\n        [\n         loss,\n         experiment_lgb.best_score['train'][metric],\n         score,\n         experiment_lgb.best_iteration,\n         params,\n         runtime\n        ]\n    )\n    \n    return score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gpyopt_output = []\noptimizer = BayesianOptimization(f=lgb_score, \n                                 domain=bds,\n                                 model_type='GP',\n                                 optimize_restarts = 1,\n                                 initial_design_numdata = initial_iter,\n                                 acquisition_type ='EI',\n                                 acquisition_jitter = 0.1,\n                                 exact_feval=True, \n                                 maximize=True)\n\nwith timer(\"Bayesian Optimisation - {} Iterations\".format(max_iter + initial_iter)):\n    optimizer.run_optimization(max_iter=max_iter)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Output\nresults = pd.DataFrame(gpyopt_output,\n        columns = ['logloss','train_auc','valid_auc',\n                   'boosting_rounds','parameters', 'runtime']\n                      )\nresults.to_csv(\"gpyopt_iterations_output.csv\")\nbest_params = results['parameters'].iloc[np.argmax(results.valid_auc)]\n\n# Index as Iteration\nresults = results.reset_index().rename(columns = {'index':'iteration'})\nresults['iteration'] = results['iteration'] + 1\n\n# Visualize Convergence\noptimizer.plot_convergence()\n\nprint(\"Best AUC: {}\".format(optimizer.fx_opt))\nprint(\"Best Parameters\")\npprint.pprint(best_params)\n\n# Json to DataFrame\nresults = pd.concat([results.drop('parameters',axis=1).reset_index(drop=True),\n                     json_normalize(results['parameters']).reset_index(drop=True)\n                    ], axis = 1)\n\n# Additional Result Features\nresults['TPM'] = results['boosting_rounds'] / results['runtime']\nresults['Total Leaves'] = results['boosting_rounds'] * results['num_leaves']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.regplot(x='iteration', y = 'valid_auc', data = results)\nplt.title(\"AUC over Iterations\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Convergence Analysis\nCan I shed light on the exploration vs exploitation trade-off?"},{"metadata":{"trusted":true},"cell_type":"code","source":"# What do the steps look like? How much exploration is there in terms of boundary proportionality?\n# Can I use the X data, scale it by bounds, and explore this exploration amount?\nbds","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Hyperparameter Relationship Analysis\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"results['TPM'] = results['boosting_rounds'] / results['runtime']\nresults['Total Leaves'] = results['boosting_rounds'] * results['num_leaves']\nresults.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"t_r,t_c = 3, 4\nf, axes = plt.subplots(t_r, t_c, figsize = [15,12],\n                       sharex=False, sharey=False)\nrow,col = 0,0\nparas = ['num_leaves', 'subsample_for_bin', 'min_sum_hessian_in_leaf','reg_alpha',\n         'reg_lambda', 'bagging_fraction','feature_fraction','min_data_in_leaf',\n         'boosting_rounds', 'runtime', 'logloss','train_auc']\nfor var in paras:\n    if col == 4:\n        col = 0\n        row += 1\n    \n    # Plot\n    sns.regplot(x=var, y = \"valid_auc\", data = results,\n                x_estimator=np.mean, logx=True,\n                truncate=True, ax = axes[row,col])\n    axes[row,col].set_title('{} vs AUC'.format(var.title()))\n    axes[row,col].grid(True, lw = 2, ls = '--', c = '.75')\n    # My last plot has a waky x limit..\n    \n    axes[row,col].set_ylim(results.valid_auc.min(),results.valid_auc.max())\n    if var == paras[-1]:\n        axes[row,col].set_xlim(.90,1)\n\n    col+=1\nplt.tight_layout(pad=0)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"t_r,t_c = 3, 4\nf, axes = plt.subplots(t_r, t_c, figsize = [15,12],\n                       sharex=False, sharey=False)\nrow,col = 0,0\nparas = ['num_leaves', 'subsample_for_bin', 'min_sum_hessian_in_leaf','reg_alpha',\n         'reg_lambda', 'bagging_fraction','feature_fraction','min_data_in_leaf',\n         'boosting_rounds', 'runtime', 'logloss','train_auc']\nfor var in paras:\n    if col == 4:\n        col = 0\n        row += 1\n    \n    # Plot\n    sns.regplot(x='iteration', y = var, data = results,\n                truncate=True, ax = axes[row,col])\n    axes[row,col].set_title('{} vs Iteration'.format(var.title()))\n    axes[row,col].grid(True, lw = 2, ls = '--', c = '.75')\n    # My last plot has a waky x limit..\n    \n#     axes[row,col].set_ylim(results.valid_auc.min(),results.valid_auc.max())\n    if var == paras[-1]:\n        axes[row,col].set_xlim(results[paras[-1]].min(),results[paras[-1]].max())\n\n    col+=1\nplt.tight_layout(pad=0)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Examine Correlations\nf, ax = plt.subplots(figsize=[10,7])\nsns.heatmap(results[paras].corr(),\n            annot=True, fmt=\".2f\",cbar_kws={'label': 'Correlation Coefficient'},cmap=\"plasma\",ax=ax, linewidths=.5)\nax.set_title(\"Gpyopt Results Correlation Matrix\")\nplt.tight_layout(pad=1)\nfilename = 'gpyopt_correlation_matrix.png'\nplt.savefig(filename)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f, ax = plt.subplots(1,2, figsize = [10,4])\n\n# Trees and Runtime\nax[0].plot(results['TPM'], '-rx')\nax[0].set_xlabel(\"Bayesian Search Iteration\")\nax[0].set_ylabel(\"TPM\")\nax[0].set_title(\"Tree per Minute through Bayesian Search\")\n\n# Num-leaves and runtime\nsns.regplot(y = 'Total Leaves', x = \"runtime\", data = results,\n            x_estimator=np.mean, logx=True,\n            truncate=True, ax = ax[1])\nax[1].set_ylabel(\"Total Leaves\")\nax[1].set_xlabel(\"Runtime\")\nax[1].set_title(\"Total Leaves vs Runtime\")\n\n\nplt.tight_layout(pad=0)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Submit Best Parameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"allmodelstart= time.time()\nEPOCHS = 5\ndel best_params['num_boost_round']\nbest_params['learning_rate'] = 0.01\nkf = KFold(n_splits = EPOCHS, shuffle = False)\ny_preds = np.zeros(sample_submission.shape[0])\ny_preds_fold = np.zeros([sample_submission.shape[0],EPOCHS])\ny_oof = np.zeros(X.shape[0])\nf,ax = plt.subplots(1,3,figsize = [15,6])\nsb_cat_cols = [x for x in cat_cols if x in feature_subset]\nall_feature_importance_df  = pd.DataFrame()\n\nmean_fpr = np.linspace(0,1,100)\ncms, tprs, aucs, y_real, y_proba,recalls, roc_aucs,f1_scores, accuracies, precisions = [],[],[],[],[],[],[],[],[],[]\n\n# Run Out of Fold\nfor i, (tr_idx, val_idx) in enumerate(kf.split(X, y)):\n    i += 1\n    best_params[seed] = i # More Diversity\n    modelstart= time.time()\n    evals_result = {}\n    lgtrain = lgb.Dataset(X.iloc[tr_idx, :][feature_subset], y.iloc[tr_idx], categorical_feature = sb_cat_cols)\n    lgvalid = lgb.Dataset(X.iloc[val_idx, :][feature_subset], y.iloc[val_idx], categorical_feature = sb_cat_cols)\n    \n    # Train Model\n    clf = lgb.train(\n            best_params,\n            lgtrain,\n            valid_sets=[lgtrain, lgvalid],\n            valid_names=['train','valid'],\n            verbose_eval=300,\n            num_boost_round = n_estimators,\n            early_stopping_rounds=ESR,\n            evals_result=evals_result,\n            )\n    # Model Evaluation\n    y_oof[val_idx] =  clf.predict(X.iloc[val_idx, :][feature_subset])\n    y_preds_fold[:,i-1] = clf.predict(test.loc[:,feature_subset])\n    y_preds += y_preds_fold[:,i-1] / EPOCHS\n    \n    evals_result['train_{}'.format(i)] = evals_result.pop('train')\n    evals_result['valid_{}'.format(i)] = evals_result.pop('valid')\n    lgb.plot_metric(evals_result, metric=metric, ax = ax[0])\n    ax[0].set_title(\"GPU LGBM Metric Convergence over {} Folds\".format(EPOCHS))\n    \n    # Feature Importance\n    fold_importance_df = pd.DataFrame()\n    fold_importance_df[\"feature\"] = feature_subset\n    fold_importance_df[\"importance\"] = clf.feature_importance()\n    all_feature_importance_df = pd.concat([all_feature_importance_df, fold_importance_df], axis=0)\n    print(\"\\nModel Runtime: %0.2f Minutes\"%((time.time() - modelstart)/60))\n    \n    # Scores \n    roc_aucs.append(roc_auc_score(y.iloc[val_idx].values,y_oof[val_idx]))\n    accuracies.append(accuracy_score(y.iloc[val_idx].values,y_oof[val_idx].round()))\n    recalls.append(recall_score(y.iloc[val_idx].values,y_oof[val_idx].round()))\n    precisions.append(precision_score(y.iloc[val_idx].values,y_oof[val_idx].round()))\n    f1_scores.append(f1_score(y.iloc[val_idx].values,y_oof[val_idx].round()))\n    \n    # Roc curve by folds\n    fpr, tpr, t = roc_curve(y.iloc[val_idx].values,y_oof[val_idx])\n    tprs.append(interp(mean_fpr, fpr, tpr))\n    roc_auc = auc(fpr, tpr)\n    aucs.append(roc_auc)\n    ax[1].plot(fpr, tpr, lw=2, alpha=0.3, label='ROC fold %d (AUC = %0.4f)' % (i,roc_auc))\n    \n    # Precion recall by folds\n    precision, recall, _ = precision_recall_curve(y.iloc[val_idx].values,y_oof[val_idx])\n    y_real.append(y.iloc[val_idx].values)\n    y_proba.append(y_oof[val_idx])\n    ax[2].plot(recall, precision, lw=2, alpha=0.3, label='P|R fold %d' % (i))\n    \n    # Confusion matrix by folds\n    cms.append(confusion_matrix(y.iloc[val_idx].values,y_oof[val_idx].round()))\n\n#ROC\n# Vincent Lugat - https://www.kaggle.com/vincentlugat/ieee-lgb-bayesian-opt\nax[1].plot([0,1],[0,1], linestyle = '--', lw = 2, color = 'grey')\nmean_tpr = np.mean(tprs, axis=0)\nmean_auc = auc(mean_fpr, mean_tpr)\nax[1].plot(mean_fpr, mean_tpr, color='blue',\n         label=r'Mean ROC (AUC = %0.4f)' % (np.mean(roc_aucs)), lw=2, alpha=1)\n\nax[1].set_xlabel('False Positive Rate')\nax[1].set_ylabel('True Positive Rate')\nax[1].set_title('LGB ROC curve by folds')\nax[1].legend(loc=\"lower right\")\n\n# PR plt\nax[2].plot([0,1],[1,0],linestyle = '--',lw = 2,color = 'grey')\ny_real = np.concatenate(y_real)\ny_proba = np.concatenate(y_proba)\nprecision, recall, _ = precision_recall_curve(y_real, y_proba)\nax[2].plot(recall, precision, color='blue',\n         label=r'Mean P|R')\nax[2].set_xlabel('Recall')\nax[2].set_ylabel('Precision')\nax[2].set_title('P|R curve by folds')\nax[2].legend(loc=\"lower left\")\n\nplt.tight_layout(pad=0)\nplt.savefig('model_eval.png')\nplt.show()\n\n# Metrics\nprint(\n'CV roc score        : {0:.4f}, std: {1:.4f}.'.format(np.mean(roc_aucs), np.std(roc_aucs)),\n'\\nCV accuracy score   : {0:.4f}, std: {1:.4f}.'.format(np.mean(accuracies), np.std(accuracies)),\n'\\nCV recall score     : {0:.4f}, std: {1:.4f}.'.format(np.mean(recalls), np.std(recalls)),\n'\\nCV precision score  : {0:.4f}, std: {1:.4f}.'.format(np.mean(precisions), np.std(precisions)),\n'\\nCV f1 score         : {0:.4f}, std: {1:.4f}.'.format(np.mean(f1_scores), np.std(f1_scores))\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot Importance\ncols = all_feature_importance_df[[\"feature\", \"importance\"]].groupby(\"feature\").mean().sort_values(\n    by=\"importance\", ascending=False)[:50].index\nbest_features = all_feature_importance_df.loc[all_feature_importance_df.feature.isin(cols)]\nplt.figure(figsize=(8,10))\nsns.barplot(x=\"importance\", y=\"feature\", \n            data=best_features.sort_values(by=\"importance\", ascending=False))\nplt.title('LightGBM Features (avg over folds)')\nplt.tight_layout()\nplt.savefig('lgbm_importances.png')\nprint(\"All Model Runtime: %0.2f Minutes\"%((time.time() - allmodelstart)/60))\n\n# Confusion matrix \ndef plot_confusion_matrix(cm, classes,\n                          normalize = False,\n                          title = 'Confusion matrix\"',\n                          cmap = plt.cm.Blues) :\n    plt.imshow(cm, interpolation = 'nearest', cmap = cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation = 0)\n    plt.yticks(tick_marks, classes)\n\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])) :\n        plt.text(j, i, cm[i, j],\n                 horizontalalignment = 'center',\n                 color = 'white' if cm[i, j] > thresh else 'black')\n \n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n\n# Confusion maxtrix & metrics\nplt.rcParams[\"axes.grid\"] = False\n\ncm = np.average(cms, axis=0).round(1)\nclass_names = [0,1]\nplt.figure()\nplot_confusion_matrix(cm, \n                      classes=class_names, \n                      title= 'LGB Confusion matrix [averaged/folds]')\nplt.show()\n\ncm = np.std(cms, axis=0).round(2)\nclass_names = [0,1]\nplt.figure()\nplot_confusion_matrix(cm, \n                      classes=class_names, \n                      title= 'LGB Confusion matrix [STD/folds]')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"importance_cutoff = .95\n\n# # Number of useless features\nnull_importance = all_feature_importance_df[[\"feature\", \"importance\"]].groupby(\"feature\").mean().sort_values(\n    by=\"importance\", ascending=False)\n    \n# null_importance.loc[null_importance.importance < 5,:]\ncumu_imp = np.cumsum(null_importance.sort_values(\n    by=\"importance\", ascending=False)['importance'].reset_index(drop=True)) / np.sum(null_importance.importance)\n\nf, ax = plt.subplots(1,2, figsize = [10,5])\nsns.distplot(null_importance.importance, ax =ax[0])\nax[1].plot(cumu_imp ,color = 'r')\nax[0].set_title(\"Feature Importance Distribution\")\nax[0].set_xlabel(\"Importance\")\nax[1].set_title(\"Cumulative Feature Importance\")\nax[1].set_xlabel(\"Number of Features\")\nax[1].axvline(cumu_imp[cumu_imp < importance_cutoff].shape[0], color = 'black')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Fold Diversity\n\nStart thinking about staking"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Examine Correlations\nf, ax = plt.subplots(figsize=[7,4])\nsns.heatmap(pd.DataFrame(y_preds_fold).corr(),\n            annot=True, fmt=\".2f\",cbar_kws={'label': 'Correlation Coefficient'},cmap=\"plasma\",ax=ax, linewidths=.5)\nax.set_title(\"Fold Correlation Matrix\")\nplt.tight_layout(pad=1)\nfilename = 'fold_correlation_matrix.png'\nplt.savefig(filename)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mad = np.zeros([EPOCHS,EPOCHS])\ncomb = combinations(list(range(0,EPOCHS)), 2) \nfor (x1,x2) in list(comb):\n    mad[x1,x2] = np.mean(np.absolute(y_preds_fold[:,x1] - y_preds_fold[:,x2]))\n\nf, ax = plt.subplots(figsize=[7,4])\nsns.heatmap(mad*1000,\n            annot=True, fmt=\".5f\",cbar_kws={'label': 'MAD Coefficient'},cmap=\"plasma\",ax=ax, linewidths=.5)\nax.set_title(\"Fold Mean Absolute Difference Matrix\")\nplt.tight_layout(pad=1)\nplt.savefig(filename)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Short EDA"},{"metadata":{"trusted":true},"cell_type":"code","source":"cols = ['card1_count_full', 'card1','card2','card2_count_full']\nplot_df = pd.concat([X.loc[:,cols], y], axis =1 )\n\nt_r,t_c = 2, 2\nf, axes = plt.subplots(t_r,t_c, figsize = [12,8],sharex=False, sharey=False)\nrow,col = 0,0\nfor c in cols:\n    if col == t_c:\n        col = 0\n        row += 1\n    sns.kdeplot(plot_df.loc[plot_df.isFraud == 0, c], shade = True, alpha = 0.6, color = 'black', ax = axes[row,col], label = 'Not Fraud')\n    sns.kdeplot(plot_df.loc[plot_df.isFraud == 1, c], shade = True, alpha = 0.6, color = 'lime', ax = axes[row,col], label = 'Fraud')\n    axes[row,col].set_title('{} and Fraud Distribution'.format(c.title()))\n    col+=1\n    \nplt.tight_layout(pad=0)\nplt.show()\ndel plot_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Submit"},{"metadata":{"trusted":true},"cell_type":"code","source":"# When doing feature selection, make sure you use the same subset on test set.\n# LGBM will not break, but it will give you broken predictions.. -_-\nassert X[feature_subset].shape[1] == test[feature_subset].shape[1]\n\nsample_submission['isFraud'] = y_preds\nsample_submission.to_csv('{}_feats_{}fold_lgbm_gpu.csv'.format(len(feature_subset),EPOCHS))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Notebook Runtime: %0.2f Hours\"%((time.time() - notebookstart)/60/60))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}