taxi-rides-time-analysis-and-oof-lgbm.ipynb

{"cells":[{"metadata":{"_uuid":"2c5288567fc9d4c856ff891642fefb02e11bc72b"},"cell_type":"markdown","source":"## Taxi Rides Time Analysis\n_By Nick Brooks, July 2018_"},{"metadata":{"trusted":true,"_uuid":"b9e3fe792e95319100fd099494e2a9950d8a075c"},"cell_type":"code","source":"# from IPython.display import HTML\n\n# HTML('''<script>\n# code_show=true; \n# function code_toggle() {\n#  if (code_show){\n#  $('div.input').hide();\n#  } else {\n#  $('div.input').show();\n#  }\n#  code_show = !code_show\n# } \n# $( document ).ready(code_toggle);\n# </script>\n# The raw code for this IPython notebook is by default hidden for easier reading.\n# To toggle on/off the raw code, click <a href=\"javascript:code_toggle()\">here</a>.''')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import time\nnotebookstart= time.time()\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport gc\n\n# Modeling\nimport lightgbm as lgb\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import KFold\n\n# Visualization\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nDebug = False\n\n# Read data\nNROWS = 6000000\nif Debug is True: NROWS = 5000\ntrain = pd.read_csv('../input/train.csv', nrows = NROWS, index_col = \"key\")\ntrain = train.dropna()\ntest_df = pd.read_csv('../input/test.csv', index_col = \"key\")\ntestdex = test_df.index","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"61fa64614870129c4e937144434d6e88d1a22f2d"},"cell_type":"markdown","source":"**Cleaning and Feature Engineering:** <br>\nThere are a few anomalies in the fair and passenger_count features which are absent from the test set. Therefore, I shall remove them."},{"metadata":{"trusted":true,"_uuid":"6b2990405dc72eb3f55f6048116db2f524f26be8"},"cell_type":"code","source":"def prepare_distance_features(df):\n    # Distance is expected to have an impact on the fare\n    df['longitude_distance'] = abs(df['pickup_longitude'] - df['dropoff_longitude'])\n    df['latitude_distance'] = abs(df['pickup_latitude'] - df['dropoff_latitude'])\n\n    # Straight distance\n    df['distance_travelled'] = (df['longitude_distance'] ** 2 + df['latitude_distance'] ** 2) ** .5\n    df['distance_travelled_sin'] = np.sin((df['longitude_distance'] ** 2 * df['latitude_distance'] ** 2) ** .5)\n    df['distance_travelled_cos'] = np.cos((df['longitude_distance'] ** 2 * df['latitude_distance'] ** 2) ** .5)\n    df['distance_travelled_sin_sqrd'] = np.sin((df['longitude_distance'] ** 2 * df['latitude_distance'] ** 2) ** .5) ** 2\n    df['distance_travelled_cos_sqrd'] = np.cos((df['longitude_distance'] ** 2 * df['latitude_distance'] ** 2) ** .5) ** 2\n\n    # Haversine formula for distance\n    # Haversine formula:\ta = sin²(Δφ/2) + cos φ1 ⋅ cos φ2 ⋅ sin²(Δλ/2)\n    R = 6371e3 # Metres\n    phi1 = np.radians(df['pickup_latitude'])\n    phi2 = np.radians(df['dropoff_latitude'])\n    phi_chg = np.radians(df['pickup_latitude'] - df['dropoff_latitude'])\n    delta_chg = np.radians(df['pickup_longitude'] - df['dropoff_longitude'])\n    a = np.sin(phi_chg / 2) + np.cos(phi1) * np.cos(phi2) * np.sin(delta_chg / 2)\n    c = 2 * np.arctan2(a ** .5, (1-a) ** .5)\n    d = R * c\n    df['haversine'] = d\n\n    # Bearing\n    # Formula:\tθ = atan2( sin Δλ ⋅ cos φ2 , cos φ1 ⋅ sin φ2 − sin φ1 ⋅ cos φ2 ⋅ cos Δλ )\n    y = np.sin(delta_chg * np.cos(phi2))\n    x = np.cos(phi1) * np.sin(phi2) - np.sin(phi1) * np.cos(phi2) * np.cos(delta_chg)\n    df['bearing'] = np.arctan2(y, x)\n\n    return df\n\ndef prepare_time_features(df):\n    df['pickup_datetime'] = df['pickup_datetime'].str.replace(\" UTC\", \"\")\n    df['pickup_datetime'] = pd.to_datetime(df['pickup_datetime'], format='%Y-%m-%d %H:%M:%S')\n    df['hour_of_day'] = df.pickup_datetime.dt.hour\n    df['week'] = df.pickup_datetime.dt.week\n    df['month'] = df.pickup_datetime.dt.month\n    df[\"year\"] = df.pickup_datetime.dt.year\n    df['day_of_year'] = df.pickup_datetime.dt.dayofyear\n    df['week_of_year'] = df.pickup_datetime.dt.weekofyear\n    df[\"weekday\"] = df.pickup_datetime.dt.weekday\n    df[\"quarter\"] = df.pickup_datetime.dt.quarter\n    df[\"day_of_month\"] = df.pickup_datetime.dt.day\n    \n    return df\n\n# Airport Features - By Albert van Breenmen\n# https://www.kaggle.com/breemen/nyc-taxi-fare-data-exploration\ndef dist(pickup_lat, pickup_long, dropoff_lat, dropoff_long):  \n    distance = np.abs(dropoff_lat - pickup_lat) + np.abs(dropoff_long - pickup_long)\n    \n    return distance\n\ndef airport_feats(train,test_df):\n    for data in [train,test_df]:\n        nyc = (-74.0063889, 40.7141667)\n        jfk = (-73.7822222222, 40.6441666667)\n        ewr = (-74.175, 40.69)\n        lgr = (-73.87, 40.77)\n        data['distance_to_center'] = dist(nyc[1], nyc[0],\n                                          data['pickup_latitude'], data['pickup_longitude'])\n        data['pickup_distance_to_jfk'] = dist(jfk[1], jfk[0],\n                                             data['pickup_latitude'], data['pickup_longitude'])\n        data['dropoff_distance_to_jfk'] = dist(jfk[1], jfk[0],\n                                               data['dropoff_latitude'], data['dropoff_longitude'])\n        data['pickup_distance_to_ewr'] = dist(ewr[1], ewr[0], \n                                              data['pickup_latitude'], data['pickup_longitude'])\n        data['dropoff_distance_to_ewr'] = dist(ewr[1], ewr[0],\n                                               data['dropoff_latitude'], data['dropoff_longitude'])\n        data['pickup_distance_to_lgr'] = dist(lgr[1], lgr[0],\n                                              data['pickup_latitude'], data['pickup_longitude'])\n        data['dropoff_distance_to_lgr'] = dist(lgr[1], lgr[0],\n                                               data['dropoff_latitude'], data['dropoff_longitude'])\n    return train, test_df\n\n# Percentile\ndef percentile(n):\n    def percentile_(x):\n        return np.percentile(x, n)\n    percentile_.__name__ = 'percentile_%s' % n\n    return percentile_\n\n# Build ime Aggregate Features\ndef time_agg(train, test_df, vars_to_agg, vars_be_agg):\n    for var in vars_to_agg:\n        agg = train.groupby(var)[vars_be_agg].agg([\"sum\",\"mean\",\"std\",\"skew\",percentile(80),percentile(20)])\n        if isinstance(var, list):\n            agg.columns = pd.Index([\"fare_by_\" + \"_\".join(var) + \"_\" + str(e) for e in agg.columns.tolist()])\n        else:\n            agg.columns = pd.Index([\"fare_by_\" + var + \"_\" + str(e) for e in agg.columns.tolist()]) \n        train = pd.merge(train,agg, on=var, how= \"left\")\n        test_df = pd.merge(test_df,agg, on=var, how= \"left\")\n    \n    return train, test_df\n\n# Clean dataset from https://www.kaggle.com/gunbl4d3/xgboost-ing-taxi-fares\ndef clean_df(df):\n    return df[(df.fare_amount > 0) & \n            (df.pickup_longitude > -80) & (df.pickup_longitude < -70) &\n            (df.pickup_latitude > 35) & (df.pickup_latitude < 45) &\n            (df.dropoff_longitude > -80) & (df.dropoff_longitude < -70) &\n            (df.dropoff_latitude > 35) & (df.dropoff_latitude < 45)]\nprint(\"Cleaning Functions Defined..\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8cf25469845c819363a98c55099ad9bffdf563bc"},"cell_type":"markdown","source":"**Run All Pre-Processing and Cleaning:**"},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"408288dea6144cbac692a343312fd5917f31fcc6"},"cell_type":"code","source":"print(\"Percent of Training Set with Zero and Below Fair: \", round(((train.loc[train[\"fare_amount\"] <= 0, \"fare_amount\"].shape[0]/train.shape[0]) * 100),5))\nprint(\"Percent of Training Set 200 and Above Fair: \", round((train.loc[train[\"fare_amount\"] >= 200, \"fare_amount\"].shape[0]/train.shape[0]) * 100,5))\ntrain = train.loc[(train[\"fare_amount\"] > 0) & (train[\"fare_amount\"] <= 200),:]\nprint(\"\\nPercent of Training Set with Zero and Below Passenger Count: \", round((train.loc[train[\"passenger_count\"] <= 0, \"passenger_count\"].shape[0]/train.shape[0]) * 100,5))\nprint(\"Percent of Training Set with Nine and Above Passenger Count: \", round((train.loc[train[\"passenger_count\"] >= 9, \"passenger_count\"].shape[0]/train.shape[0]) * 100,5))\ntrain = train.loc[(train[\"passenger_count\"] > 0) & (train[\"passenger_count\"] <= 9),:]\n\n# Clean Training Set\ntrain = clean_df(train)\n\n# Distance Features\ntrain = prepare_distance_features(train)\ntest_df = prepare_distance_features(test_df)\ntrain,test_df = airport_feats(train,test_df)\n\n# Time Features\ntrain = prepare_time_features(train)\ntest_df = prepare_time_features(test_df)\n\n# Ratios\ntrain[\"fare_to_dist_ratio\"] = train[\"fare_amount\"] / ( train[\"distance_travelled\"]+0.0001)\ntrain[\"fare_npassenger_to_dist_ratio\"] = (train[\"fare_amount\"] / train[\"passenger_count\"]) /( train[\"distance_travelled\"]+0.0001)\n\n# Time Aggregate Features\ntrain, test_df = time_agg(train, test_df,\n                          vars_to_agg  = [\"passenger_count\", \"weekday\", \"quarter\", \"month\", \"year\", \"hour_of_day\",\n                                          [\"weekday\", \"month\", \"year\"], [\"hour_of_day\", \"weekday\", \"month\", \"year\"]],\n                          vars_be_agg = \"fare_amount\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6f910c80b1d7eb60b38ff2b27b4bf21118416f89"},"cell_type":"markdown","source":"**Understanding Time Range:**"},{"metadata":{"trusted":true,"_uuid":"3a71666c7cac2455e944c8dc29f61189a9c4c71e"},"cell_type":"code","source":"train_time_start = train.pickup_datetime.min()\ntrain_time_end = train.pickup_datetime.max()\nprint(\"Train Time Starts: {}, Ends {}\".format(train_time_start,train_time_end))\ntest_time_start = test_df.pickup_datetime.min()\ntest_time_end = test_df.pickup_datetime.max()\nprint(\"Test Time Starts: {}, Ends {}\".format(test_time_start,test_time_end))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"10a240174eb86fe9f20324971814fe33d15eb495"},"cell_type":"markdown","source":"## Visualization"},{"metadata":{"trusted":true,"_uuid":"b2c3733a9d6251c0f50ae8361134dbe023841dd0"},"cell_type":"code","source":"f, ax = plt.subplots(1,2,figsize = [10,5])\nsns.countplot(train[\"passenger_count\"], ax=ax[0])\nsns.countplot(test_df[\"passenger_count\"], ax=ax[1])\nax[0].set_title(\"Train Set - Passenger Count\")\nax[1].set_title(\"Test Set - Passenger Count\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a4ca85612c943d93ff19e886c19a614c1d11a4a7"},"cell_type":"code","source":"f, ax = plt.subplots(figsize=[6,5])\nsns.kdeplot(train[\"fare_amount\"], ax=ax)\nax.set_title(\"Fare Distribution\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6bcc5596099639535ae856262c71c030f130017c"},"cell_type":"code","source":"def time_slicer(df, timeframes, value, color=\"purple\"):\n    \"\"\"\n    Function to count observation occurrence through different lenses of time.\n    \"\"\"\n    f, ax = plt.subplots(len(timeframes), figsize = [12,12])\n    for i,x in enumerate(timeframes):\n        df.loc[:,[x,value]].groupby([x]).mean().plot(ax=ax[i],color=color)\n        ax[i].set_ylabel(value.replace(\"_\", \" \").title())\n        ax[i].set_title(\"{} by {}\".format(value.replace(\"_\", \" \").title(), x.replace(\"_\", \" \").title()))\n        ax[i].set_xlabel(\"\")\n    ax[len(timeframes)-1].set_xlabel(\"Time Frame\")\n    plt.tight_layout(pad=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"afa5e26ba8b05e18d4e2e733090d0cb4aba474ad"},"cell_type":"code","source":"time_slicer(df=train, timeframes=['year',\"day_of_year\", \"month\", \"day_of_month\", \"weekday\", \"hour_of_day\"], value = \"fare_amount\", color=\"blue\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5af4e81f1dd840de4536a4b1ca2c4172381e663d"},"cell_type":"code","source":"time_slicer(df=train, timeframes=['year',\"day_of_year\", \"month\", \"day_of_month\", \"weekday\", \"hour_of_day\"], value = \"distance_travelled\", color = \"green\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"39c5050e79b2071074dd1cb23325c139df62343f"},"cell_type":"code","source":"time_slicer(df=train, timeframes=['year',\"day_of_year\", \"month\", \"day_of_month\", \"weekday\", \"hour_of_day\"], value = \"fare_to_dist_ratio\", color = \"red\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4a8e393b8663a8503fa2bc647c780d2b9c027e44","scrolled":false},"cell_type":"code","source":"time_slicer(df=train, timeframes=['year',\"day_of_year\", \"month\", \"day_of_month\", \"weekday\", \"hour_of_day\"], value = \"fare_npassenger_to_dist_ratio\", color = \"orange\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"c68833e7d1151f75db5100c12cd02c30103d1ee0"},"cell_type":"markdown","source":"## Modeling with LGBM"},{"metadata":{"trusted":true,"_uuid":"d26bf69119697167ab000b8abac2ead16a52f18a"},"cell_type":"code","source":"# Keep Relevant Variables..\ny = train.fare_amount.copy()\ntest_df.drop(\"pickup_datetime\", axis = 1, inplace=True)\ntrain = train[test_df.columns]\nprint(\"Does Train feature equal test feature?: \", all(train.columns == test_df.columns))\ntrainshape = train.shape\ntestshape = test_df.shape\n\n# print(\"\\nTrain DF..\")\n# train = reduce_mem_usage(train)\n# print(\"\\nTest DF..\")\n# test_df = reduce_mem_usage(test_df)\n\n# LGBM Dataset Formating\ndtrain = lgb.Dataset(train, label=y, free_raw_data=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d83da628984c8d72a5ef25aceff194816b5b2b29"},"cell_type":"markdown","source":"**Out of Fold Prediction Ensemble:**"},{"metadata":{"trusted":true,"_uuid":"87666ffd1c50913abe945539affc29fcd9a03d3b"},"cell_type":"code","source":"print(\"Light Gradient Boosting Regressor: \")\nlgbm_params =  {\n    'task': 'train',\n    'boosting_type': 'gbdt',\n    'objective': 'regression',\n    'metric': 'rmse'\n                }\n\nfolds = KFold(n_splits=5, shuffle=True, random_state=1)\nfold_preds = np.zeros(testshape[0])\noof_preds = np.zeros(trainshape[0])\ndtrain.construct()\n\n# Fit 5 Folds\nmodelstart = time.time()\nfor trn_idx, val_idx in folds.split(train):\n    clf = lgb.train(\n        params=lgbm_params,\n        train_set=dtrain.subset(trn_idx),\n        valid_sets=dtrain.subset(val_idx),\n        num_boost_round=3500, \n        early_stopping_rounds=125,\n        verbose_eval=500\n    )\n    oof_preds[val_idx] = clf.predict(dtrain.data.iloc[val_idx])\n    fold_preds += clf.predict(test_df) / folds.n_splits\n    print(mean_squared_error(y.iloc[val_idx], oof_preds[val_idx]) ** .5)\nprint(\"Model Runtime: %0.2f Minutes\"%((time.time() - modelstart)/60))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4c574655ba4903afa3fdebe8df9b823051d4b649"},"cell_type":"code","source":"lgsub = pd.DataFrame(fold_preds,columns=[\"fare_amount\"],index=testdex)\nlgsub.to_csv(\"lgsub.csv\",index=True,header=True)\n\nprint(\"Notebook Runtime: %0.2f Minutes\"%((time.time() - notebookstart)/60))\nlgsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ff9db9d9d318aa5ce070e21c937c48050aabbc24"},"cell_type":"code","source":"## Big shoutout to those who build many of my features.","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}