new_kernel.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/Leslie/anaconda3/lib/python3.5/site-packages/sklearn/cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from scipy.stats import skew\n",
    "import xgboost as xgb\n",
    "from sklearn.cross_validation import KFold\n",
    "from sklearn.ensemble import ExtraTreesRegressor\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.linear_model import Ridge, RidgeCV, ElasticNet, LassoCV, Lasso\n",
    "from math import sqrt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "TARGET = 'SalePrice'\n",
    "NFOLDS = 5\n",
    "SEED = 0\n",
    "NROWS = None\n",
    "SUBMISSION_FILE = '/Users/Leslie/GitHub/HousePrice/input/sample_submission.csv'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv(\"/Users/Leslie/GitHub/HousePrice/train.csv\")\n",
    "test = pd.read_csv(\"/Users/Leslie/GitHub/HousePrice/test.csv\")\n",
    "ntrain = train.shape[0]\n",
    "ntest = test.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# y_train = np.log(train[TARGET]+1)\n",
    "y_train = train[TARGET]\n",
    "train.drop([TARGET], axis=1, inplace=True)\n",
    "all_data = pd.concat((train.loc[:,'MSSubClass':'SaleCondition'],\n",
    "                      test.loc[:,'MSSubClass':'SaleCondition']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logarithm transformation on right skewed features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "numeric_feats = all_data.dtypes[all_data.dtypes != \"object\"].index\n",
    "\n",
    "skewed_feats = train[numeric_feats].apply(lambda x: skew(x.dropna())) #compute skewness\n",
    "skewed_feats = skewed_feats[skewed_feats > 0.75]\n",
    "skewed_feats = skewed_feats.index\n",
    "\n",
    "all_data[skewed_feats] = np.log1p(all_data[skewed_feats])\n",
    "all_data = pd.get_dummies(all_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_data = all_data.fillna(all_data.mean())\n",
    "\n",
    "x_train = np.array(all_data[:train.shape[0]])\n",
    "x_test = np.array(all_data[train.shape[0]:])\n",
    "\n",
    "kf = KFold(ntrain, n_folds=NFOLDS, shuffle=True, random_state=SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class XgbWrapper(object):\n",
    "    def __init__(self, seed=0, params=None):\n",
    "        self.param = params\n",
    "        self.param['seed'] = seed\n",
    "        self.nrounds = params.pop('nrounds', 250)\n",
    "\n",
    "    def train(self, x_train, y_train):\n",
    "        dtrain = xgb.DMatrix(x_train, label=y_train)\n",
    "        self.gbdt = xgb.train(self.param, dtrain, self.nrounds)\n",
    "\n",
    "    def predict(self, x):\n",
    "        return self.gbdt.predict(xgb.DMatrix(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_oof(clf):\n",
    "    oof_train = np.zeros((ntrain,))\n",
    "    oof_test = np.zeros((ntest,))\n",
    "    oof_test_skf = np.empty((NFOLDS, ntest))\n",
    "\n",
    "    for i, (train_index, test_index) in enumerate(kf):\n",
    "        x_tr = x_train[train_index]\n",
    "        y_tr = y_train[train_index]\n",
    "        x_te = x_train[test_index]\n",
    "\n",
    "        clf.train(x_tr, y_tr)\n",
    "\n",
    "        oof_train[test_index] = clf.predict(x_te)\n",
    "        oof_test_skf[i, :] = clf.predict(x_test)\n",
    "\n",
    "    oof_test[:] = oof_test_skf.mean(axis=0)\n",
    "    return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "et_params = {\n",
    "    'n_jobs': 16,\n",
    "    'n_estimators': 100,\n",
    "    'max_features': 0.5,\n",
    "    'max_depth': 12,\n",
    "    'min_samples_leaf': 2,\n",
    "}\n",
    "\n",
    "rf_params = {\n",
    "    'n_jobs': 16,\n",
    "    'n_estimators': 100,\n",
    "    'max_features': 0.2,\n",
    "    'max_depth': 12,\n",
    "    'min_samples_leaf': 2,\n",
    "}\n",
    "\n",
    "xgb_params = {\n",
    "    'seed': 0,\n",
    "    'colsample_bytree': 0.7,\n",
    "    'silent': 1,\n",
    "    'subsample': 0.7,\n",
    "    'learning_rate': 0.075,\n",
    "    'objective': 'reg:linear',\n",
    "    'max_depth': 4,\n",
    "    'num_parallel_tree': 1,\n",
    "    'min_child_weight': 1,\n",
    "    'eval_metric': 'rmse',\n",
    "    'nrounds': 500\n",
    "}\n",
    "\n",
    "\n",
    "\n",
    "rd_params={\n",
    "    'alpha': 10\n",
    "}\n",
    "\n",
    "\n",
    "ls_params={\n",
    "    'alpha': 0.005\n",
    "}\n",
    "\n",
    "\n",
    "xg = XgbWrapper(seed=SEED, params=xgb_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "XG-CV: 25831.301777220426\n"
     ]
    }
   ],
   "source": [
    "xg_oof_train, xg_oof_test = get_oof(xg)\n",
    "print(\"XG-CV: {}\".format(sqrt(mean_squared_error(y_train, xg_oof_train))))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-rmse:10.6684+0.00673027\ttest-rmse:10.6684+0.0227829\n",
      "[10]\ttrain-rmse:4.90936+0.00261722\ttest-rmse:4.90955+0.0223482\n",
      "[20]\ttrain-rmse:2.26833+0.00167086\ttest-rmse:2.26815+0.0166228\n",
      "[30]\ttrain-rmse:1.05824+0.00170887\ttest-rmse:1.05855+0.0133175\n",
      "[40]\ttrain-rmse:0.50419+0.000822533\ttest-rmse:0.509794+0.0126884\n",
      "[50]\ttrain-rmse:0.255717+0.00122875\ttest-rmse:0.269112+0.0118271\n",
      "[60]\ttrain-rmse:0.149789+0.00182402\ttest-rmse:0.174855+0.0113745\n",
      "[70]\ttrain-rmse:0.108866+0.00202226\ttest-rmse:0.143844+0.0111975\n",
      "[80]\ttrain-rmse:0.0931667+0.00215988\ttest-rmse:0.134059+0.0109572\n",
      "[90]\ttrain-rmse:0.085783+0.00224862\ttest-rmse:0.130559+0.0111554\n",
      "[100]\ttrain-rmse:0.0806975+0.00210502\ttest-rmse:0.128605+0.0110318\n",
      "[110]\ttrain-rmse:0.0765357+0.00218548\ttest-rmse:0.127402+0.0109773\n",
      "[120]\ttrain-rmse:0.0731757+0.00191427\ttest-rmse:0.126657+0.0109343\n",
      "[130]\ttrain-rmse:0.0701193+0.0018936\ttest-rmse:0.125894+0.0111663\n",
      "[140]\ttrain-rmse:0.0672485+0.00180144\ttest-rmse:0.125288+0.0111493\n",
      "[150]\ttrain-rmse:0.0647748+0.00166621\ttest-rmse:0.124836+0.0112434\n",
      "[160]\ttrain-rmse:0.0622595+0.00161778\ttest-rmse:0.124547+0.0113347\n",
      "[170]\ttrain-rmse:0.0600612+0.00170269\ttest-rmse:0.123821+0.0111101\n",
      "[180]\ttrain-rmse:0.0578632+0.00171317\ttest-rmse:0.123701+0.0113574\n",
      "[190]\ttrain-rmse:0.055869+0.00159905\ttest-rmse:0.123523+0.0114223\n",
      "[200]\ttrain-rmse:0.0538248+0.00158514\ttest-rmse:0.123496+0.0115101\n",
      "[210]\ttrain-rmse:0.051897+0.0016029\ttest-rmse:0.123291+0.0114763\n",
      "[220]\ttrain-rmse:0.0503067+0.00168518\ttest-rmse:0.123019+0.0115228\n",
      "[230]\ttrain-rmse:0.0486015+0.00163152\ttest-rmse:0.122932+0.011469\n",
      "[240]\ttrain-rmse:0.0468818+0.00144372\ttest-rmse:0.122727+0.0115531\n",
      "[250]\ttrain-rmse:0.0454975+0.00135815\ttest-rmse:0.122819+0.0117136\n",
      "[260]\ttrain-rmse:0.04409+0.00139148\ttest-rmse:0.122649+0.0119312\n",
      "[270]\ttrain-rmse:0.042675+0.00124046\ttest-rmse:0.122577+0.0121207\n",
      "[280]\ttrain-rmse:0.041276+0.00109598\ttest-rmse:0.12252+0.0119528\n",
      "[290]\ttrain-rmse:0.03994+0.00108755\ttest-rmse:0.122518+0.011849\n",
      "[300]\ttrain-rmse:0.038713+0.00104885\ttest-rmse:0.122636+0.0117961\n"
     ]
    }
   ],
   "source": [
    "res = xgb.cv(xgb_params, dtrain, num_boost_round=1000, nfold=4, seed=SEED, stratified=False,\n",
    "             early_stopping_rounds=25, verbose_eval=10, show_stdv=True)\n",
    "\n",
    "best_nrounds = res.shape[0] - 1\n",
    "cv_mean = res.iloc[-1, 0]\n",
    "cv_std = res.iloc[-1, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dtrain = xgb.DMatrix(x_train, label=y_train)\n",
    "dtest = xgb.DMatrix(x_test)\n",
    "\n",
    "gbdt = xgb.train(xgb_params, dtrain, best_nrounds)\n",
    "\n",
    "submission = pd.read_csv(SUBMISSION_FILE)\n",
    "submission.iloc[:, 1] = gbdt.predict(dtest)\n",
    "# saleprice = np.exp(submission['SalePrice'])-1\n",
    "# submission['SalePrice'] = saleprice\n",
    "submission.to_csv('/Users/Leslie/GitHub/HousePrice/input/kaggle_kernel_sub1.csv', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}