regression-models-on-house-data.ipynb

{"cells":[{"metadata":{"_uuid":"b37db2e2b557eef328277257e6ca1b0b8b63946e","_cell_guid":"ced30db8-1324-4beb-b81d-3de926049599"},"cell_type":"markdown","source":"## Supervised Regression- House Price Data\n\nResources:\n- Visualize Parameter Search: https://blancas.io/sklearn-evaluation/user_guide/grid_search.html\n- Scorer Sklearn and Mass-Histograms: https://www.kaggle.com/liyenhsu/feature-selection-and-ensemble-of-5-models/notebook\n- https://www.kaggle.com/juliencs/a-study-on-regression-applied-to-the-ames-dataset"},{"metadata":{"_uuid":"ac53ffddebaee985832f3217be9669bbf7ac6018","_cell_guid":"0cc89935-b942-4a5d-98ce-f2056ea95838","collapsed":true,"trusted":false},"cell_type":"code","source":"# General\nimport numpy as np\nimport pandas as pd\nimport os\nimport scipy.stats as st\nimport pickle\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\n# Evalaluation\nfrom sklearn import metrics\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.metrics import mean_squared_error, make_scorer\n\n# Grid\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import StratifiedShuffleSplit\nfrom sklearn.feature_selection import RFE\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.model_selection import RandomizedSearchCV\nfrom sklearn.model_selection import train_test_split\n\n# Models\n# Linear Regression\nfrom sklearn import linear_model\nfrom sklearn.linear_model import Ridge\n\n# XGBoost\nimport xgboost as xgb\nfrom xgboost.sklearn import XGBRegressor  \n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7fd05240ced0f96e462e8469383cf7bc2cf001e2","_cell_guid":"b9ca0de7-8291-4082-913e-c5f56a341be9","collapsed":true,"trusted":false},"cell_type":"code","source":"import os\nprint(os.listdir('../input/feature-engineering-and-pre-processing-house-data'))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"89df868fdd2b73b146455bdd05150103cf2b02a3","_cell_guid":"07bad570-d5e4-4a00-a404-43aa4478f889","collapsed":true,"trusted":false},"cell_type":"code","source":"# Load\ntrain_df = pd.read_csv(\"../input/feature-engineering-and-pre-processing-house-data/house_train.csv\", index_col='Id')\ntest_df = pd.read_csv(\"../input/feature-engineering-and-pre-processing-house-data/house_test.csv\", index_col='Id')\n\n# Log\ntrain_df['SalePrice']= np.log(train_df['SalePrice'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"36f381b16c4b806aaf379dbf1bcc5d4534c11709","_cell_guid":"206db9e7-28f3-444d-aa1f-8a81e793527c","collapsed":true,"trusted":false},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a58a774f1c394a3560270f08dde4038cf0a22173","_cell_guid":"8118fae1-2fca-40fa-b1ac-a345c80a360b","collapsed":true,"trusted":false},"cell_type":"code","source":"# Hyper-Parameter\nn_inter = 25\ncv = 5\nrstate = 23\nscore_name = \"Root Mean Square Error\"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2202d828f36a1d18df339d86a6dcd1e472ff6706","_cell_guid":"a40f04a2-984c-45bd-8270-85f6b7e455ff","collapsed":true,"trusted":false},"cell_type":"code","source":"# Define a function to calculate Root Mean Sqaure Error\ndef rmse(y_true, y_pred):\n    return np.sqrt(np.mean((y_true-y_pred)**2))\n\n# Define a function to calculate negative RMSE (as a score)\ndef nrmse(y_true, y_pred):\n    return -1.0*rmse(y_true, y_pred)\n\n#neg_rmse = make_scorer(nrmse)\nscoring = make_scorer(rmse, greater_is_better=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ce0893b0cc647125c8a7a0a046f29fe797c5382e","_cell_guid":"e0e8d8a0-7c9b-4576-93dd-dbdfc481a4fa","collapsed":true,"trusted":false},"cell_type":"code","source":"# Utility function to report best scores\ndef report(results, n_top=3):\n    for i in range(1, n_top + 1):\n        candidates = np.flatnonzero(results['rank_test_score'] == i)\n        for candidate in candidates:\n            print(\"Model with rank: {0}\".format(i))\n            print(\"Mean validation score: {0:.3f} (std: {1:.3f})\".format(\n                  results['mean_test_score'][candidate],\n                  results['std_test_score'][candidate]))\n            print(\"Parameters: {0}\".format(results['params'][candidate]))\n            print(\"\")\n\n# Storage for Model and Results\nresults = pd.DataFrame(columns=['Model','Para','Test_Score','CV Mean','CV STDEV'])\ndef save(model, modelname):\n    global results\n    model.best_estimator_.fit(X, y)\n    submission =  np.exp(model.predict(test_df))\n    \n    df = pd.DataFrame({'Id':test_df.index, \n                        'SalePrice':submission})\n    df.to_csv(\"{}.csv\".format(modelname),header=True,index=False)\n    \n    model.best_estimator_.fit(X_train, y_train)\n    top = np.flatnonzero(grid.cv_results_['rank_test_score'] == 1)\n    CV_scores = grid.cv_results_['mean_test_score'][top]*-1\n    STDev = grid.cv_results_['std_test_score'][top]\n    Test_scores = rmse(y_test, model.predict(X_test))\n    \n    # CV and Save Scores\n    results = results.append({'Model': modelname,'Para': model.best_params_,'Test_Score': Test_scores,\n                             'CV Mean':CV_scores, 'CV STDEV': STDev}, ignore_index=True)\n    \n    # Print Evaluation\n    print(\"\\nEvaluation Method: {}\".format(score_name))\n    print(\"Optimal Model Parameters: {}\".format(grid.best_params_))\n    print(\"Training RMSE: \", rmse(y_train, model.predict(X_train)))\n    print(\"Train CV Accuracy: %0.2f (+/- %0.2f) [%s]\" % (CV_scores, STDev, modelname))\n    print('Test_Score:', Test_scores)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d74b7ddf6e26660d32d49a0efb4a30d345ec3ea7","_cell_guid":"8a09b942-d905-483d-8cf1-d4a5ec6bb923","collapsed":true,"trusted":false},"cell_type":"code","source":"# Indepedent and Dependent\nX = train_df.drop([\"SalePrice\"] , axis=1)\ny = train_df[\"SalePrice\"]\n\n# use train/test split with different random_state values\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\nprint(\"X_train Shape: {}\".format(X_train.shape), \"\\ny_train Shape: {}\".format(y_train.shape),\n      \"\\nX_test Shape: {}\".format(X_test.shape), \"\\ny_test Shape: {}\".format(y_test.shape))\n\nprint(\"\\nDo Train and Submission Set Columns Match? {}\".format(X.columns.equals(test_df.columns)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6f1befc633f7db51de10968fec404030af1315f8","_cell_guid":"7d02da83-31a3-4759-84ce-391052963833"},"cell_type":"markdown","source":"# Models\n## Linear Regression"},{"metadata":{"_uuid":"899a9823abd812de652130a86e9bba8e2297eb9e","_cell_guid":"3cc68ecc-51f8-4be5-82ea-04f62f1e698b","collapsed":true,"trusted":false},"cell_type":"code","source":"linear_model.LinearRegression().get_params().keys()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3fb0df51fbbcb32cc121ebe62ea68669f8da3af7","_cell_guid":"5a3578d4-325a-4f59-bd36-64d152d89ccb","collapsed":true,"trusted":false},"cell_type":"code","source":"model = linear_model.LinearRegression()\nscore = cross_val_score(model, X_train, y_train, cv=2, scoring=scoring)\nprint(score.mean())\nmodel.fit(X_train,y_train)\nprint(rmse(y_test, model.predict(X_test)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a6d324429c05fa2de5a7b3120e5bbfc2cdebb339","_cell_guid":"bef014d1-47ef-4da9-b5ee-a3ff83e0efc1"},"cell_type":"markdown","source":"## Ridge Regression"},{"metadata":{"_uuid":"da551423bc23e97cec4c1504db1c8838ab4964ea","_cell_guid":"09666cb3-73e1-41cb-b157-339c3e8c0dbb","collapsed":true,"trusted":false},"cell_type":"code","source":"Ridge().get_params().keys()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9b95cba69729d7b6fdcb2938bc710ef5e1c9b404","_cell_guid":"2f69421a-a0f1-4e12-8ea3-527cb7cd9b53","collapsed":true,"trusted":false},"cell_type":"code","source":"model = Ridge()\n\nalpha= st.beta(10, 1)\nalpha = [1000,100,10, 1, 0.1, 0.01, 0.001,0.0001]\nalpha = np.logspace(4,-4,10)\n\nparam_grid = {'alpha': alpha}\n\ngrid = RandomizedSearchCV(model, param_grid,\n                          cv=cv, verbose=1, scoring=scoring,\n                         n_iter=len(alpha)-1, random_state=rstate)\n\ngrid.fit(X_train, y_train)\nsave(grid, \"Ridge\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8b2393f723a242c79bdc6a042caf2e95dce15a37","_cell_guid":"c4e808f4-85ac-48f8-9210-24437c9fe517","collapsed":true,"trusted":false},"cell_type":"code","source":"report(grid.cv_results_)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"562aa75a147f54b979a648923a40341f31acf323","_cell_guid":"d813f115-b199-457d-a9a8-f2f14fc52152"},"cell_type":"markdown","source":"## XGBoost"},{"metadata":{"scrolled":true,"_cell_guid":"f0be3e01-f440-4ba1-b4ce-262f46d11012","_uuid":"d0500fb16e71974248e37c2f0d9041d162ac8f77","collapsed":true,"trusted":false},"cell_type":"code","source":"# Human Analog Model\n# https://www.kaggle.com/humananalog/xgboost-lasso/code\nregr = xgb.XGBRegressor(\n                 colsample_bytree=0.2,\n                 gamma=0.0,\n                 learning_rate=0.01,\n                 max_depth=4,\n                 min_child_weight=1.5,\n                 n_estimators=7200,                                                                  \n                 reg_alpha=0.9,\n                 reg_lambda=0.6,\n                 subsample=0.2,\n                 seed=42,\n                 silent=1)\n\nregr.fit(X_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ab46b816cab13eb9a9529c0ef817b4f40d011786","_cell_guid":"ee0971c2-1059-4541-9e0e-824015eb07cc","collapsed":true,"trusted":false},"cell_type":"code","source":"f, ax = plt.subplots(figsize=[8,8])\nxgb.plot_importance(regr,max_num_features=50,ax=ax)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"efd150e951add74143efbad142144ddad105b1b1","_cell_guid":"f0291352-4c30-483d-8296-3925be509f11","collapsed":true,"trusted":false},"cell_type":"code","source":"# Run prediction on training set to get a rough idea of how well it does.\ny_pred = regr.predict(X_train)\nprint(\"XGBoost score on training set: \", rmse(y_train, y_pred))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7a2f5daa2f1ff9bfac57ec2fc4cadec3f30fcbb4","_cell_guid":"9c614809-f343-4996-8019-9f16bcfc08d6","collapsed":true,"trusted":false},"cell_type":"code","source":"regr.evals_result","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"62e4214fbdee6ee0c033878a60f02ec8ee9e18e9","_cell_guid":"deaae898-4738-42b7-9f87-bd981e3598fb","collapsed":true,"trusted":false},"cell_type":"code","source":"#save()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4af93b705edbc9f1507d91cec6aa2d15c6c0c40e","_cell_guid":"f5144093-f46c-489a-9bce-8d256cb4d638","collapsed":true,"trusted":false},"cell_type":"code","source":"# y_pred = np.exp(pred_xgb)\n\n# pred_df = pd.DataFrame(y_pred, index=test_df.index, columns=[\"SalePrice\"])\n# #pred_df.to_csv('output.csv', header=True, index_label='Id')\n# pred_df.to_csv((os.path.join(path, r\"submission/output.csv\")),header=True, index_label='Id')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"342501f6840203c7f2663e3f697c92accbf93cbe","_cell_guid":"4d186b36-91f0-486b-862b-f78d64500325"},"cell_type":"markdown","source":"### Nick's Optimized XGBoost\n\n- General Parameters: Guide the overall functioning\n- Booster Parameters: Guide the individual booster (tree/regression) at each step\n- Learning Task Parameters: Guide the optimization performed\n\n"},{"metadata":{"_uuid":"41c1fda70931631063267e6bbb5fd3cbd32d5cba","_cell_guid":"498a0473-189e-4e67-9487-355d52834ce0"},"cell_type":"markdown","source":"## Sklearn XGBOOST"},{"metadata":{"scrolled":true,"_cell_guid":"4017c971-9ec4-431d-8f20-43c2b468e024","_uuid":"383d79cadc5ffa533d5c06b4f165e190b37226c6","collapsed":true,"trusted":false},"cell_type":"code","source":"one_to_left = st.beta(10, 1)  \nfrom_zero_positive = st.expon(0, 50)\n\nparams = {  \n    \"learning_rate\": st.uniform(0.001, 0.3),\n    \"colsample_bytree\": one_to_left,\n    \"max_depth\": st.randint(1, 40),\n    \"gamma\": st.uniform(0, 10),\n    'reg_alpha': from_zero_positive,\n    \"n_estimators\": st.randint(3, 100),\n    \"min_child_weight\": from_zero_positive,\n    \"subsample\": one_to_left\n}\n\nmodel = XGBRegressor(nthread=-1)\n\ngrid = RandomizedSearchCV(model, params, cv=cv,\n                          scoring=scoring,\n                          n_jobs=1, verbose=1,\n                         n_iter=5)  \n\n\ngrid.fit(X_train, y_train)\nsave(grid, \"XGboostRand\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"46482d1f41762475bf7985ccc0b9981160414551","_cell_guid":"3caf5efa-96ff-4fdf-967f-82b66810e35c","collapsed":true,"trusted":false},"cell_type":"code","source":"np.logspace(2,-4,6), np.arange(50, 301, 25)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5f43e15615148b3629ec566c3a1095a98cc30c82","_cell_guid":"bdba4542-692b-4284-b8f1-ed37b7f0abf6","collapsed":true,"trusted":false},"cell_type":"code","source":"results","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2646a5bdbd1b7770a8bd1d7d5706280b713bc326","_cell_guid":"d2400621-e23d-43c2-980a-4f6bbd6f2f1b"},"cell_type":"markdown","source":"### GridSearchCV"},{"metadata":{"_uuid":"d78a5e621c1400d90c11603183c1d656b2d31b99","_cell_guid":"6d44698b-011d-407a-bebf-aeb6c3c037c7","collapsed":true,"trusted":false},"cell_type":"code","source":"from xgboost.sklearn import XGBRegressor  \nimport scipy.stats as st\n\none_to_left = st.beta(10, 1)  \nfrom_zero_positive = st.expon(0, 50)\n\nparams = {  \n    \"learning_rate\": st.uniform(0.001, 0.3),\n    \"colsample_bytree\": one_to_left,\n    \"max_depth\": st.randint(1, 40),\n    \"gamma\": st.uniform(0, 10),\n    'reg_alpha': from_zero_positive,\n    \"n_estimators\": st.randint(3, 100),\n    \"min_child_weight\": from_zero_positive,\n    \"subsample\": one_to_left\n}\n\nmodel = XGBRegressor(nthread=-1)\n\ngrid = RandomizedSearchCV(model, params, cv=cv,\n                          scoring=scoring,\n                          n_jobs=1, verbose=1)  \n\n\ngrid.fit(X_train, y_train)\nsave(grid.best_estimator_, \"XGboostGrid\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0b6217583328861af40713179bbbeb8cefe1e5d7","_cell_guid":"d3628c56-0f8e-4701-b523-541e90544ca7"},"cell_type":"markdown","source":"## Regularized Linear Models"},{"metadata":{"_uuid":"bed56ef66212609351a348308559515735e8db2b","_cell_guid":"17df23d4-0864-40d3-b7c2-f41d92153570","collapsed":true,"trusted":false},"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\n\nfrom sklearn.metrics import r2_score\nfrom sklearn.linear_model import Lasso\n%matplotlib inline\nfrom matplotlib.pylab import rcParams\nrcParams['figure.figsize'] = 12, 6\n\n\nalpha = 0.1\nlasso = Lasso(alpha=alpha)\n\ny_pred_lasso = lasso.fit(X_train, y_train).predict(X_test)\nr2_score_lasso = r2_score(y_test, y_pred_lasso)\nprint(lasso)\nprint(\"r^2 on test data : %f\" % r2_score_lasso)\n\n# #############################################################################\n# ElasticNet\nfrom sklearn.linear_model import ElasticNet\n\nenet = ElasticNet(alpha=alpha, l1_ratio=0.7)\n\ny_pred_enet = enet.fit(X_train, y_train).predict(X_test)\nr2_score_enet = r2_score(y_test, y_pred_enet)\nprint(enet)\nprint(\"r^2 on test data : %f\" % r2_score_enet)\n\nplt.plot(enet.coef_, color='lightgreen', linewidth=2,\n         label='Elastic net coefficients')\nplt.plot(lasso.coef_, color='gold', linewidth=2,\n         label='Lasso coefficients')\n#plt.plot(coef, '--', color='navy', label='original coefficients')\nplt.legend(loc='best')\nplt.title(\"Lasso R^2: %f, Elastic Net R^2: %f\"\n          % (r2_score_lasso, r2_score_enet))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0ebc08b016bfc993c687d437479c4df418484f52","_cell_guid":"b9a1982a-6d20-49e7-a72b-c506759677b5","collapsed":true,"trusted":false},"cell_type":"code","source":"results","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0214ffde864eae2d55c1a92d030c58377fb4694b","_cell_guid":"abac8461-f144-48dd-80ae-27d38815a045","collapsed":true,"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}