From c58c567329f15fdefffaeae0898361b80a648439 Mon Sep 17 00:00:00 2001 From: Blunde1 Date: Mon, 1 Aug 2022 21:28:15 +0200 Subject: [PATCH 1/3] Introduce lossfunction enum --- R-package/inst/include/loss_functions.hpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/R-package/inst/include/loss_functions.hpp b/R-package/inst/include/loss_functions.hpp index 6c8aba6..7f8c55c 100644 --- a/R-package/inst/include/loss_functions.hpp +++ b/R-package/inst/include/loss_functions.hpp @@ -8,6 +8,16 @@ // ----------- LOSS -------------- namespace loss_functions { + enum LossFunction + { + MSE, + LOGLOSS, + POISSON, + GAMMANEGINV, + GAMMALOG, + NEGBINOM + }; + double link_function(double pred_observed, std::string loss_function){ // Returns g(mu) From 0ad9fbf5adabbe6a44f86e28b82191872be54f71 Mon Sep 17 00:00:00 2001 From: Blunde1 Date: Mon, 1 Aug 2022 22:07:43 +0200 Subject: [PATCH 2/3] Require functions in lossfunction namespace to utilize lossfunction enum --- R-package/inst/include/loss_functions.hpp | 202 ++++++---------------- 1 file changed, 52 insertions(+), 150 deletions(-) diff --git a/R-package/inst/include/loss_functions.hpp b/R-package/inst/include/loss_functions.hpp index 7f8c55c..fba44c2 100644 --- a/R-package/inst/include/loss_functions.hpp +++ b/R-package/inst/include/loss_functions.hpp @@ -10,49 +10,53 @@ namespace loss_functions { enum LossFunction { - MSE, - LOGLOSS, - POISSON, - GAMMANEGINV, - GAMMALOG, - NEGBINOM + MSE, // Gaussian distribution with identity-link (mean squared error) + LOGLOSS, // Bernoulli distribution with logit-link + POISSON, // Poisson distribution with log-link + GAMMANEGINV, // Gamma distribution with negative-inverse link + GAMMALOG, // Gamma dsitribution with log-link + NEGBINOM // Negative binomial distribution with log-link }; - double link_function(double pred_observed, std::string loss_function){ + double link_function(double pred_observed, LossFunction loss_function){ // Returns g(mu) double pred_transformed=0.0; - if(loss_function=="mse"){ + switch(loss_function) + { + case MSE: pred_transformed = pred_observed; - }else if(loss_function=="logloss"){ + case LOGLOSS: pred_transformed = log(pred_observed) - log(1 - pred_observed); - }else if(loss_function=="poisson"){ + case POISSON: pred_transformed = log(pred_observed); - }else if(loss_function=="gamma::neginv"){ + case GAMMANEGINV: pred_transformed = - 1.0 / pred_observed; - }else if(loss_function=="gamma::log"){ + case GAMMALOG: pred_transformed = log(pred_observed); - }else if(loss_function=="negbinom"){ + case NEGBINOM: pred_transformed = log(pred_observed); } return pred_transformed; } - double inverse_link_function(double pred_transformed, std::string loss_function){ + double inverse_link_function(double pred_transformed, LossFunction loss_function){ // Returns g^{-1}(pred) double pred_observed = 0.0; - if(loss_function=="mse"){ + switch(loss_function) + { + case MSE: pred_observed = pred_transformed; - }else if(loss_function=="logloss"){ + case LOGLOSS: pred_observed = 1.0 / (1.0+exp(-pred_transformed)); - }else if(loss_function=="poisson"){ + case POISSON: pred_observed = exp(pred_transformed); - }else if(loss_function=="gamma::neginv"){ - pred_observed = -1.0 / pred_transformed;; - }else if(loss_function=="gamma::log"){ + case GAMMANEGINV: + pred_observed = -1.0 / pred_transformed; + case GAMMALOG: pred_observed = exp(pred_transformed); - }else if(loss_function=="negbinom"){ + case NEGBINOM: pred_observed = exp(pred_transformed); } return pred_observed; @@ -62,74 +66,43 @@ namespace loss_functions { double loss( Tvec &y, Tvec &pred, - std::string loss_type, + LossFunction loss_function, Tvec &w, double extra_param=0.0 ){ // Evaluates the loss function at pred int n = y.size(); double res = 0; - - if(loss_type=="mse"){ - // MSE + switch(loss_function) + { + case MSE: for(int i=0; i0, LOG LINK - // for(int i=0; i0) - // } - // }else if(loss_type=="zero_inflation"){ - // // ZERO-INFLATION PROBABILITY MIX - // Tvec lprob_weights = ens_ptr->param["log_prob_weights"]; - // for(int i=0; i 0){ - // // avoid comparing equality to zero... - // res += pred[i] + log(1.0+exp(-pred[i])) - lprob_weights[i]; // Weight is log probability weight!! - // }else{ - // // get y[i] == 0 - // res += -log(1.0/(1.0+exp(-pred[i])) + (1.0 - 1.0/(1.0+exp(-pred[i])))*exp(lprob_weights[i]) ); - // } - // } - // }else if(loss_type=="negbinom::zinb"){ - // // NEGBINOM COND Y>0, LOG LINK - // double dispersion = ens_ptr -> extra_param; - // for(int i=0; i0) - // } - // } - return res/n; } @@ -138,72 +111,40 @@ namespace loss_functions { Tvec dloss( Tvec &y, Tvec &pred, - std::string loss_type, + LossFunction loss_function, double extra_param=0.0 ){ // Returns the first order derivative of the loss function at pred int n = y.size(); Tvec g(n); - - if(loss_type == "mse"){ - // MSE + switch(loss_function) + { + case MSE: for(int i=0; i0, LOG LINK - // for(int i=0; i lprob_weights = ens_ptr->param["log_prob_weights"]; - // for(int i=0; i 0){ - // // avoid comparing equality to zero... - // g[i] = exp(pred[i]) / (exp(pred[i]) + 1.0); - // }else{ - // // get y[i] == 0 - // g[i] = (exp(lprob_weights[i])-1.0)*exp(pred[i]) / ( (exp(pred[i])+1.0)*(exp(lprob_weights[i])+exp(pred[i])) ); - // } - // } - // }else if(loss_type=="negbinom::zinb"){ - // // NEGBINOM COND Y>0, LOG LINK - // double dispersion = ens_ptr -> extra_param; - // for(int i=0; i ddloss( Tvec &y, Tvec &pred, - std::string loss_type, + LossFunction loss_function, double extra_param=0.0 ){ // Returns the second order derivative of the loss function at pred int n = y.size(); Tvec h(n); - - if( loss_type == "mse" ){ - // MSE + switch(loss_function) + { + case MSE: for(int i=0; i0, LOG LINK - // for(int i=0; i lprob_weights = ens_ptr->param["log_prob_weights"]; - // for(int i=0; i 0){ - // // avoid comparing equality to zero... - // h[i] = exp(pred[i]) / ((exp(pred[i]) + 1.0)*(exp(pred[i]) + 1.0)); - // }else{ - // // get y[i] == 0 - // h[i] = -(exp(lprob_weights[i])-1.0)*exp(pred[i])*(exp(2.0*pred[i])-exp(lprob_weights[i])) / - // ( (exp(pred[i])+1.0)*(exp(pred[i])+1.0)*(exp(lprob_weights[i])+exp(pred[i]))*(exp(lprob_weights[i])+exp(pred[i])) ); - // } - // } - // }else if(loss_type=="negbinom::zinb"){ - // // NEGBINOM COND Y>0, LOG LINK - // double dispersion = ens_ptr -> extra_param; - // for(int i=0; i0)) - // -dispersion*dispersion*exp(pred[i])* - // ((exp(pred[i])-1.0)*exp(dispersion*(log(dispersion+exp(pred[i]))-log(dispersion))) +1.0 ) / - // (exp(2.0*log(dispersion+exp(pred[i]))) * - // pow(exp(dispersion*(log(dispersion+exp(pred[i]))-log(dispersion))) - 1.0, 2.0 ) ); - // } - // } - return h; } } From 9078ade640683d6a14e98b26fade4d3d9d9a622d Mon Sep 17 00:00:00 2001 From: Blunde1 Date: Sat, 20 Aug 2022 17:52:18 +0200 Subject: [PATCH 3/3] Use lossfunction enum --- R-package/R/gbt.pred.R | 17 ++++-- R-package/R/gbt.train.R | 22 ++++++- R-package/inst/include/agtboost.hpp | 2 +- R-package/inst/include/ensemble.hpp | 9 +-- R-package/inst/include/loss_functions.hpp | 71 +++++++++++++++++++---- R-package/src/agtboost.cpp | 51 +++++++++------- 6 files changed, 131 insertions(+), 41 deletions(-) diff --git a/R-package/R/gbt.pred.R b/R-package/R/gbt.pred.R index 08fb546..d22aa37 100644 --- a/R-package/R/gbt.pred.R +++ b/R-package/R/gbt.pred.R @@ -100,15 +100,24 @@ predict.Rcpp_ENSEMBLE <- function(object, newdata, ...){ # This is default: Predict g^{-1}(preds) # Get link function + # Mock loss_function enum + LossFunction <- list( + MSE = 0L, + LOGLOSS = 1L, + POISSON = 2L, + GAMMANEGINV = 3L, + GAMMALOG = 4L, + NEGBINOM = 5L + ) loss_fun_type <- object$get_loss_function() link_type = "" - if(loss_fun_type %in% c("mse")){ + if(loss_fun_type %in% c(LossFunction$MSE)){ link_type = "identity" - }else if(loss_fun_type %in% c("logloss")){ + }else if(loss_fun_type %in% c(LossFunction$LOGLOSS)){ link_type = "logit" - }else if(loss_fun_type %in% c("poisson", "gamma::log", "negbinom")){ + }else if(loss_fun_type %in% c(LossFunction$POISSON, LossFunction$GAMMALOG, LossFunction$NEGBINOM)){ link_type = "log" - }else if(loss_fun_type %in% c("gamma::neginv")){ + }else if(loss_fun_type %in% c(LossFunction$GAMMANEGINV)){ link_type = "neginv" }else{ # if no match diff --git a/R-package/R/gbt.train.R b/R-package/R/gbt.train.R index 824083b..c3926a4 100644 --- a/R-package/R/gbt.train.R +++ b/R-package/R/gbt.train.R @@ -259,6 +259,26 @@ gbt.train <- function(y, x, learning_rate = 0.01, # mod$train(y,x, verbose, gsub_compare) # # }else + + # Mock loss_function enum + LossFunction <- list( + MSE = 0L, + LOGLOSS = 1L, + POISSON = 2L, + GAMMANEGINV = 3L, + GAMMALOG = 4L, + NEGBINOM = 5L + ) + loss_function_enum = switch( + loss_function, + "mse" = LossFunction$MSE, + "logloss" = LossFunction$LOGLOSS, + "poisson" = LossFunction$POISSON, + "gamma::neginv" = LossFunction$GAMMANEGINV, + "gamma::log" = LossFunction$GAMMALOG, + "negbinom" = LossFunction$NEGBINOM + ) + if(loss_function %in% c("count::auto")){ mod <- new(GBT_COUNT_AUTO) mod$set_param(param) @@ -267,7 +287,7 @@ gbt.train <- function(y, x, learning_rate = 0.01, }else{ # create agtboost ensemble object mod <- new(ENSEMBLE) - mod$set_param(nrounds, learning_rate, extra_param, loss_function) + mod$set_param(nrounds, learning_rate, extra_param, loss_function_enum) # train ensemble if(is.null(previous_pred)){ diff --git a/R-package/inst/include/agtboost.hpp b/R-package/inst/include/agtboost.hpp index 18d4597..53c64a7 100644 --- a/R-package/inst/include/agtboost.hpp +++ b/R-package/inst/include/agtboost.hpp @@ -20,7 +20,7 @@ #include "tree.hpp" #include "ensemble.hpp" #include "optimization.hpp" -#include "loss_functions.hpp" +//#include "loss_functions.hpp" #include "gbt_count_auto.hpp" #include "initial_prediction.hpp" diff --git a/R-package/inst/include/ensemble.hpp b/R-package/inst/include/ensemble.hpp index 59d5794..5d2c757 100644 --- a/R-package/inst/include/ensemble.hpp +++ b/R-package/inst/include/ensemble.hpp @@ -5,6 +5,7 @@ #include "tree.hpp" +#include "loss_functions.hpp" // -- TRY WITHOUT EXPORT //' @export ENSEMBLE @@ -16,7 +17,7 @@ class ENSEMBLE double learning_rate; double initial_score; double extra_param; // Needed for certain distributions s.a. negative binomial, typically a dispersion param - std::string loss_function; + LossFunction loss_function; GBTREE* first_tree; //Rcpp::List param; @@ -26,7 +27,7 @@ class ENSEMBLE ENSEMBLE(double learning_rate_); // Getters and setters - void set_param(int nrounds_, double learning_rate_, double extra_param_, std::string loss_function_); + void set_param(int nrounds_, double learning_rate_, double extra_param_, LossFunction loss_function_); int get_nrounds(); @@ -34,7 +35,7 @@ class ENSEMBLE double get_extra_param(); - std::string get_loss_function(); + LossFunction get_loss_function(); // Loss-related functions double loss(Tvec &y, Tvec &pred, Tvec &w); @@ -47,7 +48,7 @@ class ENSEMBLE double inverse_link_function(double pred); - double initial_prediction(Tvec &y, std::string loss_function, Tvec &w); + double initial_prediction(Tvec &y, Tvec &w); // Training and prediction void train( diff --git a/R-package/inst/include/loss_functions.hpp b/R-package/inst/include/loss_functions.hpp index fba44c2..6c7f5ca 100644 --- a/R-package/inst/include/loss_functions.hpp +++ b/R-package/inst/include/loss_functions.hpp @@ -5,19 +5,40 @@ #include "external_rcpp.hpp" + +// Define enum class for loss functions +enum LossFunction +{ + MSE, // Gaussian distribution with identity-link (mean squared error) + LOGLOSS, // Bernoulli distribution with logit-link + POISSON, // Poisson distribution with log-link + GAMMANEGINV, // Gamma distribution with negative-inverse link + GAMMALOG, // Gamma dsitribution with log-link + NEGBINOM // Negative binomial distribution with log-link +}; +// Define stream operators for enum class +// https://stackoverflow.com/questions/21691354/enum-serialization-c +std::istream& operator >> (std::istream& in, LossFunction& loss_function) +{ + unsigned u = 0; + in >> u; + //TODO: check that u is a valid LossFunction value + loss_function = static_cast(u); + return in; +} + +std::ostream& operator << (std::ostream& out, LossFunction loss_function) +{ + //TODO: check that loss_function is a valid LossFunction value + unsigned u = loss_function; + out << u; + return out; +} + + // ----------- LOSS -------------- namespace loss_functions { - enum LossFunction - { - MSE, // Gaussian distribution with identity-link (mean squared error) - LOGLOSS, // Bernoulli distribution with logit-link - POISSON, // Poisson distribution with log-link - GAMMANEGINV, // Gamma distribution with negative-inverse link - GAMMALOG, // Gamma dsitribution with log-link - NEGBINOM // Negative binomial distribution with log-link - }; - double link_function(double pred_observed, LossFunction loss_function){ // Returns g(mu) @@ -26,16 +47,22 @@ namespace loss_functions { { case MSE: pred_transformed = pred_observed; + break; case LOGLOSS: pred_transformed = log(pred_observed) - log(1 - pred_observed); + break; case POISSON: pred_transformed = log(pred_observed); + break; case GAMMANEGINV: pred_transformed = - 1.0 / pred_observed; + break; case GAMMALOG: pred_transformed = log(pred_observed); + break; case NEGBINOM: pred_transformed = log(pred_observed); + break; } return pred_transformed; } @@ -48,16 +75,22 @@ namespace loss_functions { { case MSE: pred_observed = pred_transformed; + break; case LOGLOSS: pred_observed = 1.0 / (1.0+exp(-pred_transformed)); + break; case POISSON: pred_observed = exp(pred_transformed); + break; case GAMMANEGINV: pred_observed = -1.0 / pred_transformed; + break; case GAMMALOG: pred_observed = exp(pred_transformed); + break; case NEGBINOM: pred_observed = exp(pred_transformed); + break; } return pred_observed; } @@ -79,29 +112,35 @@ namespace loss_functions { for(int i=0; inrounds = 5000; this->learning_rate=0.01; this->extra_param = 0.0; - this->loss_function = "mse"; + this->loss_function = MSE; } ENSEMBLE::ENSEMBLE(double learning_rate_){ @@ -21,10 +25,10 @@ ENSEMBLE::ENSEMBLE(double learning_rate_){ this->nrounds = 5000; this->learning_rate=learning_rate_; this->extra_param = 0.0; - this->loss_function = "mse"; + this->loss_function = MSE; } -void ENSEMBLE::set_param(int nrounds_, double learning_rate_, double extra_param_, std::string loss_function_) +void ENSEMBLE::set_param(int nrounds_, double learning_rate_, double extra_param_, LossFunction loss_function_) { this->nrounds = nrounds_; this->learning_rate = learning_rate_; @@ -44,7 +48,7 @@ double ENSEMBLE::get_extra_param(){ return this->extra_param; } -std::string ENSEMBLE::get_loss_function(){ +LossFunction ENSEMBLE::get_loss_function(){ return this->loss_function; } @@ -103,28 +107,31 @@ void ENSEMBLE::load_model(std::string filepath) f.close(); } -double ENSEMBLE::initial_prediction(Tvec &y, std::string loss_function, Tvec &w){ +double ENSEMBLE::initial_prediction(Tvec &y, Tvec &w){ double pred=0; double pred_g_transform = y.sum()/w.sum(); // should be optim given weights... - - if(loss_function=="mse"){ + switch(loss_function) + { + case MSE: pred = pred_g_transform; - }else if(loss_function=="logloss"){ - //double pred_g_transform = (y*w).sum()/n; // naive probability + break; + case LOGLOSS: pred = log(pred_g_transform) - log(1 - pred_g_transform); - }else if(loss_function=="poisson"){ - //double pred_g_transform = (y*w).sum()/n; // naive intensity + break; + case POISSON: pred = log(pred_g_transform); - }else if(loss_function=="gamma::neginv"){ - //double pred_g_transform = (y*w).sum()/n; + break; + case GAMMANEGINV: pred = - 1.0 / pred_g_transform; - }else if(loss_function=="gamma::log"){ + break; + case GAMMALOG: pred = log(pred_g_transform); - }else if(loss_function=="negbinom"){ + break; + case NEGBINOM: pred = log(pred_g_transform); + break; } - return pred; } @@ -535,7 +542,7 @@ Tvec ENSEMBLE::convergence(Tvec &y, Tmat &X){ w.setOnes(); // After each update (tree), compute loss - loss_val[0] = loss_functions::loss(y, pred, this->loss_function, w, extra_param); + loss_val[0] = loss_functions::loss(y, pred, loss_function, w, extra_param); GBTREE* current = this->first_tree; for(int k=1; k<(K+1); k++) @@ -544,7 +551,7 @@ Tvec ENSEMBLE::convergence(Tvec &y, Tmat &X){ pred = pred + (this->learning_rate) * (current->predict_data(X)); // Compute loss - loss_val[k] = loss_functions::loss(y, pred, this->loss_function, w, extra_param); + loss_val[k] = loss_functions::loss(y, pred, loss_function, w, extra_param); // Update to next tree current = current->next_tree; @@ -644,6 +651,7 @@ double GBT_COUNT_AUTO::get_overdispersion(){ return this->count_mod->get_extra_param(); } +/* std::string GBT_COUNT_AUTO::get_model_name(){ std::string count_loss = this->count_mod->get_loss_function(); if(count_loss == "poisson"){ @@ -654,6 +662,7 @@ std::string GBT_COUNT_AUTO::get_model_name(){ return "unknown"; } } + */ void GBT_COUNT_AUTO::train(Tvec &y, Tmat &X, int verbose, bool greedy_complexities) { @@ -676,7 +685,7 @@ void GBT_COUNT_AUTO::train(Tvec &y, Tmat &X, int verbose, bool g // --- 1.0 Poisson --- ENSEMBLE* mod_pois = new ENSEMBLE; - mod_pois->set_param(param["nrounds"], param["learning_rate"], param["extra_param"], "poisson"); + mod_pois->set_param(param["nrounds"], param["learning_rate"], param["extra_param"], POISSON); /* mod_pois->set_param( Rcpp::List::create( @@ -702,7 +711,7 @@ void GBT_COUNT_AUTO::train(Tvec &y, Tmat &X, int verbose, bool g { // --- 3.1 Train negbinom ---- ENSEMBLE* mod_nbinom = new ENSEMBLE; - mod_nbinom->set_param(param["nrounds"], param["learning_rate"], dispersion, "negbinom"); + mod_nbinom->set_param(param["nrounds"], param["learning_rate"], dispersion, NEGBINOM); /* mod_nbinom->set_param( Rcpp::List::create( @@ -804,6 +813,6 @@ RCPP_MODULE(aGTBModule) { .method("train", &GBT_COUNT_AUTO::train) .method("predict", &GBT_COUNT_AUTO::predict) .method("get_overdispersion", &GBT_COUNT_AUTO::get_overdispersion) - .method("get_model_name", &GBT_COUNT_AUTO::get_model_name) + //.method("get_model_name", &GBT_COUNT_AUTO::get_model_name) ; }