l-step lookahead evaluated using synthetic datasets

changed psl scoring to brier score

experiments/performance.ipynb

@@ -37,23 +37,29 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "0.9604\n",
-      "CPU times: user 157 ms, sys: 108 ms, total: 266 ms\n",
-      "Wall time: 18 s\n"
+      "0.0274\n",
+      "CPU times: user 33.9 ms, sys: 87 ms, total: 121 ms\n",
+      "Wall time: 3.03 s\n"
      ]
     }
    ],
    "source": [
     "%%time\n",
     "clf = ProbabilisticScoringList([-1, 1, 2])\n",
-    "print(f\"{cross_val_score(clf, X, y, fit_params=dict(l=1), cv=ShuffleSplit(100, test_size=.2, random_state=42), n_jobs=-1).mean():.4f}\")"
+    "print(f\"{cross_val_score(clf, X, y, fit_params=dict(l=1), cv=ShuffleSplit(5, test_size=.2, random_state=42), n_jobs=-1).mean():.4f}\")"
    ]
   },
   {
    "cell_type": "markdown",
    "id": "eb1d191d-38b6-4f1c-8740-7101c9bd192c",
    "metadata": {},
    "source": [
+    "non-neg 0.2349\n",
+    "l1 0.0265\n",
+    "l2 0.0276 20\n",
+    "l2 0.0315 5\n",
+    "l3 0.0320 5\n",
+    "\n",
     "### 0.2.0\n",
     "- l=2 14min 11s, 0.9599\n",
     "- l=1 30s, 0.9604\n",

pyproject.toml

@@ -14,7 +14,8 @@ python = ">=3.9,<3.13"
 scikit-learn = "^1.3.0"
 numpy = "^1.25.2"
 scipy = "^1.11.1"
-joblib = "^1.3.1"
+joblib = "^1.3.2"
+pandas = "^2.0.3"
 
 sphinx = { version = "^7.1", optional = true }
 sphinx_rtd_theme = { version = "^1.2", optional = true  }

skpsl/data/dataset.py

@@ -0,0 +1,52 @@
+import numpy as np
+from itertools import permutations
+
+from sklearn.model_selection import ShuffleSplit
+
+from skpsl.probabilistic_scoring_list import _ClassifierAtK
+from skpsl import ProbabilisticScoringList
+
+
+def lookahead_example():
+    """
+               scores→ [2,1]   [1,2]
+    (2, 5, 1, 4, 3)   0.7839  0.7783 l2_psl.scores=[1, 2] l2_psl.stage_clfs[-1].score(X_)=0.7783  score_l1=0.1944 score_l2=0.1926
+    """
+    psl = ProbabilisticScoringList(s)
+    psl.fit(X_, y_)
+    if psl.features == [0, 1] and psl.scores == [2, 1]:
+        # the psl ordered the features really in the way that feature 0 is the better one
+        # now lets test if we can improve the performance by inverting the scores
+        score, invscore = [_ClassifierAtK(features=f, scores=s_).fit(X_, y_, ).score(X_) for s_ in permutations(s)]
+        if score > invscore:
+            score_l1 = psl.score(X_, y_)
+            l2_psl = ProbabilisticScoringList(s).fit(X_, y_, l=2)
+            score_l2 = l2_psl.score(X_, y_)
+            if score_l2 < score_l1:
+                print(
+                    f"{w}   {score:.4f}  {invscore:.4f} {l2_psl.scores=} {l2_psl.stage_clfs[-1].score(X_)=}  {score_l1=} {score_l2=}")
+                print(np.corrcoef(np.hstack([X_, y_.reshape(-1, 1)].T)))
+
+
+if __name__ == '__main__':
+    X = np.array([[1, 0], [0, 1], [1, 1], [1, 0], [0, 1], [1, 1]])
+    y = np.array([0, 0, 0, 1, 1, 1])
+    f = [0, 1]
+    # s = [2, 1]
+    s = [2, 1, -1]
+
+    for w in permutations(range(1, X.shape[0] + 1)):
+        X_ = np.repeat(X, np.array(w), axis=0)
+        y_ = np.repeat(y, np.array(w), axis=0)
+
+        rs = ShuffleSplit(n_splits=5, test_size=0.2, random_state=0)
+        for (train_index, test_index) in rs.split(X):
+            l1_psl = ProbabilisticScoringList(s).fit(X_[train_index], y_[train_index])
+            l1_out = l1_psl.score(X_[test_index], y_[test_index])
+            l1_in = l1_psl.score(X_[train_index], y_[train_index])
+
+            l2_psl = ProbabilisticScoringList(s).fit(X_[train_index], y_[train_index], l=2)
+            l2_out = l2_psl.score(X_[test_index], y_[test_index])
+            l2_in = l2_psl.score(X_[train_index], y_[train_index])
+            if l1_in > l2_in:
+                print(f"{l1_in=} {l2_in=} {l1_out=} {l2_out=}")

skpsl/probabilistic_scoring_list.py

@@ -7,6 +7,7 @@
 from sklearn.base import BaseEstimator, ClassifierMixin
 from sklearn.exceptions import NotFittedError
 from sklearn.isotonic import IsotonicRegression
+from sklearn.metrics import brier_score_loss
 
 
 class _ClassifierAtK(BaseEstimator, ClassifierMixin):
@@ -97,7 +98,6 @@ def __init__(self, score_set, entropy_threshold=-1):
         self.total_scores_at_k = []
         self.probabilities_at_k = []
         self.stage_clfs = []
-        self.entropy_at_k = []
         self._stage_clf = _ClassifierAtK
 
     def fit(self, X, y, l=1, n_jobs=1, predef_features=None, predef_scores=None) -> "ProbabilisticScoringList":
@@ -179,6 +179,16 @@ def predict_proba(self, X, k=-1):
 
         return self.stage_clfs[k].predict_proba(X)
 
+    def score(self, X, y, sample_weight=None):
+        """
+        Calculates the Brier score of the model
+        :param X:
+        :param y:
+        :param sample_weight:
+        :return:
+        """
+        return brier_score_loss(y, self.predict_proba(X)[:, 1])
+
     @staticmethod
     def _optimize(features, feature_extension, scores, score_extension, clfcls, X, y):
         clf = clfcls(features=features + feature_extension, scores=scores + score_extension).fit(X, y)