Repository: xcsf-dev/xcsf (language: python)

Issue:
Bug in Python `fit()` when `shuffle=False`
Due to the way the Python wrapper splits the data supplied to `fit()` into "epochs",  if `shuffle=False` it will repeatedly iterate the first `perf_trials`  samples instead of iterating the whole dataset:

The function needs to pass the starting index within the loop so that subsequent samples can continue to be drawn from the sequence.

https://github.com/xcsf-dev/xcsf/blob/37aa409aaf9898dc90c8d40917cf47a8cdf58580/xcsf/pybind_wrapper.cpp#L514-L531

https://github.com/xcsf-dev/xcsf/blob/37aa409aaf9898dc90c8d40917cf47a8cdf58580/xcsf/xcs_supervised.c#L97-L128

https://github.com/xcsf-dev/xcsf/blob/37aa409aaf9898dc90c8d40917cf47a8cdf58580/xcsf/xcs_supervised.c#L40-L48




Discussion / hints:

Fix patch:
diff --git a/.codecov.yml b/.codecov.yml
index 793f3191b..a3259c928 100644
--- a/.codecov.yml
+++ b/.codecov.yml
@@ -2,3 +2,4 @@
 
 ignore:
   - "test/*"
+  - "xcsf/main.c"
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 970d112e6..3eaefcdf7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,7 @@
 Changes:
 *   Improve parameter checks ([#112](https://github.com/xcsf-dev/xcsf/pull/112), [#114](https://github.com/xcsf-dev/xcsf/pull/114))
 *   Improve hyperrectangle CSR semantics ([#117](https://github.com/xcsf-dev/xcsf/pull/117))
+*   Fix a sampling bug in Python `fit()` when `shuffle=False` ([#122](https://github.com/xcsf-dev/xcsf/pull/122))
 
 ## Version 1.4.3 (Nov 27, 2023)
 
diff --git a/xcsf/main.c b/xcsf/main.c
index 0dd179868..b44a6566a 100644
--- a/xcsf/main.c
+++ b/xcsf/main.c
@@ -17,7 +17,7 @@
  * @file main.c
  * @author Richard Preen <rpreen@gmail.com>
  * @copyright The Authors.
- * @date 2015--2021.
+ * @date 2015--2024.
  * @brief Main function for stand-alone binary execution.
  */
 
@@ -54,7 +54,7 @@ main(int argc, char **argv)
     param_print(xcsf); // print parameters used
     if (strcmp(argv[1], "csv") == 0) { // supervised regression - csv file
         const struct EnvCSV *env = xcsf->env;
-        xcs_supervised_fit(xcsf, env->train_data, env->test_data, true,
+        xcs_supervised_fit(xcsf, env->train_data, env->test_data, true, 0,
                            xcsf->MAX_TRIALS);
     } else { // reinforcement learning - maze or mux
         xcs_rl_exp(xcsf);
diff --git a/xcsf/pybind_wrapper.cpp b/xcsf/pybind_wrapper.cpp
index a48f6ce9d..643af1621 100644
--- a/xcsf/pybind_wrapper.cpp
+++ b/xcsf/pybind_wrapper.cpp
@@ -18,7 +18,7 @@
  * @author Richard Preen <rpreen@gmail.com>
  * @author David Pätzel
  * @copyright The Authors.
- * @date 2020--2023.
+ * @date 2020--2024.
  * @brief Python library wrapper functions.
  */
 
@@ -515,13 +515,14 @@ class XCS
         const int n = ceil(xcs.MAX_TRIALS / (double) xcs.PERF_TRIALS);
         const int n_trials = std::min(xcs.MAX_TRIALS, xcs.PERF_TRIALS);
         for (int i = 0; i < n; ++i) {
-            const double train =
-                xcs_supervised_fit(&xcs, train_data, NULL, shuffle, n_trials);
-            double val = 0;
+            const int start = i * n_trials;
+            const double train_error = xcs_supervised_fit(
+                &xcs, train_data, NULL, shuffle, start, n_trials);
+            double val_error = 0;
             if (val_data != NULL) {
-                val = xcs_supervised_score(&xcs, val_data, xcs.cover);
+                val_error = xcs_supervised_score(&xcs, val_data, xcs.cover);
             }
-            update_metrics(train, val, n_trials);
+            update_metrics(train_error, val_error, n_trials);
             if (verbose) {
                 print_status();
             }
diff --git a/xcsf/xcs_supervised.c b/xcsf/xcs_supervised.c
index 2774d8064..0e5e78412 100644
--- a/xcsf/xcs_supervised.c
+++ b/xcsf/xcs_supervised.c
@@ -17,7 +17,7 @@
  * @file xcs_supervised.c
  * @author Richard Preen <rpreen@gmail.com>
  * @copyright The Authors.
- * @date 2015--2023.
+ * @date 2015--2024.
  * @brief Supervised regression learning functions.
  */
 
@@ -91,20 +91,21 @@ xcs_supervised_trial(struct XCSF *xcsf, const double *x, const double *y,
  * @param [in] train_data The input data to use for training.
  * @param [in] test_data The input data to use for testing.
  * @param [in] shuffle Whether to randomise the instances during training.
+ * @param [in] start Index to begin sampling.
  * @param [in] trials Number of trials to execute.
  * @return The average XCSF training error using the loss function.
  */
 double
 xcs_supervised_fit(struct XCSF *xcsf, const struct Input *train_data,
                    const struct Input *test_data, const bool shuffle,
-                   const int trials)
+                   const int start, const int trials)
 {
     double err = 0; // training error: total over all trials
     double werr = 0; // training error: windowed total
     double wterr = 0; // testing error: windowed total
     for (int cnt = 0; cnt < trials; ++cnt) {
         // training sample
-        int row = xcs_supervised_sample(train_data, cnt, shuffle);
+        int row = xcs_supervised_sample(train_data, cnt + start, shuffle);
         const double *x = &train_data->x[row * train_data->x_dim];
         const double *y = &train_data->y[row * train_data->y_dim];
         param_set_explore(xcsf, true);
@@ -115,7 +116,7 @@ xcs_supervised_fit(struct XCSF *xcsf, const struct Input *train_data,
         xcsf->error += (error - xcsf->error) * xcsf->BETA;
         // test sample
         if (test_data != NULL) {
-            row = xcs_supervised_sample(test_data, cnt, shuffle);
+            row = xcs_supervised_sample(test_data, cnt + start, shuffle);
             x = &test_data->x[row * test_data->x_dim];
             y = &test_data->y[row * test_data->y_dim];
             param_set_explore(xcsf, false);
diff --git a/xcsf/xcs_supervised.h b/xcsf/xcs_supervised.h
index a2e596244..6c3ed1088 100644
--- a/xcsf/xcs_supervised.h
+++ b/xcsf/xcs_supervised.h
@@ -17,7 +17,7 @@
  * @file xcs_supervised.h
  * @author Richard Preen <rpreen@gmail.com>
  * @copyright The Authors.
- * @date 2015--2023.
+ * @date 2015--2024.
  * @brief Supervised regression learning functions.
  */
 
@@ -28,7 +28,7 @@
 double
 xcs_supervised_fit(struct XCSF *xcsf, const struct Input *train_data,
                    const struct Input *test_data, const bool shuffle,
-                   const int trials);
+                   const int start, const int trials);
 
 double
 xcs_supervised_score(struct XCSF *xcsf, const struct Input *data,


Test patch:
diff --git a/test/xcs_supervised_test.cpp b/test/xcs_supervised_test.cpp
index 084a12949..e9c0561d6 100644
--- a/test/xcs_supervised_test.cpp
+++ b/test/xcs_supervised_test.cpp
@@ -17,7 +17,7 @@
  * @file xcs_supervised_test.cpp
  * @author Richard Preen <rpreen@gmail.com>
  * @copyright The Authors.
- * @date 2023.
+ * @date 2023--2024.
  * @brief High-level supervised learning function tests.
  */
 
@@ -66,7 +66,7 @@ TEST_CASE("SUPERVISED")
     train_data.y = y;
     double *cover = (double *) calloc(y_dim, sizeof(double));
     // fit() with only train data
-    xcs_supervised_fit(&xcsf, &train_data, NULL, true, 100);
+    xcs_supervised_fit(&xcsf, &train_data, NULL, true, 0, 100);
     // score()
     double score = xcs_supervised_score(&xcsf, &train_data, cover);
     CHECK_EQ(doctest::Approx(score), 0.11742);
@@ -82,7 +82,7 @@ TEST_CASE("SUPERVISED")
         CHECK_EQ(doctest::Approx(output[i]), expected[i]);
     }
     // fit() with train and test data
-    xcs_supervised_fit(&xcsf, &train_data, &train_data, true, 100);
+    xcs_supervised_fit(&xcsf, &train_data, &train_data, true, 0, 100);
     score = xcs_supervised_score(&xcsf, &train_data, cover);
     CHECK_EQ(doctest::Approx(score), 0.0359);