Repository: pykale/pykale (language: python)

Issue:
Dev shuo 
MPCA and CMR


Discussion / hints:

Fix patch:
diff --git a/docs/source/kale.evaluate.rst b/docs/source/kale.evaluate.rst
index 271e71ee6..11ce345e9 100644
--- a/docs/source/kale.evaluate.rst
+++ b/docs/source/kale.evaluate.rst
@@ -6,6 +6,14 @@ Evaluate
 Submodules
 ----------
 
+kale.evaluate.cross\_validation module
+----------------------------
+
+.. automodule:: kale.evaluate.cross_validation
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 kale.evaluate.metrics module
 ----------------------------
 
diff --git a/examples/multisite_neuroimg_adapt/README.md b/examples/multisite_neuroimg_adapt/README.md
index 591a2f969..a693a036a 100644
--- a/examples/multisite_neuroimg_adapt/README.md
+++ b/examples/multisite_neuroimg_adapt/README.md
@@ -33,14 +33,10 @@ autism detection.
 
 ### References
 
-[1] Cameron Craddock, Yassine Benhajali, Carlton Chu, Francois Chouinard, Alan Evans, András Jakab, Budhachandra Singh
-Khundrakpam, John David Lewis, Qingyang Li, Michael Milham, Chaogan Yan, Pierre Bellec (2013). The Neuro Bureau
-Preprocessing Initiative: open sharing of preprocessed neuroimaging data and derivatives. In *Neuroinformatics 2013*,
-Stockholm, Sweden.
+[1] Craddock C., Benhajali Y., Chu C., Chouinard F., Evans A., Jakab A., Khundrakpam BS., Lewis JD., Li Q., Milham M., Yan C. and Bellec P. (2013). [The Neuro Bureau Preprocessing Initiative: Open Sharing of Preprocessed Neuroimaging Data and Derivatives](https://doi.org/10.3389/conf.fninf.2013.09.00041). Frontiers in Neuroinformatics, 7.
 
-[2] Abraham, A., Pedregosa, F., Eickenberg, M., Gervais, P., Mueller, A., Kossaifi, J., ... & Varoquaux, G. (2014).
-Machine learning for neuroimaging with scikit-learn. *Frontiers in neuroinformatics*, 14.
+[2] Abraham A., Pedregosa F., Eickenberg M., Gervais P., Mueller A., Kossaifi J., Gramfort A., Thirion B. and Varoquaux G. (2014). [Machine Learning for Neuroimaging with scikit-learn](https://doi.org/10.3389/fninf.2014.00014). Frontiers in Neuroinformatics, 8.
 
-[3] Zhou, S., Li, W., Cox, C.R., & Lu, H. (2020). [Side Information Dependence as a Regularizer for Analyzing Human Brain Conditions across Cognitive Experiments](https://ojs.aaai.org//index.php/AAAI/article/view/6179). in *AAAI 2020*, New York, USA.
+[3] Zhou S., Li W., Cox C. and Lu H. (2020). [Side Information Dependence as a Regularizer for Analyzing Human Brain Conditions across Cognitive Experiments](https://doi.org/10.1609/aaai.v34i04.6179). Proceedings of the AAAI Conference on Artificial Intelligence, 34(04), 6957-6964.
 
-[4] Zhou, S. (2022). [Interpretable Domain-Aware Learning for Neuroimage Classification](https://etheses.whiterose.ac.uk/31044/1/PhD_thesis_ShuoZhou_170272834.pdf) (Doctoral dissertation, University of Sheffield).
+[4] Zhou S. (2022). [Interpretable Domain-Aware Learning for Neuroimage Classification](https://etheses.whiterose.ac.uk/31044/1/PhD_thesis_ShuoZhou_170272834.pdf) (Doctoral Dissertation, University of Sheffield).
diff --git a/examples/multisite_neuroimg_adapt/config.py b/examples/multisite_neuroimg_adapt/config.py
index 0fb401b2e..6bcb2a6a7 100644
--- a/examples/multisite_neuroimg_adapt/config.py
+++ b/examples/multisite_neuroimg_adapt/config.py
@@ -20,6 +20,11 @@
 # options: {rois_aal, rois_cc200, rois_cc400, rois_dosenbach160, rois_ez, rois_ho, rois_tt}
 _C.DATASET.SITE_IDS = None  # list of site ids to use, if None, use all sites
 _C.DATASET.TARGET = "NYU"  # target site ids, e.g. "UM_1", "UCLA_1", "USM"
+# ---------------------------------------------------------
+# Solver
+# ---------------------------------------------------------
+_C.SOLVER = CfgNode()
+_C.SOLVER.SEED = 2023
 # ---------------------------------------------------------------------------- #
 # Machine learning pipeline
 # ---------------------------------------------------------------------------- #
diff --git a/examples/multisite_neuroimg_adapt/main.py b/examples/multisite_neuroimg_adapt/main.py
new file mode 100644
index 000000000..e1635a45f
--- /dev/null
+++ b/examples/multisite_neuroimg_adapt/main.py
@@ -0,0 +1,97 @@
+"""
+Autism Detection: Domain Adaptation for Multi-Site Neuroimaging Data Analysis
+
+Reference:
+[1] Craddock C., Benhajali Y., Chu C., Chouinard F., Evans A., Jakab A., Khundrakpam BS., Lewis JD., Li Q., Milham M., Yan C. and Bellec P. (2013). The Neuro Bureau Preprocessing Initiative: Open Sharing of Preprocessed Neuroimaging Data and Derivatives. Frontiers in Neuroinformatics, 7. https://doi.org/10.3389/conf.fninf.2013.09.00041
+
+[2] Abraham A., Pedregosa F., Eickenberg M., Gervais P., Mueller A., Kossaifi J., Gramfort A., Thirion B. and Varoquaux G. (2014). Machine Learning for Neuroimaging with scikit-learn. Frontiers in Neuroinformatics, 8. https://doi.org/10.3389/fninf.2014.00014
+
+[3] Zhou S., Li W., Cox C. and Lu H. (2020). Side Information Dependence as a Regularizer for Analyzing Human Brain Conditions across Cognitive Experiments. Proceedings of the AAAI Conference on Artificial Intelligence, 34(04), 6957-6964. https://doi.org/10.1609/aaai.v34i04.6179
+
+[4] Zhou S. (2022). Interpretable Domain-Aware Learning for Neuroimage Classification (Doctoral Dissertation, University of Sheffield). https://etheses.whiterose.ac.uk/31044/1/PhD_thesis_ShuoZhou_170272834.pdf
+"""
+import argparse
+import os
+
+import numpy as np
+import pandas as pd
+from config import get_cfg_defaults
+from nilearn.connectome import ConnectivityMeasure
+from nilearn.datasets import fetch_abide_pcp
+from sklearn.linear_model import RidgeClassifier
+
+import kale.utils.seed as seed
+from kale.evaluate import cross_validation
+from kale.pipeline.multi_domain_adapter import CoIRLS
+
+
+def arg_parse():
+    parser = argparse.ArgumentParser(
+        description="Autism Detection: Domain Adaptation for Multi-Site Neuroimaging Data Analysis"
+    )
+    parser.add_argument("--cfg", required=True, help="path to config file", type=str)
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = arg_parse()
+
+    # ---- Set up configs ----
+    cfg = get_cfg_defaults()
+    cfg.merge_from_file(args.cfg)
+    cfg.freeze()
+    seed.set_seed(cfg.SOLVER.SEED)
+
+    # ---- Fetch ABIDE fMRI timeseries ----
+    fetch_abide_pcp(
+        data_dir=cfg.DATASET.ROOT,
+        pipeline=cfg.DATASET.PIPELINE,
+        band_pass_filtering=True,
+        global_signal_regression=False,
+        derivatives=cfg.DATASET.ATLAS,
+        quality_checked=False,
+        SITE_ID=cfg.DATASET.SITE_IDS,
+        verbose=1,
+    )
+
+    # ---- Read Phenotypic data ----
+    pheno_file = os.path.join(cfg.DATASET.ROOT, "ABIDE_pcp/Phenotypic_V1_0b_preprocessed1.csv")
+    pheno_info = pd.read_csv(pheno_file, index_col=0)
+
+    # ---- Read timeseries from files ----
+    data_dir = os.path.join(cfg.DATASET.ROOT, "ABIDE_pcp/%s/filt_noglobal" % cfg.DATASET.PIPELINE)
+    use_idx = []
+    time_series = []
+    for i in pheno_info.index:
+        data_file_name = "%s_%s.1D" % (pheno_info.loc[i, "FILE_ID"], cfg.DATASET.ATLAS)
+        data_path = os.path.join(data_dir, data_file_name)
+        if os.path.exists(data_path):
+            time_series.append(np.loadtxt(data_path, skiprows=0))
+            use_idx.append(i)
+
+    # ---- Use "DX_GROUP" (autism vs control) as labels, and "SITE_ID" as covariates ----
+    pheno = pheno_info.loc[use_idx, ["SITE_ID", "DX_GROUP"]].reset_index(drop=True)
+
+    # ---- Extracting Brain Networks Features ----
+    correlation_measure = ConnectivityMeasure(kind="correlation", vectorize=True)
+    brain_networks = correlation_measure.fit_transform(time_series)
+
+    # ---- Machine Learning for Multi-site Data ----
+    print("Baseline")
+    estimator = RidgeClassifier()
+    results = cross_validation.leave_one_group_out(
+        brain_networks, pheno["DX_GROUP"].values, pheno["SITE_ID"].values, estimator
+    )
+    print(pd.DataFrame.from_dict(results))
+
+    print("Domain Adaptation")
+    estimator = CoIRLS(kernel=cfg.MODEL.KERNEL, lambda_=cfg.MODEL.LAMBDA_, alpha=cfg.MODEL.ALPHA)
+    results = cross_validation.leave_one_group_out(
+        brain_networks, pheno["DX_GROUP"].values, pheno["SITE_ID"].values, estimator, use_domain_adaptation=True
+    )
+    print(pd.DataFrame.from_dict(results))
+
+
+if __name__ == "__main__":
+    main()
diff --git a/examples/multisite_neuroimg_adapt/tutorial.ipynb b/examples/multisite_neuroimg_adapt/tutorial.ipynb
index c902ac9a4..74e9260f6 100644
--- a/examples/multisite_neuroimg_adapt/tutorial.ipynb
+++ b/examples/multisite_neuroimg_adapt/tutorial.ipynb
@@ -20,18 +20,18 @@
         "    - [Data loading](#Data-Preparation)\n",
         "    - [Construct brain networks](#Extracting-Brain-Networks-Features)\n",
         "- Machine learning pipeline:\n",
-        "    - [Baseline: Ridge classifier](#Baseline)\n",
+        "    - [Baseline: Ridge classifier](#Baseline-Model)\n",
         "    - [Domain adaptation](#Domain-Adaptation)\n",
         "\n",
         "**Reference:**\n",
         "\n",
-        "[1] Cameron Craddock, Yassine Benhajali, Carlton Chu, Francois Chouinard, Alan Evans, Andr\u00e1s Jakab, Budhachandra Singh Khundrakpam, John David Lewis, Qingyang Li, Michael Milham, Chaogan Yan, Pierre Bellec (2013). The Neuro Bureau Preprocessing Initiative: open sharing of preprocessed neuroimaging data and derivatives. In *Neuroinformatics 2013*, Stockholm, Sweden.\n",
+        "[1] Craddock C., Benhajali Y., Chu C., Chouinard F., Evans A., Jakab A., Khundrakpam BS., Lewis JD., Li Q., Milham M., Yan C. and Bellec P. (2013). [The Neuro Bureau Preprocessing Initiative: Open Sharing of Preprocessed Neuroimaging Data and Derivatives](https://doi.org/10.3389/conf.fninf.2013.09.00041). Frontiers in Neuroinformatics, 7.\n",
         "\n",
-        "[2] Abraham, A., Pedregosa, F., Eickenberg, M., Gervais, P., Mueller, A., Kossaifi, J., ... & Varoquaux, G. (2014). Machine learning for neuroimaging with scikit-learn. *Frontiers in neuroinformatics*, 14.\n",
+        "[2] Abraham A., Pedregosa F., Eickenberg M., Gervais P., Mueller A., Kossaifi J., Gramfort A., Thirion B. and Varoquaux G. (2014). [Machine Learning for Neuroimaging with scikit-learn](https://doi.org/10.3389/fninf.2014.00014). Frontiers in Neuroinformatics, 8.\n",
         "\n",
-        "[3] Zhou, S., Li, W., Cox, C.R., & Lu, H. (2020). [Side Information Dependence as a Regularizer for Analyzing Human Brain Conditions across Cognitive Experiments](https://ojs.aaai.org//index.php/AAAI/article/view/6179). in *AAAI 2020*, New York, USA. \n",
+        "[3] Zhou S., Li W., Cox C. and Lu H. (2020). [Side Information Dependence as a Regularizer for Analyzing Human Brain Conditions across Cognitive Experiments](https://doi.org/10.1609/aaai.v34i04.6179). Proceedings of the AAAI Conference on Artificial Intelligence, 34(04), 6957-6964.\n",
         "\n",
-        "[4] Zhou, S. (2022). [Interpretable Domain-Aware Learning for Neuroimage Classification](https://etheses.whiterose.ac.uk/31044/1/PhD_thesis_ShuoZhou_170272834.pdf) (Doctoral dissertation, University of Sheffield)."
+        "[4] Zhou S. (2022). [Interpretable Domain-Aware Learning for Neuroimage Classification](https://etheses.whiterose.ac.uk/31044/1/PhD_thesis_ShuoZhou_170272834.pdf) (Doctoral Dissertation, University of Sheffield)."
       ],
       "cell_type": "markdown"
     },
@@ -50,13 +50,21 @@
         "    !pip uninstall --yes imgaug && pip uninstall --yes albumentations && pip install git+https://github.com/aleju/imgaug.git\n",
         "    !git clone https://github.com/pykale/pykale.git\n",
         "    %cd pykale\n",
-        "    !pip install .[image,example] \n",
+        "    !pip install .[image,example]\n",
         "    %cd examples/multisite_neuroimg_adapt\n",
         "else:\n",
         "    print('Not running on CoLab')"
       ],
       "cell_type": "code",
-      "outputs": [],
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Not running on CoLab\n"
+          ]
+        }
+      ],
       "execution_count": null
     },
     {
@@ -70,18 +78,16 @@
       "metadata": {},
       "source": [
         "import os\n",
-        "\n",
-        "%matplotlib inline\n",
-        "import matplotlib.pyplot as plt\n",
         "import numpy as np\n",
-        "from nilearn.datasets import fetch_abide_pcp\n",
         "import pandas as pd\n",
         "from config import get_cfg_defaults\n",
+        "from nilearn.connectome import ConnectivityMeasure\n",
+        "from nilearn.datasets import fetch_abide_pcp\n",
+        "from sklearn.linear_model import RidgeClassifier\n",
         "\n",
-        "import sys\n",
-        "\n",
-        "from kale.utils.download import download_file_by_url\n",
-        "from kale.interpret import visualize"
+        "import kale.utils.seed as seed\n",
+        "from kale.evaluate import cross_validation\n",
+        "from kale.pipeline.multi_domain_adapter import CoIRLS"
       ],
       "cell_type": "code",
       "outputs": [],
@@ -90,11 +96,12 @@
     {
       "metadata": {},
       "source": [
-        "cfg_path = \"configs/tutorial.yaml\" # Path to `.yaml` config file\n",
-        "\n",
+        "# Path to `.yaml` config file\n",
+        "cfg_path = \"configs/tutorial.yaml\" \n",
         "cfg = get_cfg_defaults()\n",
         "cfg.merge_from_file(cfg_path)\n",
         "cfg.freeze()\n",
+        "seed.set_seed(cfg.SOLVER.SEED)\n",
         "print(cfg)"
       ],
       "cell_type": "code",
@@ -117,10 +124,10 @@
         "pipeline = cfg.DATASET.PIPELINE  # fmri pre-processing pipeline\n",
         "atlas = cfg.DATASET.ATLAS\n",
         "site_ids = cfg.DATASET.SITE_IDS\n",
-        "abide = fetch_abide_pcp(data_dir=root_dir, pipeline=pipeline, \n",
-        "                        band_pass_filtering=True, global_signal_regression=False, \n",
+        "abide = fetch_abide_pcp(data_dir=root_dir, pipeline=pipeline,\n",
+        "                        band_pass_filtering=True, global_signal_regression=False,\n",
         "                        derivatives=atlas, quality_checked=False,\n",
-        "                        SITE_ID=site_ids, \n",
+        "                        SITE_ID=site_ids,\n",
         "                        verbose=0)"
       ],
       "cell_type": "code",
@@ -210,8 +217,6 @@
     {
       "metadata": {},
       "source": [
-        "from nilearn.connectome import ConnectivityMeasure\n",
-        "\n",
         "correlation_measure = ConnectivityMeasure(kind='correlation', vectorize=True)\n",
         "brain_networks = correlation_measure.fit_transform(time_series)"
       ],
@@ -228,50 +233,6 @@
       ],
       "cell_type": "markdown"
     },
-    {
-      "metadata": {},
-      "source": [
-        "from sklearn.metrics import accuracy_score\n",
-        "from sklearn.preprocessing import OneHotEncoder\n",
-        "import torch\n",
-        "\n",
-        "def cross_validation(x, y, covariates, estimator, domain_adaptation=False):\n",
-        "    results = {\"Target\": [], \"Num_samples\": [], \"Accuracy\": []}\n",
-        "    unique_covariates = np.unique(covariates)\n",
-        "    n_covariates = len(unique_covariates)\n",
-        "    enc = OneHotEncoder(handle_unknown=\"ignore\")\n",
-        "    covariate_mat = enc.fit_transform(covariates.reshape(-1, 1)).toarray()\n",
-        "    \n",
-        "    for tgt in unique_covariates:\n",
-        "        idx_tgt = np.where(covariates == tgt)\n",
-        "        idx_src = np.where(covariates != tgt)\n",
-        "        x_tgt = brain_networks[idx_tgt]\n",
-        "        x_src = brain_networks[idx_src]\n",
-        "        y_tgt = y[idx_tgt]\n",
-        "        y_src = y[idx_src]        \n",
-        "        \n",
-        "        if domain_adaptation:\n",
-        "            estimator.fit(np.concatenate((x_src, x_tgt)), y_src, \n",
-        "                          np.concatenate((covariate_mat[idx_src], covariate_mat[idx_tgt])))\n",
-        "        else:            \n",
-        "            estimator.fit(x_src, y_src)\n",
-        "        y_pred = estimator.predict(x_tgt)\n",
-        "        results[\"Accuracy\"].append(accuracy_score(y_tgt, y_pred))\n",
-        "        results[\"Target\"].append(tgt)\n",
-        "        results[\"Num_samples\"].append(x_tgt.shape[0])\n",
-        "    \n",
-        "    mean_acc = sum([results[\"Num_samples\"][i] * results[\"Accuracy\"][i] for i in range(n_covariates)])\n",
-        "    mean_acc /= x.shape[0]\n",
-        "    results[\"Target\"].append(\"Average\")\n",
-        "    results[\"Num_samples\"].append(x.shape[0])\n",
-        "    results[\"Accuracy\"].append(mean_acc)\n",
-        "    \n",
-        "    return pd.DataFrame(results)"
-      ],
-      "cell_type": "code",
-      "outputs": [],
-      "execution_count": null
-    },
     {
       "metadata": {},
       "source": [
@@ -282,10 +243,10 @@
     {
       "metadata": {},
       "source": [
-        "from sklearn.linear_model import RidgeClassifier\n",
-        "\n",
         "estimator = RidgeClassifier()\n",
-        "res_df = cross_validation(brain_networks, pheno[\"DX_GROUP\"].values, pheno[\"SITE_ID\"].values, estimator)"
+        "results = cross_validation.leave_one_group_out(\n",
+        "    brain_networks, pheno[\"DX_GROUP\"].values, pheno[\"SITE_ID\"].values, estimator\n",
+        ")"
       ],
       "cell_type": "code",
       "outputs": [],
@@ -294,7 +255,7 @@
     {
       "metadata": {},
       "source": [
-        "res_df"
+        "pd.DataFrame.from_dict(results)"
       ],
       "cell_type": "code",
       "outputs": [],
@@ -310,10 +271,10 @@
     {
       "metadata": {},
       "source": [
-        "from kale.pipeline.multi_domain_adapter import CoIRLS\n",
         "estimator = CoIRLS(kernel=cfg.MODEL.KERNEL, lambda_=cfg.MODEL.LAMBDA_, alpha=cfg.MODEL.ALPHA)\n",
-        "res_df = cross_validation(brain_networks, pheno[\"DX_GROUP\"].values, pheno[\"SITE_ID\"].values, \n",
-        "                          estimator, domain_adaptation=True)"
+        "results = cross_validation.leave_one_group_out(\n",
+        "  brain_networks, pheno[\"DX_GROUP\"].values, pheno[\"SITE_ID\"].values, estimator, use_domain_adaptation=True\n",
+        ")"
       ],
       "cell_type": "code",
       "outputs": [],
@@ -322,7 +283,7 @@
     {
       "metadata": {},
       "source": [
-        "res_df"
+        "pd.DataFrame.from_dict(results)"
       ],
       "cell_type": "code",
       "outputs": [],
diff --git a/kale/evaluate/cross_validation.py b/kale/evaluate/cross_validation.py
new file mode 100644
index 000000000..03bf0d365
--- /dev/null
+++ b/kale/evaluate/cross_validation.py
@@ -0,0 +1,56 @@
+"""
+Functions implementing cross-validation methods for assessing model fit.
+"""
+
+import numpy as np
+from sklearn.metrics import accuracy_score
+from sklearn.model_selection import LeaveOneGroupOut
+from sklearn.preprocessing import OneHotEncoder
+
+
+def leave_one_group_out(x, y, groups, estimator, use_domain_adaptation=False) -> dict:
+    """
+    Perform leave one group out cross validation for a given estimator.
+
+    Args:
+        x (np.ndarray or torch.tensor): Input data [n_samples, n_features].
+        y (np.ndarray or torch.tensor): Target labels [n_samples].
+        groups (np.ndarray or torch.tensor): Group labels to be left out [n_samples].
+        estimator (estimator object): Machine learning estimator to be evaluated from kale or scikit-learn.
+        use_domain_adaptation (bool): Whether to use domain adaptation, i.e., leveraging test data, during training.
+
+    Returns:
+        dict: A dictionary containing results for each target group with 3 keys.
+            - 'Target': A list of unique target groups or classes. The final entry is "Average".
+            - 'Num_samples': A list where each entry indicates the number of samples in its corresponding target group.
+                            The final entry represents the total number of samples.
+            - 'Accuracy': A list where each entry indicates the accuracy score for its corresponding target group.
+                            The final entry represents the overall mean accuracy.
+    """
+    enc = OneHotEncoder(handle_unknown="ignore")
+    group_mat = enc.fit_transform(groups.reshape(-1, 1)).toarray()
+    target, num_samples, accuracy = np.unique(groups).tolist(), [], []
+    y_pred = np.zeros(y.shape)  # Store all predicted labels to compute accuracy
+
+    for train, test in LeaveOneGroupOut().split(x, y, groups=groups):
+        x_src, x_tgt = x[train], x[test]
+        y_src, y_tgt = y[train], y[test]
+
+        if use_domain_adaptation:
+            estimator.fit(np.concatenate((x_src, x_tgt)), y_src, np.concatenate((group_mat[train], group_mat[test])))
+        else:
+            estimator.fit(x_src, y_src)
+
+        y_pred[test] = estimator.predict(x_tgt)
+        num_samples.append(x_tgt.shape[0])
+        accuracy.append(accuracy_score(y_tgt, y_pred[test]))
+
+    target.append("Average")
+    num_samples.append(x.shape[0])
+    accuracy.append(accuracy_score(y, y_pred))
+
+    return {
+        "Target": target,
+        "Num_samples": num_samples,
+        "Accuracy": accuracy,
+    }


Test patch:
diff --git a/tests/evaluate/test_cross_validation.py b/tests/evaluate/test_cross_validation.py
new file mode 100644
index 000000000..76c06e0b4
--- /dev/null
+++ b/tests/evaluate/test_cross_validation.py
@@ -0,0 +1,49 @@
+import numpy as np
+import pytest
+from sklearn.dummy import DummyClassifier
+
+from kale.evaluate import cross_validation
+from kale.pipeline.multi_domain_adapter import CoIRLS
+
+
+@pytest.fixture
+def sample_data():
+    # Sample data for testing
+    x = np.array([np.random.rand(100)] * 8)
+    y = np.array([0, 0, 0, 0, 1, 1, 1, 1])
+    groups = np.array(["A", "A", "B", "B", "A", "A", "B", "B"])
+    return x, y, groups
+
+
+# Checks leave-one-group-out results for above sample data
+def check_leave_one_group_out_results(sample_data, estimator, domain_adaptation=False):
+    x, y, groups = sample_data
+    results = cross_validation.leave_one_group_out(x, y, groups, estimator, domain_adaptation)
+
+    # Check if all keys are present in the result dictionary
+    assert "Target" in results
+    assert "Num_samples" in results
+    assert "Accuracy" in results
+
+    # Check if the length of 'Target', 'Num_samples', and 'Accuracy' lists is the same
+    assert len(results["Target"]) == len(results["Num_samples"]) == len(results["Accuracy"])
+
+    # Check if the last element of 'Target' is "Average"
+    assert results["Target"] == ["A", "B", "Average"]
+
+    # Check if the number of samples is correct for each target group
+    assert results["Num_samples"] == [4, 4, 8]
+
+    # Check if the accuracy scores are within the expected range
+    for accuracy in results["Accuracy"]:
+        assert 0 <= accuracy <= 1
+
+
+def test_leave_one_group_out_without_domain_adaptation(sample_data):
+    estimator = DummyClassifier()
+    check_leave_one_group_out_results(sample_data, estimator, domain_adaptation=False)
+
+
+def test_leave_one_group_out_with_domain_adaptation(sample_data):
+    estimator = CoIRLS(kernel="linear", lambda_=1.0, alpha=1.0)
+    check_leave_one_group_out_results(sample_data, estimator, domain_adaptation=True)