//  [python]
def common_pitfalls_and_recommended_practices_example():
	# Data leakage.
	# Data leakage occurs when information that would not be available at prediction time is used when building the model.

	# In the resampling setting, there is a common pitfall that corresponds to resample the entire dataset before splitting it into a train and a test partitions.
	# Note that it would be equivalent to resample the train and test partitions as well.
	# Such of a processing leads to two issues:
	#	The model will not be tested on a dataset with class distribution similar to the real use-case.
	#		Indeed, by resampling the entire dataset, both the training and testing set will be potentially balanced while the model should be tested on the natural imbalanced dataset to evaluate the potential bias of the model.
	#	The resampling procedure might use information about samples in the dataset to either generate or select some of the samples.
	#		Therefore, we might use information of samples which will be later used as testing samples which is the typical data leakage issue.

	# For the sake of simplicity, we will only use the numerical features.
	# Also, we will make the dataset more imbalanced to increase the effect of the wrongdoings.
	X, y = sklearn.datasets.fetch_openml(data_id=1119, as_frame=True, return_X_y=True)
	X = X.select_dtypes(include="number")
	X, y = imblearn.datasets.make_imbalance(X, y, sampling_strategy={">50K": 300}, random_state=1)

	print(y.value_counts(normalize=True))

	# To later highlight some of the issue, we will keep aside a left-out set that we will not use for the evaluation of the model.
	X, X_left_out, y, y_left_out = sklearn.model_selection.train_test_split(X, y, stratify=y, random_state=0)

	# We will train and check the performance of this classifier, without any preprocessing to alleviate the bias toward the majority class.
	# We evaluate the generalization performance of the classifier via cross-validation.
	model = sklearn.ensemble.HistGradientBoostingClassifier(random_state=0)
	cv_results = sklearn.model_selection.cross_validate(
		model, X, y, scoring="balanced_accuracy",
		return_train_score=True, return_estimator=True,
		n_jobs=-1,
	)
	print(
		f"Balanced accuracy mean +/- std. dev.: "
		f"{cv_results['test_score'].mean():.3f} +/- "
		f"{cv_results['test_score'].std():.3f}"
	)

	scores = []
	for fold_id, cv_model in enumerate(cv_results["estimator"]):
		scores.append(sklearn.metrics.balanced_accuracy_score(y_left_out, cv_model.predict(X_left_out)))
	print(
		f"Balanced accuracy mean +/- std. dev.: "
		f"{np.mean(scores):.3f} +/- {np.std(scores):.3f}"
	)

	# Let's now show the wrong pattern to apply when it comes to resampling to alleviate the class imbalance issue.
	# We will use a sampler to balance the entire dataset and check the statistical performance of our classifier via cross-validation.
	sampler = imblearn.under_sampling.RandomUnderSampler(random_state=0)
	X_resampled, y_resampled = sampler.fit_resample(X, y)
	model = sklearn.ensemble.HistGradientBoostingClassifier(random_state=0)
	cv_results = sklearn.model_selection.cross_validate(
		model, X_resampled, y_resampled, scoring="balanced_accuracy",
		return_train_score=True, return_estimator=True,
		n_jobs=-1,
	)
	print(
		f"Balanced accuracy mean +/- std. dev.: "
		f"{cv_results['test_score'].mean():.3f} +/- "
		f"{cv_results['test_score'].std():.3f}"
	)

	# We will now illustrate the correct pattern to use.
	# Indeed, as in scikit-learn, using a Pipeline avoids to make any data leakage because the resampling will be delegated to imbalanced-learn and does not require any manual steps.
	model = imblearn.pipeline.make_pipeline(
		imblearn.under_sampling.RandomUnderSampler(random_state=0),
		sklearn.ensemble.HistGradientBoostingClassifier(random_state=0),
	)
	cv_results = sklearn.model_selection.cross_validate(
		model, X, y, scoring="balanced_accuracy",
		return_train_score=True, return_estimator=True,
		n_jobs=-1,
	)
	print(
		f"Balanced accuracy mean +/- std. dev.: "
		f"{cv_results['test_score'].mean():.3f} +/- "
		f"{cv_results['test_score'].std():.3f}"
	)

	# We see that the statistical performance are very close to the cross-validation study that we perform, without any sign of over-optimistic results.
	scores = []
	for fold_id, cv_model in enumerate(cv_results["estimator"]):
		scores.append(sklearn.metrics.balanced_accuracy_score(y_left_out, cv_model.predict(X_left_out)))
	print(
		f"Balanced accuracy mean +/- std. dev.: "
		f"{np.mean(scores):.3f} +/- {np.std(scores):.3f}"
	)