Add cross validation

app/cli.py

@@ -90,6 +90,13 @@ def predict(model_path: Path, text: list[str]) -> None:
     show_default=True,
     type=click.IntRange(1, None),
 )
+@click.option(
+    "--cv",
+    default=5,
+    help="Number of cross-validation folds",
+    show_default=True,
+    type=click.IntRange(1, 50),
+)
 @click.option(
     "--seed",
     default=42,
@@ -97,19 +104,26 @@ def predict(model_path: Path, text: list[str]) -> None:
     show_default=True,
     type=click.IntRange(-1, None),
 )
+@click.option(
+    "--force",
+    is_flag=True,
+    help="Overwrite the model file if it already exists",
+)
 def train(
     dataset: Literal["sentiment140", "amazonreviews", "imdb50k"],
     max_features: int,
+    cv: int,
     seed: int,
+    force: bool,
 ) -> None:
     """Train the model on the provided dataset"""
     import joblib
 
     from app.constants import MODELS_DIR
-    from app.model import create_model, load_data, train_model
+    from app.model import create_model, evaluate_model, load_data, train_model
 
     model_path = MODELS_DIR / f"{dataset}_tfidf_ft-{max_features}.pkl"
-    if model_path.exists():
+    if model_path.exists() and not force:
         click.confirm(f"Model file '{model_path}' already exists. Overwrite?", abort=True)
 
     click.echo("Preprocessing dataset... ", nl=False)
@@ -122,16 +136,17 @@ def train(
 
     # click.echo("Training model... ", nl=False)
     click.echo("Training model... ")
-    accuracy = train_model(model, text_data, label_data)
-    joblib.dump(model, model_path)
-    click.echo("Model saved to: ", nl=False)
-    click.secho(str(model_path), fg="blue")
-
+    accuracy, text_test, text_label = train_model(model, text_data, label_data)
     click.echo("Model accuracy: ", nl=False)
     click.secho(f"{accuracy:.2%}", fg="blue")
 
-    # TODO: Add hyperparameter options
-    # TODO: Random/grid search for finding best classifier and hyperparameters
+    click.echo("Model saved to: ", nl=False)
+    joblib.dump(model, model_path)
+    click.secho(str(model_path), fg="blue")
+
+    click.echo("Evaluating model... ", nl=False)
+    acc_mean, acc_std = evaluate_model(model, text_test, text_label, cv=cv)
+    click.secho(f"{acc_mean:.2%} ± {acc_std:.2%}", fg="blue")
 
 
 def cli_wrapper() -> None:

app/model.py

@@ -13,7 +13,7 @@ from nltk.stem import WordNetLemmatizer
 from sklearn.base import BaseEstimator, TransformerMixin
 from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
 from sklearn.linear_model import LogisticRegression
-from sklearn.model_selection import train_test_split
+from sklearn.model_selection import cross_val_score, train_test_split
 from sklearn.pipeline import Pipeline
 
 from app.constants import (
@@ -28,7 +28,7 @@ from app.constants import (
     URL_REGEX,
 )
 
-__all__ = ["load_data", "create_model", "train_model"]
+__all__ = ["load_data", "create_model", "train_model", "evaluate_model"]
 
 
 class TextCleaner(BaseEstimator, TransformerMixin):
@@ -293,7 +293,7 @@ def train_model(
     text_data: list[str],
     label_data: list[int],
     seed: int = 42,
-) -> float:
+) -> tuple[float, list[str], list[int]]:
     """Train the sentiment analysis model.
 
     Args:
@@ -303,7 +303,7 @@ def train_model(
         seed: Random seed (None for random seed)
 
     Returns:
-        Accuracy score
+        Model accuracy and test data
     """
     text_train, text_test, label_train, label_test = train_test_split(
         text_data,
@@ -316,4 +316,33 @@ def train_model(
         warnings.simplefilter("ignore")
         model.fit(text_train, label_train)
 
-    return model.score(text_test, label_test)
+    return model.score(text_test, label_test), text_test, label_test
+
+
+def evaluate_model(
+    model: Pipeline,
+    text_test: list[str],
+    label_test: list[int],
+    cv: int = 5,
+) -> tuple[float, float]:
+    """Evaluate the model using cross-validation.
+
+    Args:
+        model: Trained model
+        text_test: Text data
+        label_test: Label data
+        seed: Random seed (None for random seed)
+        cv: Number of cross-validation folds
+
+    Returns:
+        Mean accuracy and standard deviation
+    """
+    scores = cross_val_score(
+        model,
+        text_test,
+        label_test,
+        cv=cv,
+        scoring="accuracy",
+        n_jobs=-1,
+    )
+    return scores.mean(), scores.std()

notebook.ipynb

@@ -668,9 +668,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 24,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Fitting 3 folds for each of 10 candidates, totalling 30 fits\n"
+     ]
+    }
+   ],
    "source": [
     "# SVM\n",
     "svm_clf = SVC(random_state=SEED)\n",
@@ -680,7 +688,7 @@
     "    svm_clf,\n",
     "    {\n",
     "        \"C\": np.logspace(-4, 4, 20),\n",
-    "        \"kernel\": [\"linear\", \"poly\", \"rbf\", \"sigmoid\"],\n",
+    "        \"kernel\": [\"linear\", \"poly\", \"rbf\"],\n",
     "        \"degree\": [2, 3, 4],\n",
     "    },\n",
     ")\n",