RajTheBuilder/memora-intent-minilm

Memora intent classifier (Ask Memora)

Part of Memora, an AI memory-care companion.

all-MiniLM-L6-v2 sentence embeddings + multinomial logistic regression, stored as plain arrays (weights.npz; no pickle). Trained on author-written questions.

Training metadata

{
  "encoder": "sentence-transformers/all-MiniLM-L6-v2",
  "C": 20,
  "oos_threshold": 0.26,
  "cv_macro_f1": 0.874488622364253,
  "n_train": 189,
  "per_intent": {
    "feeling": 18,
    "medicine": 20,
    "need_help": 17,
    "orientation_place": 16,
    "orientation_time": 16,
    "out_of_scope": 18,
    "today_schedule": 20,
    "where_is_object": 24,
    "who_is_person": 20,
    "who_visited": 20
  },
  "data": "author-written example questions (training/data/intents_train.jsonl)",
  "git_commit": "5ed489d"
}

Evaluation (reproduce with the scripts in eval/)

{
  "keyword_rules": {
    "accuracy": 0.3605,
    "accuracy_95ci": [
      0.2674,
      0.4651
    ],
    "macro_f1": 0.3754,
    "macro_f1_95ci": [
      0.2576,
      0.4599
    ],
    "need_help_recall": 0.3333,
    "false_need_help": 0,
    "accuracy_by_style": {
      "fragment": {
        "n": 5,
        "accuracy": 0.8
      },
      "hinglish": {
        "n": 9,
        "accuracy": 0.0
      },
      "indian-english": {
        "n": 2,
        "accuracy": 0.0
      },
      "indirect": {
        "n": 1,
        "accuracy": 0.0
      },
      "natural": {
        "n": 62,
        "accuracy": 0.371
      },
      "rambling": {
        "n": 3,
        "accuracy": 0.333
      },
      "typo": {
        "n": 4,
        "accuracy": 0.75
      }
    }
  },
  "tfidf_logreg": {
    "accuracy": 0.6628,
    "accuracy_95ci": [
      0.5581,
      0.7674
    ],
    "macro_f1": 0.6693,
    "macro_f1_95ci": [
      0.5593,
      0.7599
    ],
    "need_help_recall": 0.5556,
    "false_need_help": 3,
    "accuracy_by_style": {
      "fragment": {
        "n": 5,
        "accuracy": 0.6
      },
      "hinglish": {
        "n": 9,
        "accuracy": 0.111
      },
      "indian-english": {
        "n": 2,
        "accuracy": 1.0
      },
      "indirect": {
        "n": 1,
        "accuracy": 0.0
      },
      "natural": {
        "n": 62,
        "accuracy": 0.71
      },
      "rambling": {
        "n": 3,
        "accuracy": 1.0
      },
      "typo": {
        "n": 4,
        "accuracy": 1.0
      }
    }
  },
  "minilm_logreg (ours)": {
    "accuracy": 0.8256,
    "accuracy_95ci": [
      0.7442,
      0.907
    ],
    "macro_f1": 0.8401,
    "macro_f1_95ci": [
      0.7507,
      0.9068
    ],
    "need_help_recall": 0.7778,
    "false_need_help": 1,
    "accuracy_by_style": {
      "fragment": {
        "n": 5,
        "accuracy": 0.8
      },
      "hinglish": {
        "n": 9,
        "accuracy": 0.111
      },
      "indian-english": {
        "n": 2,
        "accuracy": 1.0
      },
      "indirect": {
        "n": 1,
        "accuracy": 1.0
      },
      "natural": {
        "n": 62,
        "accuracy": 0.919
      },
      "rambling": {
        "n": 3,
        "accuracy": 1.0
      },
      "typo": {
        "n": 4,
        "accuracy": 0.75
      }
    }
  },
  "minilm_logreg without OOS threshold (ablation)": {
    "accuracy": 0.8372,
    "accuracy_95ci": [
      0.7558,
      0.907
    ],
    "macro_f1": 0.8498,
    "macro_f1_95ci": [
      0.7654,
      0.9153
    ],
    "need_help_recall": 0.7778,
    "false_need_help": 1,
    "accuracy_by_style": {
      "fragment": {
        "n": 5,
        "accuracy": 1.0
      },
      "hinglish": {
        "n": 9,
        "accuracy": 0.111
      },
      "indian-english": {
        "n": 2,
        "accuracy": 1.0
      },
      "indirect": {
        "n": 1,
        "accuracy": 1.0
      },
      "natural": {
        "n": 62,
        "accuracy": 0.919
      },
      "rambling": {
        "n": 3,
        "accuracy": 1.0
      },
      "typo": {
        "n": 4,
        "accuracy": 0.75
      }
    }
  }
}

Limitations

Training and test questions were written by the project author, not collected from patients. English only (romanised Hindi is not understood; spoken Hindi is translated by Whisper first). Not a medical device: the assistant answers only from Memora's own records.

Usage

weights.npz holds the logistic-regression weights as plain arrays (no pickle); embed questions with sentence-transformers/all-MiniLM-L6-v2 (normalized) and apply softmax(X·Wᵀ + b).