Upload 14 files

.gitattributes

@@ -1,35 +1,2 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
 *.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
 *.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,9 @@
+Apache License 2.0
+
+Copyright 2025 hmnshudhmn24
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0

README.md

@@ -1,3 +1,62 @@
----
-license: apache-2.0
----
+---
+language: en
+license: apache-2.0
+datasets: [go_emotions, empathetic_dialogues]
+pipeline_tag: text-generation
+library_name: transformers
+tags:
+  - multimodal
+  - emotion-detection
+  - empathetic-chatbot
+  - t5
+  - clip
+  - streamlit
+base_model: t5-small
+---
+
+# Emo — Multimodal Emotion-Aware Assistant
+
+**Repository:** `hmnshudhmn24/emo-multimodal-assistant`
+
+**Short:** An advanced assistant that detects user emotion from text *and image*, and responds empathetically by conditioning a text-generator (T5) on the detected emotions.
+
+**Components**
+- Text emotion classifier (DistilBERT fine-tuned on GoEmotions)
+- Image emotion detector (CLIP zero-shot with emotion labels)
+- Response generator (T5-small fine-tuned on EmpatheticDialogues)
+- Inference script combining everything
+- Streamlit app for quick demo (text + optional image upload)
+
+## Quick usage (inference)
+
+```python
+from inference import EmoAssistant
+
+assistant = EmoAssistant(
+    text_emotion_model="hmnshudhmn24/emo-text-emotion",
+    response_model="hmnshudhmn24/emo-response-generator"
+)
+
+# text-only
+reply = assistant.respond(user_text="I'm so stressed about exams.")
+print(reply)
+
+# text + image (image path)
+reply = assistant.respond(user_text="I had a rough day", image_path="example.jpg")
+print(reply)
+```
+
+## How to train (short)
+1. Train text emotion classifier:
+   ```bash
+   python train_text_emotion.py --save-dir ./emo-text-emotion
+   ```
+2. Train response generator (empathetic responses):
+   ```bash
+   python train_response_generator.py --save-dir ./emo-response-generator
+   ```
+3. After training, add `pytorch_model.bin`, tokenizer files, and README for each model and upload to Hugging Face or put them in local folders referenced by `inference.py`.
+
+## Notes & Ethics
+- This is not for medical/mental-health diagnosis. It’s built for supportive, empathetic responses only.
+- Always add content / safety filters before production.

README_ASSETS.md

@@ -0,0 +1,8 @@
+# Assets & Where to put model files
+
+- After training `train_text_emotion.py`, save the model and tokenizer to `./emo-text-emotion/` or push to `hmnshudhmn24/emo-text-emotion`.
+- After training `train_response_generator.py`, save to `./emo-response-generator/` or push to `hmnshudhmn24/emo-response-generator`.
+- The inference script expects:
+  - a text classifier model (HF name or local path)
+  - a response generator (HF name or local path)
+  - CLIP is loaded from `openai/clip-vit-base-patch32` via transformers

app.py

@@ -0,0 +1,41 @@
+# app.py - Streamlit demo for Emo Multimodal Assistant
+import streamlit as st
+from inference import EmoAssistant
+from PIL import Image
+import tempfile
+import os
+
+st.set_page_config(page_title="Emo Assistant", layout="centered")
+st.title("Emo — Multimodal Emotion-Aware Assistant")
+
+st.markdown("This demo detects emotion from text and optional image, then generates an empathetic response.")
+
+# model selection / paths
+text_model = st.text_input("Text emotion model (HF repo or local path)", value="distilbert-base-uncased")
+response_model = st.text_input("Response generator model (HF repo or local path)", value="t5-small")
+
+assistant = None
+if st.button("Load models"):
+    with st.spinner("Loading models — this may take a minute..."):
+        assistant = EmoAssistant(text_emotion_model=text_model, response_model=response_model)
+    st.success("Models loaded — ready to go!")
+
+user_text = st.text_area("Your message", value="I had a rough day at work and feel exhausted.", height=120)
+
+uploaded_file = st.file_uploader("Upload an image (optional)", type=["jpg","jpeg","png"])
+image_path = None
+if uploaded_file is not None:
+    tfile = tempfile.NamedTemporaryFile(delete=False, suffix=os.path.splitext(uploaded_file.name)[1])
+    tfile.write(uploaded_file.read())
+    tfile.flush()
+    image_path = tfile.name
+    st.image(Image.open(image_path), caption="Uploaded image", use_column_width=True)
+
+if st.button("Get empathetic reply"):
+    if assistant is None:
+        with st.spinner("Loading models (first time)..."):
+            assistant = EmoAssistant(text_emotion_model=text_model, response_model=response_model)
+    with st.spinner("Detecting emotion and generating response..."):
+        reply = assistant.respond(user_text, image_path=image_path)
+    st.subheader("Assistant reply")
+    st.write(reply)

configs/response_config.json

@@ -0,0 +1,8 @@
+{
+  "model_name_or_path": "t5-small",
+  "max_input_length": 256,
+  "max_target_length": 64,
+  "learning_rate": 0.0003,
+  "batch_size": 8,
+  "num_train_epochs": 3
+}

configs/text_emotion_config.json

@@ -0,0 +1,8 @@
+{
+  "model_name_or_path": "distilbert-base-uncased",
+  "num_labels": 28,
+  "max_length": 128,
+  "learning_rate": 2e-05,
+  "batch_size": 16,
+  "num_train_epochs": 3
+}

example_inputs/example_images.txt

@@ -0,0 +1,4 @@
+# Put image file paths or URLs here for testing image emotion detection.
+# Example:
+# ./examples/sad_person.jpg
+# ./examples/happy_group.jpg

example_inputs/example_texts.txt

@@ -0,0 +1,3 @@
+I had a terrible day at work, nothing I did went right.
+I’m so excited for my new project — I can’t wait!
+I feel like nobody understands me.

inference.py

@@ -0,0 +1,33 @@
+# inference.py
+from transformers import pipeline, T5ForConditionalGeneration, T5TokenizerFast, AutoTokenizer, AutoModelForSequenceClassification
+from utils import predict_image_emotion, combine_emotions
+import torch
+
+class EmoAssistant:
+    def __init__(self, text_emotion_model: str, response_model: str, device: int = None):
+        self.device = device if device is not None else (0 if torch.cuda.is_available() else -1)
+        # text emotion pipeline (single-label)
+        self.text_clf = pipeline("text-classification", model=text_emotion_model, device=self.device, return_all_scores=False)
+        # response generator
+        self.response_tokenizer = T5TokenizerFast.from_pretrained(response_model)
+        self.response_model = T5ForConditionalGeneration.from_pretrained(response_model).to("cuda" if torch.cuda.is_available() else "cpu")
+
+    def detect_text_emotion(self, text: str):
+        res = self.text_clf(text)[0]
+        return res.get("label", "neutral")
+
+    def respond(self, user_text: str, image_path: str = None, max_length: int = 64):
+        text_emotion = self.detect_text_emotion(user_text)
+        image_emotions = None
+        if image_path:
+            image_emotions = predict_image_emotion(image_path)
+        combined = combine_emotions(text_emotion, image_emotions)
+        prompt = f"emotion: {combined} context: {user_text}"
+        inputs = self.response_tokenizer(prompt, return_tensors="pt", truncation=True, max_length=256).to(self.response_model.device)
+        outputs = self.response_model.generate(**inputs, max_length=max_length, num_beams=4, early_stopping=True)
+        reply = self.response_tokenizer.decode(outputs[0], skip_special_tokens=True)
+        return reply
+
+if __name__ == "__main__":
+    assistant = EmoAssistant(text_emotion_model="distilbert-base-uncased", response_model="t5-small")
+    print(assistant.respond("I failed my exam today and feel terrible."))

requirements.txt

@@ -0,0 +1,13 @@
+transformers>=4.30.0
+datasets>=2.10.0
+torch>=1.12.0
+accelerate>=0.18.0
+sentencepiece
+torchvision
+pillow
+streamlit
+python-multipart
+scikit-learn
+evaluate
+ftfy
+rouge_score

train_response_generator.py

@@ -0,0 +1,75 @@
+# train_response_generator.py
+import argparse
+from datasets import load_dataset
+from transformers import (T5TokenizerFast, T5ForConditionalGeneration, Trainer, TrainingArguments)
+import numpy as np
+import evaluate
+import os
+
+def parse_args():
+    p = argparse.ArgumentParser()
+    p.add_argument("--save-dir", type=str, default="./emo-response-generator")
+    p.add_argument("--num_train_epochs", type=int, default=3)
+    p.add_argument("--per_device_train_batch_size", type=int, default=8)
+    return p.parse_args()
+
+def main():
+    args = parse_args()
+    dataset = load_dataset("empathetic_dialogues")
+
+    tokenizer = T5TokenizerFast.from_pretrained("t5-small")
+    model = T5ForConditionalGeneration.from_pretrained("t5-small")
+
+    def preprocess(examples):
+        prompts = []
+        targets = []
+        for ctx, resp, emo in zip(examples["context"], examples["response"], examples["emotion"]):
+            prefix = f"emotion: {emo} context: "
+            ctx_text = " ".join(ctx) if isinstance(ctx, list) else ctx
+            prompts.append(prefix + ctx_text)
+            targets.append(resp)
+        model_inputs = tokenizer(prompts, max_length=256, truncation=True, padding="max_length")
+        labels = tokenizer(text_target=targets, max_length=64, truncation=True, padding="max_length")
+        model_inputs["labels"] = labels["input_ids"]
+        return model_inputs
+
+    tokenized = dataset.map(preprocess, batched=True, remove_columns=dataset["train"].column_names)
+
+    training_args = TrainingArguments(
+        output_dir=args.save_dir,
+        evaluation_strategy="epoch",
+        save_strategy="epoch",
+        learning_rate=3e-4,
+        per_device_train_batch_size=args.per_device_train_batch_size,
+        per_device_eval_batch_size=16,
+        num_train_epochs=args.num_train_epochs,
+        weight_decay=0.01,
+        logging_steps=200,
+        predict_with_generate=True
+    )
+
+    rouge = evaluate.load("rouge")
+    def compute_metrics(eval_pred):
+        preds, labels = eval_pred
+        decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)
+        labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
+        decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)
+        result = rouge.compute(predictions=decoded_preds, references=decoded_labels, use_stemmer=True)
+        return {k: float(v.mid.fmeasure * 100) for k, v in result.items()}
+
+    trainer = Trainer(
+        model=model,
+        args=training_args,
+        train_dataset=tokenized["train"],
+        eval_dataset=tokenized["validation"],
+        tokenizer=tokenizer,
+        compute_metrics=compute_metrics
+    )
+
+    trainer.train()
+    trainer.save_model(args.save_dir)
+    tokenizer.save_pretrained(args.save_dir)
+    print(f"Saved response generator to {args.save_dir}")
+
+if __name__ == "__main__":
+    main()

train_text_emotion.py

@@ -0,0 +1,72 @@
+# train_text_emotion.py
+import argparse
+from datasets import load_dataset, ClassLabel
+from transformers import (DistilBertTokenizerFast, DistilBertForSequenceClassification,
+                          Trainer, TrainingArguments)
+import numpy as np
+import evaluate
+import os
+
+def parse_args():
+    p = argparse.ArgumentParser()
+    p.add_argument("--save-dir", type=str, default="./emo-text-emotion")
+    p.add_argument("--num_train_epochs", type=int, default=3)
+    p.add_argument("--per_device_train_batch_size", type=int, default=16)
+    return p.parse_args()
+
+def main():
+    args = parse_args()
+    dataset = load_dataset("go_emotions")
+    # Simplify multi-label to single-label for demo: pick first label if exists
+    def to_single_label(example):
+        labels = example.get("labels", [])
+        example["label"] = labels[0] if labels else 27  # 27 ~ neutral
+        return example
+
+    dataset = dataset.map(to_single_label)
+
+    tokenizer = DistilBertTokenizerFast.from_pretrained("distilbert-base-uncased")
+
+    def preprocess(examples):
+        return tokenizer(examples["text"], truncation=True, padding="max_length", max_length=128)
+
+    tokenized = dataset.map(preprocess, batched=True)
+    tokenized = tokenized.rename_column("label", "labels")
+    tokenized.set_format(type="torch", columns=["input_ids", "attention_mask", "labels"])
+
+    model = DistilBertForSequenceClassification.from_pretrained("distilbert-base-uncased", num_labels=28)
+
+    metric = evaluate.load("accuracy")
+    def compute_metrics(eval_pred):
+        logits, labels = eval_pred
+        preds = np.argmax(logits, axis=-1)
+        return metric.compute(predictions=preds, references=labels)
+
+    training_args = TrainingArguments(
+        output_dir=args.save_dir,
+        evaluation_strategy="epoch",
+        save_strategy="epoch",
+        learning_rate=2e-5,
+        per_device_train_batch_size=args.per_device_train_batch_size,
+        per_device_eval_batch_size=64,
+        num_train_epochs=args.num_train_epochs,
+        weight_decay=0.01,
+        logging_steps=200
+    )
+
+    trainer = Trainer(
+        model=model,
+        args=training_args,
+        train_dataset=tokenized["train"],
+        eval_dataset=tokenized["validation"],
+        tokenizer=tokenizer,
+        compute_metrics=compute_metrics
+    )
+
+    trainer.train()
+    trainer.save_model(args.save_dir)
+    tokenizer.save_pretrained(args.save_dir)
+    print(f"Saved text emotion model to {args.save_dir}")
+
+if __name__ == "__main__":
+    main()

utils.py

@@ -0,0 +1,50 @@
+# utils.py
+from typing import Optional, List
+import torch
+from transformers import CLIPProcessor, CLIPModel
+import numpy as np
+
+EMOTION_LABELS = [
+    "admiration","amusement","anger","annoyance","approval","caring","confusion","curiosity",
+    "desire","disappointment","disapproval","disgust","embarrassment","excitement","fear",
+    "gratitude","grief","joy","love","nervousness","optimism","pride","realization","relief",
+    "remorse","sadness","surprise","neutral"
+]
+
+def load_clip():
+    model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
+    processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
+    return model, processor
+
+def predict_image_emotion(image_path: str, top_k: int = 3):
+    """Zero-shot emotion detection using CLIP: compute similarity between image and emotion text prompts."""
+    model, processor = load_clip()
+    from PIL import Image
+    img = Image.open(image_path).convert("RGB")
+    inputs = processor(text=EMOTION_LABELS, images=img, return_tensors="pt", padding=True)
+    with torch.no_grad():
+        image_features = model.get_image_features(inputs=inputs["pixel_values"])
+        text_inputs = processor(text=EMOTION_LABELS, return_tensors="pt", padding=True)
+        text_features = model.get_text_features(input_ids=text_inputs["input_ids"], attention_mask=text_inputs["attention_mask"])
+    # normalize and compute cosine
+    img_feat = image_features / image_features.norm(p=2, dim=-1, keepdim=True)
+    txt_feat = text_features / text_features.norm(p=2, dim=-1, keepdim=True)
+    sims = (img_feat @ txt_feat.T).squeeze(0).cpu().numpy()
+    idx = np.argsort(-sims)[:top_k]
+    return [EMOTION_LABELS[i] for i in idx]
+
+def combine_emotions(text_emotion: Optional[str], image_emotions: Optional[List[str]]):
+    parts = []
+    if text_emotion:
+        parts.append(text_emotion)
+    if image_emotions:
+        parts.extend(image_emotions[:2])
+    seen = set()
+    combined = []
+    for p in parts:
+        if p not in seen:
+            combined.append(p)
+            seen.add(p)
+    if not combined:
+        return "neutral"
+    return ", ".join(combined)