Harper - Final Assignment submission

.gitattributes

@@ -56,3 +56,18 @@ images/6.png filter=lfs diff=lfs merge=lfs -text
 images/7.png filter=lfs diff=lfs merge=lfs -text
 images/8.png filter=lfs diff=lfs merge=lfs -text
 images/9.png filter=lfs diff=lfs merge=lfs -text
+Photos/20240325_133025.jpg filter=lfs diff=lfs merge=lfs -text
+Photos/20250106_075208.jpg filter=lfs diff=lfs merge=lfs -text
+Photos/20250805_161445.jpg filter=lfs diff=lfs merge=lfs -text
+Photos/20250808_131957.jpg filter=lfs diff=lfs merge=lfs -text
+Photos/20250808_164442.jpg filter=lfs diff=lfs merge=lfs -text
+Photos/20250808_183114.jpg filter=lfs diff=lfs merge=lfs -text
+Photos/20250813_113228.jpg filter=lfs diff=lfs merge=lfs -text
+Photos/20250918_170635.jpg filter=lfs diff=lfs merge=lfs -text
+Photos/20250920_114728.jpg filter=lfs diff=lfs merge=lfs -text
+Photos/20251101_155708.jpg filter=lfs diff=lfs merge=lfs -text
+Photos/20251106_170359.jpg filter=lfs diff=lfs merge=lfs -text
+Photos/20251106_192036.jpg filter=lfs diff=lfs merge=lfs -text
+Photos/20251107_100830.jpg filter=lfs diff=lfs merge=lfs -text
+Photos/20251107_101822.jpg filter=lfs diff=lfs merge=lfs -text
+Photos/20251107_150015.jpg filter=lfs diff=lfs merge=lfs -text

app.py

@@ -1,41 +1,131 @@
+# Caption Generator w/English-to-Spanish Translation
+# A. Harper | ARIN 460 | December 2025
+
+# Load into Hugging Face Space (using the Gradio Framework)
+# Include requirements.txt file (list: gradio, pandas, torch, sentencepiece, tensorflow, Image, transformers)
+
+# To run, navigate to the App tab. Click the red Generate button. 
+# The app will randomly select image, generate (English) caption, 
+    # then generate Spanish translation. 
+
+
+# Import gradio - app framework
 import gradio as gr
-from transformers import Blip2Processor, Blip2ForConditionalGeneration
-import torch
 
-# Load pre-trained BLIP-2 model and processor
-processor = Blip2Processor.from_pretrained("Salesforce/blip2-opt-2.7b")
-model = Blip2ForConditionalGeneration.from_pretrained(
-    "Salesforce/blip2-opt-2.7b",
-    torch_dtype=torch.float16
+
+# Two image datasources are available. 
+# Minor adjustments (add/remove # to deactivate/activate) to switch between datasources.
+# AA comments refer to images in the DataFrame / from Coco database
+# BB comments refer to images stored in local Gradio app folder
+
+
+# Import os and random to support random selection of image (from folder)
+import os
+import random
+
+
+# Import pandas datasets, transformers, torch
+import pandas as pd
+
+from datasets import load_dataset
+
+from transformers import (
+    BlipProcessor, 
+    BlipForConditionalGeneration,
+    AutoTokenizer,
+    AutoModelForSeq2SeqLM,
+    MarianMTModel, 
+    MarianTokenizer
 )
 
-def predict(image, question=None):
-    # If no question provided, generate a caption
-    if question is None or question.strip() == "":
-        inputs = processor(image, return_tensors="pt")
-    else:
-        inputs = processor(image, question, return_tensors="pt")
-    
-    # Move to GPU if available
-    device = "cuda" if torch.cuda.is_available() else "cpu"
-    inputs = inputs.to(device)
-    model.to(device)
+from PIL import Image
+import torch
+
+
+# AA: Load dataset. Initial image source. 
+#Load dataset (henryscheible/coco_val2014_tiny)
+dataset = load_dataset("henryscheible/coco_val2014_tiny", split="validation")
+
+
+# Reduce dataset to 20 rows, i.e., get sample
+samples = dataset.select(range(20))
+
+
+#Convert to dataframe
+df = pd.DataFrame(samples)
+
+
+# BB: Direct to Photos folder
+IMAGE_FOLDER = "Photos"
+
+image_paths = [
+    os.path.join(IMAGE_FOLDER, f)
+    for f in os.listdir(IMAGE_FOLDER)
+    if f.lower().endswith((".jpg", ".jpeg", ".png"))
+]
+
+#Load the image captioning model (Salesforce/blip-image-captioning-large)
+processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-large")
+model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-large")
+
+
+#Load transformer for translating captions from English to Spanish
+model_name = "Helsinki-NLP/opus-mt-en-es"
+trans_tokenizer = MarianTokenizer.from_pretrained(model_name)
+trans_model = MarianMTModel.from_pretrained(model_name)
+
+
+#Configure captioning function
+
+def caption_random_image():
+
+
+    # AA: pick random row - from DF
+    ##sample = df.sample(1).iloc[0]
+
+
+    # BB: Pick a random image path - image from folder
+    img_path = random.choice(image_paths)
+
+
+    # BB: Load into PIL - image from folder - image from folder
+    image = Image.open(img_path).convert("RGB")
+
     
-    # Generate output
-    out = model.generate(**inputs, max_new_tokens=50)
-    result = processor.decode(out[0], skip_special_tokens=True)
-    return result
-
-# Gradio interface
-iface = gr.Interface(
-    fn=predict,
-    inputs=[
-        gr.Image(type="pil", label="Upload Image"),
-        gr.Textbox(label="Optional Question", placeholder="Ask something about the image...")
+    # AA: Image - for DF
+    ##image = sample["image"]
+
+
+    # Unconditional image captioning
+    inputs = processor(image, return_tensors="pt")
+
+
+    out = model.generate(**inputs)
+    caption_eng = processor.decode(out[0], skip_special_tokens=True)
+
+
+    # Translate caption from English to Spanish
+    trans_inputs = trans_tokenizer.encode(caption_eng, return_tensors="pt")
+    trans_out = trans_model.generate(trans_inputs)
+    caption_es = trans_tokenizer.decode(trans_out[0], skip_special_tokens=True)
+
+
+    return image, caption_eng, caption_es
+
+
+
+
+demo = gr.Interface(
+    fn=caption_random_image,
+    inputs=None,
+    outputs=[
+        gr.Image(type="pil", label="Random Image"),
+        gr.Textbox(label="Caption (English)"),
+        gr.Textbox(label="Caption (Spanish)")
     ],
-    outputs=gr.Textbox(label="Result"),
-    title="BLIP-2 Multimodal Assistant",
-    description="Upload an image and get a caption. Optionally, ask a question about the image."
+    title="Image Captioning (with English to Spanish translation)",
+    description="Selects a random image (from either the local folder or henryscheible/coco data subset); generates a BLIP caption; then translates the (English) caption to Spanish."
 )
 
-iface.launch()
+
+demo.launch()

requirements.txt

@@ -1,4 +1,8 @@
-gradio>=4.0
-transformers>=4.30
+gradio
+pandas
 torch
-pillow
+sentencepiece
+tensorflow
+Image
+transformers
+