Add new models

compute_accuracy.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python3
+"""
+Compute top-1 accuracy for each model by comparing predictions with ground truth.
+"""
+
+import json
+import os
+from collections import OrderedDict
+
+# Species mapping from demo/app.py
+SPECIES_MAP = OrderedDict([
+    (24, "Jaguar"),           # panthera onca
+    (10, "Ocelot"),           # leopardus pardalis
+    (6, "Mountain Lion"),     # puma concolor
+    (101, "Common Eland"),    # tragelaphus oryx
+    (102, "Waterbuck"),       # kobus ellipsiprymnus
+    (163, "African Wild Dog") # lycaon pictus
+])
+
+def load_ground_truth():
+    """Load ground truth labels from annotations."""
+    with open('iwildcam_demo_annotations.json', 'r') as f:
+        data = json.load(f)
+
+    # Create mapping from filename to true label
+    ground_truth = {}
+    for annotation in data['annotations']:
+        image_id = annotation['image_id']
+        category_id = annotation['category_id']
+        image_info = next((img for img in data['images'] if img['id'] == image_id), None)
+        if image_info:
+            filename = image_info['file_name']
+            true_label = SPECIES_MAP.get(category_id, "Unknown")
+            if true_label != "Unknown":
+                ground_truth[filename] = true_label
+
+    return ground_truth
+
+def compute_accuracy(results_file, ground_truth):
+    """Compute top-1 accuracy for a model's results."""
+    with open(results_file, 'r') as f:
+        data = json.load(f)
+
+    model_name = data['model']
+    results = data['results']
+
+    correct = 0
+    total = 0
+
+    for filename, scores in results.items():
+        if filename in ground_truth:
+            # Get predicted class (highest score)
+            predicted_class = max(scores, key=scores.get)
+            true_class = ground_truth[filename]
+
+            if predicted_class == true_class:
+                correct += 1
+            total += 1
+
+    accuracy = correct / total if total > 0 else 0.0
+    return accuracy, correct, total
+
+def main():
+    """Compute accuracy for all models."""
+    print("Computing top-1 accuracy for each model...\n")
+
+    # Load ground truth
+    ground_truth = load_ground_truth()
+    print(f"Loaded ground truth for {len(ground_truth)} images")
+
+    # Find all results files
+    results_files = [f for f in os.listdir('.') if f.startswith('zeroshot_results_') and f.endswith('.json')]
+
+    if not results_files:
+        print("No results files found!")
+        return
+
+    print(f"Found {len(results_files)} results files\n")
+
+    # Compute accuracy for each model
+    accuracies = {}
+    for results_file in sorted(results_files):
+        try:
+            accuracy, correct, total = compute_accuracy(results_file, ground_truth)
+
+            # Extract model name from filename
+            model_name = results_file.replace('zeroshot_results_', '').replace('.json', '').replace('_', '/')
+
+            accuracies[model_name] = {
+                'accuracy': accuracy,
+                'correct': correct,
+                'total': total
+            }
+
+            print(f"{model_name}:")
+            print(f"  Accuracy: {accuracy:.4f} ({correct}/{total})")
+            print()
+
+        except Exception as e:
+            print(f"Error processing {results_file}: {e}")
+
+    # Summary
+    print("="*60)
+    print("SUMMARY")
+    print("="*60)
+
+    # Sort by accuracy
+    sorted_models = sorted(accuracies.items(), key=lambda x: x[1]['accuracy'], reverse=True)
+
+    for i, (model_name, stats) in enumerate(sorted_models, 1):
+        print(f"{i}. {model_name}: {stats['accuracy']:.4f}")
+
+    # Show some example predictions vs ground truth
+    print("\n" + "="*60)
+    print("SAMPLE PREDICTIONS (first 10 images)")
+    print("="*60)
+
+    if results_files:
+        # Use the first model's results to show examples
+        with open(results_files[0], 'r') as f:
+            data = json.load(f)
+
+        results = data['results']
+        count = 0
+
+        for filename, scores in results.items():
+            if filename in ground_truth and count < 10:
+                predicted_class = max(scores, key=scores.get)
+                true_class = ground_truth[filename]
+                confidence = scores[predicted_class]
+
+                status = "✓" if predicted_class == true_class else "✗"
+
+                print(f"{filename}:")
+                print(f"  True: {true_class}")
+                print(f"  Pred: {predicted_class} ({confidence:.4f}) {status}")
+                print()
+
+                count += 1
+
+if __name__ == "__main__":
+    main()

iwildcam_demo.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b44bb58d31c2f26a17a754861ed39d30f38afd0a6a263c358b5dcfafbe287b21
-size 77715
+oid sha256:22671a83d662556a98fe14daaa122da9e9a22f9f08e24c7c2a4467c58fc3d206
+size 128467

iwildcam_demo_labels.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c6c3f2f49fe5b1dce7f4c9e45380fa68fe4040f004d9bcf7ff73bb3323d096f7
+oid sha256:67935d5cd3e5616d16f1105afdfdb936b5fc3bacd6fba733584efc2c2a951b90
 size 11780

models.txt

@@ -1,3 +1,5 @@
+facebook/PE-Core-L14-336
 google/siglip2-so400m-patch16-naflex
 openai/clip-vit-large-patch14
-imageomics/bioclip
+imageomics/bioclip-2
+laion/CLIP-ViT-L-14-laion2B-s32B-b82K

process_iwildcam_data.py

@@ -0,0 +1,133 @@
+import json
+import torch
+import numpy as np
+
+def load_annotations(annotation_file):
+    with open(annotation_file, 'r') as f:
+        data = json.load(f)
+
+    # Create mapping from image_id to category_id
+    image_to_category = {}
+    for annotation in data['annotations']:
+        image_to_category[annotation['image_id']] = annotation['category_id']
+
+    # Get image order and their corresponding labels
+    images = []
+    labels = []
+    for image in data['images']:
+        image_id = image['id']
+        file_name = image['file_name']
+        if image_id in image_to_category:
+            images.append(file_name)
+            labels.append(image_to_category[image_id])
+
+    return images, labels
+
+def load_model_predictions(model_files):
+    models = []
+    all_predictions = {}
+    class_names = None
+
+    for model_file in model_files:
+        with open(model_file, 'r') as f:
+            data = json.load(f)
+
+        model_name = data['model']
+        models.append(model_name)
+
+        if class_names is None:
+            class_names = data['class_names']
+
+        # Store predictions for each image
+        for image_name, predictions in data['results'].items():
+            if image_name not in all_predictions:
+                all_predictions[image_name] = {}
+            all_predictions[image_name][model_name] = predictions
+
+    return models, all_predictions, class_names
+
+def create_tensors():
+    # Load annotations
+    images, labels = load_annotations('iwildcam_demo_annotations.json')
+
+    # Load model predictions
+    model_files = [
+	'zeroshot_results_facebook_PE_Core_L14_336.json',
+        'zeroshot_results_google_siglip2_so400m_patch16_naflex.json',
+	'zeroshot_results_openai_clip_vit_large_patch14.json',
+        'zeroshot_results_imageomics_bioclip_2.json',
+	'zeroshot_results_laion_CLIP_ViT_L_14_laion2B_s32B_b82K.json',
+    ]
+    models, all_predictions, class_names = load_model_predictions(model_files)
+
+    # Create class to index mapping (0-indexed)
+    class_to_idx = {class_name: idx for idx, class_name in enumerate(class_names)}
+
+    # Filter images that have predictions from all models
+    valid_images = []
+    valid_labels = []
+    for i, image_name in enumerate(images):
+        if image_name in all_predictions and len(all_predictions[image_name]) == len(models):
+            valid_images.append(image_name)
+            valid_labels.append(labels[i])
+
+    print(f"Found {len(valid_images)} images with predictions from all {len(models)} models")
+
+    # Create prediction tensor: H x N x C
+    H = len(models)  # number of models
+    N = len(valid_images)  # number of images
+    C = len(class_names)  # number of classes
+
+    prediction_tensor = torch.zeros(H, N, C)
+
+    for h, model_name in enumerate(models):
+        for n, image_name in enumerate(valid_images):
+            predictions = all_predictions[image_name][model_name]
+            for class_name, score in predictions.items():
+                c = class_to_idx[class_name]
+                prediction_tensor[h, n, c] = score
+
+    # Convert category_ids to 0-indexed labels based on class_names order
+    # Need to map from species names back to category IDs
+    from collections import OrderedDict
+    SPECIES_MAP = OrderedDict([
+        (24, "Jaguar"),
+        (10, "Ocelot"),
+        (6, "Mountain Lion"),
+        (101, "Common Eland"),
+        (102, "Waterbuck")
+    ])
+    reverse_species_map = {v: k for k, v in SPECIES_MAP.items()}
+
+    # Create category_to_label mapping based on class_names order (not sorted category IDs)
+    category_order = [reverse_species_map[name] for name in class_names]
+    category_to_label = {cat: idx for idx, cat in enumerate(category_order)}
+    label_tensor = torch.tensor([category_to_label[cat] for cat in valid_labels])
+
+    # Save tensors
+    torch.save(prediction_tensor, 'iwildcam_demo.pt')
+    torch.save(label_tensor, 'iwildcam_demo_labels.pt')
+
+    # Save text files
+    with open('models.txt', 'w') as f:
+        for model in models:
+            f.write(f"{model}\n")
+
+    with open('images.txt', 'w') as f:
+        for image in valid_images:
+            f.write(f"{image}\n")
+
+    with open('classes.txt', 'w') as f:
+        for class_name in class_names:
+            f.write(f"{class_name}\n")
+
+    print(f"Saved prediction tensor of shape {prediction_tensor.shape} to iwildcam_demo.pt")
+    print(f"Saved label tensor of shape {label_tensor.shape} to iwildcam_demo_labels.pt")
+    print(f"Saved {len(models)} models to models.txt")
+    print(f"Saved {len(valid_images)} images to images.txt")
+    print(f"Saved {len(class_names)} classes to classes.txt")
+
+    return prediction_tensor, label_tensor, models, valid_images, class_names
+
+if __name__ == "__main__":
+    create_tensors()