average_precision_score not working as expected

### Describe the bug

When compute AP with average_precision_score, I get unexpected results. The y_scores (output from the models) are very low for positive samples, so my AP should be very low. Instead I get a perfect 1.0 AP score.

### Steps/Code to Reproduce

```python
from sklearn.metrics import average_precision_score

# labels
y_true = [1, 1, 1, 1, 1]  # 5 positives
# Predicted scores
y_scores = [0.1, 0.3, 0.1, 0.2, 0.1]  # Model's confidence in predictions

# Calculate average precision
average_precision = average_precision_score(y_true, y_scores)
print("AP score:", average_precision)
```

### Expected Results

An AP score close to 0.

### Actual Results

AP score: 1.0

### Versions

```shell
System:
    python: 3.9.19 (main, May  6 2024, 19:43:03)  [GCC 11.2.0]
executable: /bin/python
   machine: Linux-5.15.0-121-generic-x86_64-with-glibc2.31

Python dependencies:
      sklearn: 1.5.2
          pip: 24.2
   setuptools: 75.1.0
        numpy: 1.26.4
        scipy: 1.13.1
       Cython: None
       pandas: 1.4.0
   matplotlib: 3.5.3
       joblib: 1.4.2
threadpoolctl: 3.5.0

Built with OpenMP: True

threadpoolctl info:
       user_api: blas
   internal_api: openblas
    num_threads: 32
         prefix: libopenblas
       filepath: user/anaconda3/envs/perception39/lib/python3.9/site-packages/numpy.libs/libopenblas64_p-r0-0cf96a72.3.23.dev.so
        version: 0.3.23.dev
threading_layer: pthreads
   architecture: SkylakeX

       user_api: blas
   internal_api: openblas
    num_threads: 32
         prefix: libopenblas
       filepath: user/anaconda3/envs/perception39/lib/python3.9/site-packages/scipy.libs/libopenblasp-r0-01191904.3.27.so
        version: 0.3.27
threading_layer: pthreads
   architecture: SkylakeX

       user_api: openmp
   internal_api: openmp
    num_threads: 32
         prefix: libgomp
       filepath: user/anaconda3/envs/perception39/lib/python3.9/site-packages/scikit_learn.libs/libgomp-a34b3233.so.1.0.0
        version: None
```

COMMENT:
I draw it like in my code to compute AP. I tested different method with different interpolations. They all give about 0.5 AP on the example input from above, but the direct sklearn method that is giving 0.97 AP.
This is because in all methods the general idea is to compute the area under the PR curve. In my example, all methods stop at recall ~0.53, as my model never goes higher. But scikit-learn assumes a recall of 1 when the threshold is 0, this is not true in object detection.

```python
def compute_ap(precision, recall):
    """Compute AP given precision and recall values."""
    precision = np.concatenate(([0.0], precision, [0.0]))
    recall = np.concatenate(([0.0], recall, [1.0]))

    # Ensure precision is non-increasing
    for i in range(len(precision) - 1, 0, -1):
        precision[i - 1] = max(precision[i - 1], precision[i])

    # Find points where recall changes
    indices = np.where(recall[1:] != recall[:-1])[0]
    
    # Sum (recall_delta * precision)
    ap = np.sum((recall[indices + 1] - recall[indices]) * precision[indices + 1])
    return ap

# Compute AP using the COCO method
def compute_ap_coco(precision, recall):
    """Compute AP using the COCO method."""
    recall_thresholds = np.arange(0.0, 1.01, 0.01)
    precisions = np.zeros_like(recall_thresholds)
    for i, t in enumerate(recall_thresholds):
        precisions[i] = np.max(precision[recall >= t]) if (recall >= t).any() else 0.0
    return np.mean(precisions)

def compute_ap_sum(precision, recall):
    """Compute AP using the sum method."""
    return np.sum((recall[1:] - recall[:-1]) * precision[1:])

# Compute AP scipy area under curve
def compute_ap_scipy(precision, recall):
    """Compute AP using the scipy method."""
    return auc(recall, precision)


def compute_class_ap(preds, annots, iou_threshold, classes):
    """Compute AP per class and mAP.
    
    Args:
        preds: List (N_images, N_preds, 6) -> (class_idx, x1, y1, x2, y2, conf)
        annots: List (N_images, N_boxes, 5) -> (class_idx, x1, y1, x2, y2)
        iou_threshold: IOU threshold to match predictions and ground truths.
    
    Returns:
        class_ap: Dictionary of AP per class.
        mAP: Mean AP across all classes.
    """
    # Initialize lists to store true positives, false positives, and scores for each class
    all_true_positives = defaultdict(list)
    all_false_positives = defaultdict(list)
    all_scores = defaultdict(list)
    n_gt_per_class = defaultdict(int)
    y_true = defaultdict(list)
    y_scores = defaultdict(list)
    
    N_images = len(preds)

    for img_idx in range(N_images):
        pred_boxes = preds[img_idx]
        annot_boxes = annots[img_idx]

        for class_idx in range(len(classes)):
            pred_class = pred_boxes[pred_boxes[:, 0] == class_idx] if len(pred_boxes) > 0 else np.array([])
            annot_class = annot_boxes[annot_boxes[:, 0] == class_idx] if len(annot_boxes) > 0 else np.array([])

            # Sort predictions by confidence score
            if len(pred_class) > 0:
                pred_class = pred_class[np.argsort(-pred_class[:, 5])]

            # Track matched ground truths
            matched_gt = np.zeros(len(annot_class), dtype=bool)

            for pred in pred_class:
                if len(annot_class) > 0:
                    ious = np.array([get_iou(pred[1:5], annot[1:5]) for annot in annot_class])
                    
                    max_iou_idx = np.argmax(ious)
                    max_iou = ious[max_iou_idx]
                    if max_iou >= iou_threshold and not matched_gt[max_iou_idx]:
                        # True positive
                        all_true_positives[class_idx].append(1)
                        all_false_positives[class_idx].append(0)
                        matched_gt[max_iou_idx] = True
                        y_true[class_idx].append(1)
                    else:
                        # False positive
                        all_true_positives[class_idx].append(0)
                        all_false_positives[class_idx].append(1)
                        y_true[class_idx].append(0)
                else:
                    # False positive because there are no annotations
                    all_true_positives[class_idx].append(0)
                    all_false_positives[class_idx].append(1)
                    y_true[class_idx].append(0)
                
                all_scores[class_idx].append(pred[5])
                y_scores[class_idx].append(pred[5])

            # Count number of ground truths for the class
            n_gt_per_class[class_idx] += len(annot_class)
            missed_gt = len(annot_class) - np.sum(matched_gt)
            y_true[class_idx].extend([1] * missed_gt)
            y_scores[class_idx].extend([0] * missed_gt)

    # Compute AP per class
    class_ap = {}
    for class_idx in all_scores:
        # Sort by score
        indices = np.argsort(-np.array(all_scores[class_idx]))
        true_positives = np.array(all_true_positives[class_idx])[indices]
        false_positives = np.array(all_false_positives[class_idx])[indices]

        # Compute cumulative sum of true positives and false positives
        cum_tp = np.cumsum(true_positives)
        cum_fp = np.cumsum(false_positives)

        # Compute precision and recall
        recall = cum_tp / n_gt_per_class[class_idx]
        precision = cum_tp / (cum_tp + cum_fp)

        #plot precision recall curve
        plt.figure(figsize=(10, 6))
        plt.plot(recall, precision, marker='.')
        plt.xlabel('Recall')
        plt.ylabel('Precision')
        plt.title('Precision-Recall Curve')
        plt.show()

        #compute precision and recall with sklearn
        from sklearn.metrics import precision_recall_curve
        precision, recall, thresholds = precision_recall_curve(y_true[class_idx], y_scores[class_idx])

        plt.figure(figsize=(10, 6))
        plt.plot(recall, precision, marker='.')
        plt.xlabel('Recall')
        plt.ylabel('Precision')
        plt.title('Precision-Recall Curve')
        plt.show()

        # Compute AP
        #class_ap[class_idx] = compute_ap(precision, recall)
        #class_ap[class_idx] = compute_ap_voc(precision, recall)
        #class_ap[class_idx] = compute_ap_coco(precision, recall)
        #class_ap[class_idx] = compute_ap_scipy(precision, recall)
        #class_ap[class_idx] = compute_ap_sum(precision, recall)
        from sklearn.metrics import average_precision_score
        print(f"Y_true: {y_true[class_idx]}")
        print(f"Y_scores: {y_scores[class_idx]}")
        class_ap[class_idx] = average_precision_score(y_true[class_idx], y_scores[class_idx])


    # Compute mAP
    mAP = np.mean(list(class_ap.values())) if class_ap else 0.0
    
    return mAP, class_ap
```




Another way to just have the plots would be:
```python
from sklearn.metrics import average_precision_score, precision_recall_curve
import matplotlib.pyplot as plt

y_true = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1]
y_scores = [0.9997937083244324, 0.9848889112472534, 0, 0.9497167468070984, 0, 0, 0.9908866882324219, 0.9809730052947998, 0.9870287179946899, 0.9999470710754395, 0, 0, 0.9267389178276062, 0, 0.9746383428573608, 0.9993682503700256, 0.8148384094238281, 0, 0, 0, 0.6392842531204224, 0.9999898672103882, 1.0, 0, 0, 0, 0, 0.9375026822090149, 0.9581012725830078, 0.6106476783752441, 0.9974182844161987, 0, 0, 0, 1.0, 0.999996542930603, 0.8294159173965454, 0.7495031356811523, 0, 0, 0, 0.907082200050354]

# Calculate precision-recall curve using sklearn
precision, recall, thresholds = precision_recall_curve(y_true, y_scores)

print("Precision:", precision)
print("Recall:", recall)
print("Thresholds:", thresholds)

# Plot the curve
plt.figure(figsize=(10, 6))
plt.plot(recall, precision, marker='.')
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.title('Precision-Recall Curve')
plt.show()

# Calculate precision-recall curve manually


def precision_recall(y_true, y_scores, threshold):
    y_pred = [1 if score > threshold else 0 for score in y_scores]
    tp = sum([1 for yt, yp in zip(y_true, y_pred) if yt == 1 and yp == 1])
    fp = sum([1 for yt, yp in zip(y_true, y_pred) if yt == 0 and yp == 1])
    fn = sum([1 for yt, yp in zip(y_true, y_pred) if yt == 1 and yp == 0])
    precision = tp / (tp + fp) if tp + fp > 0 else 1
    recall = tp / (tp + fn)
    return precision, recall

def precision_recall_curve(y_true, y_scores):
    thresholds = sorted(set(y_scores), reverse=True)
    precisions = []
    recalls = []
    for threshold in thresholds:
        precision, recall = precision_recall(y_true, y_scores, threshold)
        precisions.append(precision)
        recalls.append(recall)
    return precisions, recalls, thresholds

precision, recall, thresholds = precision_recall_curve(y_true, y_scores)

print("Precision:", precision)
print("Recall:", recall)
print("Thresholds:", thresholds)

# Plot the curve
plt.figure(figsize=(10, 6))
plt.plot(recall, precision, marker='.')
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.title('Precision-Recall Curve')
plt.show()
```

Labels: Bug