internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: Cloud-CV/EvalAI

Issue:
Send email to EvalAI admin when the EKS cluster creation is completed on AWS



Discussion:

Resolving patch:
diff --git a/apps/challenges/aws_utils.py b/apps/challenges/aws_utils.py
--- a/apps/challenges/aws_utils.py
+++ b/apps/challenges/aws_utils.py
@@ -10,7 +10,10 @@
 from django.core.files.temp import NamedTemporaryFile
 from http import HTTPStatus
 
-from .challenge_notification_util import construct_and_send_worker_start_mail
+from .challenge_notification_util import (
+    construct_and_send_worker_start_mail,
+    construct_and_send_eks_cluster_creation_mail,
+)
 
 from base.utils import get_boto3_client, send_email
 from evalai.celery import app
@@ -879,6 +882,7 @@ def create_eks_nodegroup(challenge, cluster_name):
         return response
     waiter = client.get_waiter("nodegroup_active")
     waiter.wait(clusterName=cluster_name, nodegroupName=nodegroup_name)
+    construct_and_send_eks_cluster_creation_mail(challenge)
 
 
 @app.task
diff --git a/apps/challenges/challenge_notification_util.py b/apps/challenges/challenge_notification_util.py
--- a/apps/challenges/challenge_notification_util.py
+++ b/apps/challenges/challenge_notification_util.py
@@ -34,3 +34,25 @@ def construct_and_send_worker_start_mail(challenge):
             template_id=template_id,
             template_data=template_data,
         )
+
+
+def construct_and_send_eks_cluster_creation_mail(challenge):
+    if settings.DEBUG:
+        return
+
+    template_data = {
+        "CHALLENGE_NAME": challenge.title,
+    }
+    if challenge.image:
+        template_data["CHALLENGE_IMAGE_URL"] = challenge.image.url
+
+    template_id = settings.SENDGRID_SETTINGS.get("TEMPLATES").get(
+        "CLUSTER_CREATION_TEMPLATE"
+    )
+
+    send_email(
+        sender=settings.CLOUDCV_TEAM_EMAIL,
+        recipient=settings.ADMIN_EMAIL,
+        template_id=template_id,
+        template_data=template_data,
+    )
diff --git a/settings/common.py b/settings/common.py
--- a/settings/common.py
+++ b/settings/common.py
@@ -312,15 +312,12 @@
     "TEMPLATES": {
         "CHALLENGE_INVITATION": "d-60825bcf014f4958bdb1b9173471d420",
         "CHALLENGE_APPROVAL_EMAIL": "d-45e0adc0597b4b60bd7c384aa903c488",
-        "WORKER_RESTART_EMAIL": "d-3d9a474a5e2b4ac4ad5a45ba9c0b84bd"
+        "WORKER_RESTART_EMAIL": "d-3d9a474a5e2b4ac4ad5a45ba9c0b84bd",
+        "CLUSTER_CREATION_TEMPLATE": "d-6de90fd760df4a41bb9bff1872eaab82",
     }
 }
 
 # EKS configs for Rl-worker
-EKS_CLUSTER_ROLE_ARN = os.environ.get(
-    "EKS_CLUSTER_ROLE_ARN"
-)
+EKS_CLUSTER_ROLE_ARN = os.environ.get("EKS_CLUSTER_ROLE_ARN")
 
-EKS_NODEGROUP_ROLE_ARN = os.environ.get(
-    "EKS_NODEGROUP_ROLE_ARN"
-)
+EKS_NODEGROUP_ROLE_ARN = os.environ.get("EKS_NODEGROUP_ROLE_ARN")


Test patch:
Repository: Cloud-CV/EvalAI

Issue:
Check if workers are running before triggering start_workers method after approving a challenge.
**Description:**
The workers may be launched through the manage tab before challenge approval, and in this case the code results in an exception being logged to sentry when executing start_workers. 

This shouldn't happen, and the start_workers method shouldn't be executed for this case.


Discussion:

Resolving patch:
diff --git a/apps/challenges/aws_utils.py b/apps/challenges/aws_utils.py
--- a/apps/challenges/aws_utils.py
+++ b/apps/challenges/aws_utils.py
@@ -1145,23 +1145,25 @@ def challenge_approval_callback(sender, instance, field_name, **kwargs):
 
     if not challenge.is_docker_based and challenge.remote_evaluation is False:
         if curr and not prev:
-            response = start_workers([challenge])
-            count, failures = response["count"], response["failures"]
-            if count != 1:
-                logger.error(
-                    "Worker for challenge {} couldn't start! Error: {}".format(
-                        challenge.id, failures[0]["message"]
+            if not challenge.workers:
+                response = start_workers([challenge])
+                count, failures = response["count"], response["failures"]
+                if not count:
+                    logger.error(
+                        "Worker for challenge {} couldn't start! Error: {}".format(
+                            challenge.id, failures[0]["message"]
+                        )
                     )
-                )
-            else:
-                construct_and_send_worker_start_mail(challenge)
+                else:
+                    construct_and_send_worker_start_mail(challenge)
 
         if prev and not curr:
-            response = delete_workers([challenge])
-            count, failures = response["count"], response["failures"]
-            if count != 1:
-                logger.error(
-                    "Worker for challenge {} couldn't be deleted! Error: {}".format(
-                        challenge.id, failures[0]["message"]
+            if challenge.workers:
+                response = delete_workers([challenge])
+                count, failures = response["count"], response["failures"]
+                if not count:
+                    logger.error(
+                        "Worker for challenge {} couldn't be deleted! Error: {}".format(
+                            challenge.id, failures[0]["message"]
+                        )
                     )
-                )


Test patch:
Repository: Cloud-CV/EvalAI

Issue:
Add check for docker based challenges in start/stop/restart/delete worker function
- Add a check in the `start_woker` method under `challenges/aws_utils.py` file to make sure that we are not starting workers for the code upload challenges from UI.



Discussion:

Resolving patch:
diff --git a/apps/challenges/aws_utils.py b/apps/challenges/aws_utils.py
--- a/apps/challenges/aws_utils.py
+++ b/apps/challenges/aws_utils.py
@@ -637,7 +637,12 @@ def start_workers(queryset):
     count = 0
     failures = []
     for challenge in queryset:
-        if (challenge.workers == 0) or (challenge.workers is None):
+        if challenge.is_docker_based:
+            response = "Sorry. This feature is not available for code upload/docker based challenges."
+            failures.append(
+                {"message": response, "challenge_pk": challenge.pk}
+            )
+        elif (challenge.workers == 0) or (challenge.workers is None):
             response = service_manager(
                 client, challenge=challenge, num_of_tasks=1
             )
@@ -686,7 +691,12 @@ def stop_workers(queryset):
     count = 0
     failures = []
     for challenge in queryset:
-        if (challenge.workers is not None) and (challenge.workers > 0):
+        if challenge.is_docker_based:
+            response = "Sorry. This feature is not available for code upload/docker based challenges."
+            failures.append(
+                {"message": response, "challenge_pk": challenge.pk}
+            )
+        elif (challenge.workers is not None) and (challenge.workers > 0):
             response = service_manager(
                 client, challenge=challenge, num_of_tasks=0
             )
@@ -788,7 +798,12 @@ def delete_workers(queryset):
     count = 0
     failures = []
     for challenge in queryset:
-        if challenge.workers is not None:
+        if challenge.is_docker_based:
+            response = "Sorry. This feature is not available for code upload/docker based challenges."
+            failures.append(
+                {"message": response, "challenge_pk": challenge.pk}
+            )
+        elif challenge.workers is not None:
             response = delete_service_by_challenge_pk(challenge=challenge)
             if response["ResponseMetadata"]["HTTPStatusCode"] != HTTPStatus.OK:
                 failures.append(
@@ -837,7 +852,12 @@ def restart_workers(queryset):
     count = 0
     failures = []
     for challenge in queryset:
-        if (challenge.workers is not None) and (challenge.workers > 0):
+        if challenge.is_docker_based:
+            response = "Sorry. This feature is not available for code upload/docker based challenges."
+            failures.append(
+                {"message": response, "challenge_pk": challenge.pk}
+            )
+        elif (challenge.workers is not None) and (challenge.workers > 0):
             response = service_manager(
                 client,
                 challenge=challenge,


Test patch:
Repository: Cloud-CV/EvalAI

Issue:
Delete log group from CloudWatch when a challenge worker is deleted.



Discussion:

Resolving patch:
diff --git a/apps/challenges/aws_utils.py b/apps/challenges/aws_utils.py
--- a/apps/challenges/aws_utils.py
+++ b/apps/challenges/aws_utils.py
@@ -374,9 +374,12 @@ def register_task_def_by_challenge_pk(client, queue_name, challenge):
     if execution_role_arn:
         if challenge.is_docker_based:
             from .models import ChallengeEvaluationCluster
+
             # Cluster detail to be used by code-upload-worker
             try:
-                cluster_details = ChallengeEvaluationCluster.objects.get(challenge=challenge)
+                cluster_details = ChallengeEvaluationCluster.objects.get(
+                    challenge=challenge
+                )
                 cluster_name = cluster_details.name
                 cluster_endpoint = cluster_details.cluster_endpoint
                 cluster_certificate = cluster_details.cluster_ssl
@@ -796,6 +799,8 @@ def delete_workers(queryset):
                 )
                 continue
             count += 1
+            log_group_name = "challenge-pk-{}-workers".format(challenge.pk)
+            delete_log_group(log_group_name)
         else:
             response = "Please select challenges with active workers only."
             failures.append(
@@ -963,6 +968,17 @@ def get_logs_from_cloudwatch(
     return logs
 
 
+def delete_log_group(log_group_name):
+    if settings.DEBUG:
+        pass
+    else:
+        try:
+            client = get_boto3_client("logs", aws_keys)
+            client.delete_log_group(logGroupName=log_group_name)
+        except Exception as e:
+            logger.exception(e)
+
+
 @app.task
 def create_eks_nodegroup(challenge, cluster_name):
     """


Test patch:
Repository: Cloud-CV/EvalAI

Issue:
Handle the case where log group hasn't been created yet when displaying logs.
It takes about 1.5 minutes for the logs group to get created on Cloudwatch once the workers are started for the first time. And if the host tries to view the logs in this period, they'll get an error message.

**ToDo:**

- [x] Change the message to something more appropriate, or wait for the creation of log group before sending any message back to the frontend.



Discussion:

Resolving patch:
diff --git a/apps/challenges/aws_utils.py b/apps/challenges/aws_utils.py
--- a/apps/challenges/aws_utils.py
+++ b/apps/challenges/aws_utils.py
@@ -961,6 +961,9 @@ def get_logs_from_cloudwatch(
             for event in response["events"]:
                 logs.append(event["message"])
         except Exception as e:
+            if e.response["Error"]["Code"] == "ResourceNotFoundException":
+                return logs
+
             logger.exception(e)
             return [
                 f"There is an error in displaying logs. Please find the full error traceback here {e}"


Test patch:
Repository: Cloud-CV/EvalAI

Issue:
Dedup environment variables for EvalAI django server endpoint
### Description

We are using multiple environment variables for EvalAI django server endpoint, `API_HOST_URL` `EVALAI_API_SERVER` `DJANGO_SERVER`. Ideally we should only have a common environment variable `DJANGO_SERVER` for the endpoint which should be used by all the features.

Refer: 
1. [dev/docker.env](https://github.com/Cloud-CV/EvalAI/blob/master/docker/dev/docker.env)
2. [prod/docker_staging.env](https://github.com/Cloud-CV/EvalAI/blob/master/docker/prod/docker_staging.env)
3. [prod/docker_production.env](https://github.com/Cloud-CV/EvalAI/blob/master/docker/prod/docker_production.env)


Discussion:

Resolving patch:
diff --git a/apps/challenges/views.py b/apps/challenges/views.py
--- a/apps/challenges/views.py
+++ b/apps/challenges/views.py
@@ -731,7 +731,7 @@ def create_challenge_using_zip_file(request, challenge_host_team_pk):
             )
 
         if settings.DEBUG or settings.TEST:
-            template_zip_s3_url = settings.API_HOST_URL + challenge_template.template_file.url
+            template_zip_s3_url = settings.EVALAI_API_SERVER + challenge_template.template_file.url
         else:
             template_zip_s3_url = challenge_template.template_file.url
 
diff --git a/settings/common.py b/settings/common.py
--- a/settings/common.py
+++ b/settings/common.py
@@ -33,7 +33,7 @@
 
 ALLOWED_HOSTS = []
 
-API_HOST_URL = os.environ.get("API_HOST_URL", "http://localhost:8000")
+EVALAI_API_SERVER = os.environ.get("EVALAI_API_SERVER", "http://localhost:8000")
 
 # Application definition
 


Test patch:
Repository: Cloud-CV/EvalAI

Issue:
[Frontend V2] Fix the media assets endpoint
### Description

We recently moved to `https://evalai.s3.amazonaws.com/` endpoint for our media assets. Frontend v2 is still using `https://staging-evalai.s3.amazonaws.com/` endpoint. We should switch to new enpdoint in frontend v2.


Discussion:
Can I work on it? @Ram81 
@gautamjajoo sure, go ahead

Resolving patch:
diff --git a/settings/staging.py b/settings/staging.py
--- a/settings/staging.py
+++ b/settings/staging.py
@@ -5,6 +5,7 @@
 CORS_ORIGIN_ALLOW_ALL = False
 
 CORS_ORIGIN_WHITELIST = (
+    "https://evalai.s3.amazonaws.com",
     "https://staging-evalai.s3.amazonaws.com",
     "https://staging.eval.ai",
     "https://beta-staging.eval.ai",


Test patch:
Repository: Cloud-CV/EvalAI

Issue:
Phase is in random order in leaderboard
Phases are in random order in the leaderboard (undesirable), unlike the sorted behavior in the phases tab.

This is due to the backend API [`challenge_phase_split_list`](https://github.com/Cloud-CV/EvalAI/blob/master/apps/challenges/views.py#L824-L850) not ordering the filtered results by `pk`, unlike other APIs like [`challenge_phase_list`](https://github.com/Cloud-CV/EvalAI/blob/master/apps/challenges/views.py#L717) which are doing the sorting.

Will create a pull request to fix this.


Discussion:

Resolving patch:
diff --git a/apps/challenges/views.py b/apps/challenges/views.py
--- a/apps/challenges/views.py
+++ b/apps/challenges/views.py
@@ -833,7 +833,7 @@ def challenge_phase_split_list(request, challenge_pk):
 
     challenge_phase_split = ChallengePhaseSplit.objects.filter(
         challenge_phase__challenge=challenge
-    )
+    ).order_by("pk")
 
     # Check if user is a challenge host or participant
     challenge_host = is_user_a_host_of_challenge(request.user, challenge_pk)


Test patch:
Repository: Cloud-CV/EvalAI

Issue:
Duplicate Challenge Phase split
When creating a challenge phase split, we are able to create challenge phase split with same dataset split id,  challenge phase id and leaderboard id, this shouldn't be the case as it creates a duplicate entry for no use.

To maintain a well-organized and clear competition environment, each challenge phase split should have a unique combination of identifiers:

1. Dataset split ID: This identifier designates a specific split of the dataset, such as "train," "validation," or "test."
2. Challenge phase ID: This identifier represents the particular challenge phase, such as "Phase 1" or "Phase 2."
3. Leaderboard ID: This identifier is used to differentiate between multiple leaderboards, for instance, "Leaderboard 1" or "Leaderboard 2."


Discussion:

Resolving patch:
diff --git a/apps/challenges/challenge_config_utils.py b/apps/challenges/challenge_config_utils.py
--- a/apps/challenges/challenge_config_utils.py
+++ b/apps/challenges/challenge_config_utils.py
@@ -279,6 +279,7 @@ def get_value_from_field(data, base_location, field_name):
     "start_date_greater_than_end_date_challenge_phase": "ERROR: Start date cannot be greater than end date in challenge phase {}.",
     "extra_tags": "ERROR: Tags are limited to 4. Please remove extra tags then try again!",
     "wrong_domain": "ERROR: Domain name is incorrect. Please enter correct domain name then try again!",
+    "duplicate_combinations_in_challenge_phase_splits": "ERROR: Duplicate combinations of leaderboard_id {}, challenge_phase_id {} and dataset_split_id {} found in challenge phase splits.",
 }
 
 
@@ -795,6 +796,34 @@ def validate_challenge_phase_splits(self, current_phase_split_ids):
         challenge_phase_splits = self.yaml_file_data.get(
             "challenge_phase_splits"
         )
+        # Check for duplicate combinations
+        challenge_phase_split_set = set()
+        duplicates_found = False
+        total_duplicates = []
+
+        for data in challenge_phase_splits:
+            combination = (
+                data["leaderboard_id"],
+                data["challenge_phase_id"],
+                data["dataset_split_id"],
+            )
+            if combination in challenge_phase_split_set:
+                duplicates_found = True
+                total_duplicates += [combination]
+            else:
+                challenge_phase_split_set.add(combination)
+
+        if duplicates_found:
+            message = self.error_messages_dict[
+                "duplicate_combinations_in_challenge_phase_splits"
+            ]
+            self.error_messages.append(message)
+            for combination in total_duplicates:
+                message = self.error_messages_dict[
+                    "duplicate_combinations_in_challenge_phase_splits"
+                ].format(combination[0], combination[1], combination[2])
+                self.error_messages.append(message)
+
         challenge_phase_split_uuids = []
         if challenge_phase_splits:
             phase_split = 1


Test patch:
Repository: Cloud-CV/EvalAI

Issue:
Problem with get submission metrics API
There is an issue with `get_submission_metrics` API that it is open to all. I think it should be only restricted to admins now that we have a `is_user_a_staff` utility.




Discussion:

Resolving patch:
diff --git a/apps/challenges/views.py b/apps/challenges/views.py
--- a/apps/challenges/views.py
+++ b/apps/challenges/views.py
@@ -53,6 +53,7 @@
     paginated_queryset,
     send_email,
     send_slack_notification,
+    is_user_a_staff,
 )
 from challenges.utils import (
     complete_s3_multipart_file_upload,
@@ -711,6 +712,9 @@ def get_all_challenges_submission_metrics(request):
     """
     Returns the submission metrics for all challenges and their phases
     """
+    if not is_user_a_staff(request.user):
+        response_data = {"error": "Sorry, you are not authorized to make this request"}
+        return Response(response_data, status=status.HTTP_403_FORBIDDEN)
     challenges = Challenge.objects.all()
     submission_metrics = {}
 
@@ -729,7 +733,7 @@ def get_all_challenges_submission_metrics(request):
 
         submission_metrics[challenge_id] = challenge_metrics
 
-    return Response(submission_metrics)
+    return Response(submission_metrics, status=status.HTTP_200_OK)
 
 
 @api_view(["GET"])


Test patch:
diff --git a/tests/unit/challenges/test_views.py b/tests/unit/challenges/test_views.py
--- a/tests/unit/challenges/test_views.py
+++ b/tests/unit/challenges/test_views.py
@@ -4198,6 +4198,20 @@ def test_get_all_submissions_when_user_is_neither_host_nor_participant_of_challe
         self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
 
     def test_get_all_challenges_submission_metrics(self):
+
+        self.user8 = User.objects.create(
+            username="admin_test",
+            password="admin@123",
+            is_staff=True,
+        )
+
+        EmailAddress.objects.create(
+            user=self.user8,
+            email="user8@test.com",
+            primary=True,
+            verified=True,
+        )
+
         self.maxDiff = None
 
         url = reverse_lazy("challenges:get_all_challenges_submission_metrics")
@@ -4227,6 +4241,7 @@ def test_get_all_challenges_submission_metrics(self):
             }
         }
 
+        self.client.force_authenticate(user=self.user8)
         response = self.client.get(url)
 
         self.assertEqual(response.status_code, status.HTTP_200_OK)
Repository: Cloud-CV/EvalAI

Issue:
Leaderboard id replaced in challenge phase split
Currently when a challenge host tries to update his challenge using the config and he has more than one leaderboard then the other leaderboards are removed and only one remains. This might be due to the https://github.com/Cloud-CV/EvalAI/blob/master/apps/challenges/views.py#L3890.


Discussion:

Resolving patch:
diff --git a/apps/challenges/views.py b/apps/challenges/views.py
--- a/apps/challenges/views.py
+++ b/apps/challenges/views.py
@@ -4030,6 +4030,7 @@ def create_or_update_github_challenge(request, challenge_host_team_pk):
                     challenge_phase_split_qs = ChallengePhaseSplit.objects.filter(
                         challenge_phase__pk=challenge_phase,
                         dataset_split__pk=dataset_split,
+                        leaderboard__pk=leaderboard,
                     )
                     if challenge_phase_split_qs:
                         challenge_phase_split = challenge_phase_split_qs.first()


Test patch:
Repository: Cloud-CV/EvalAI

Issue:
[BugFix] Allow staff to access leaderboard data



Discussion:

Resolving patch:
diff --git a/apps/jobs/utils.py b/apps/jobs/utils.py
--- a/apps/jobs/utils.py
+++ b/apps/jobs/utils.py
@@ -14,7 +14,7 @@
 
 from base.utils import get_model_object, suppress_autotime
 from challenges.utils import get_challenge_model, get_challenge_phase_model
-from hosts.utils import is_user_a_host_of_challenge
+from base.utils import is_user_a_staff_or_host
 from participants.utils import get_participant_team_id_of_user_for_a_challenge
 
 from .constants import submission_status_to_exclude
@@ -342,14 +342,14 @@ def calculate_distinct_sorted_leaderboard_data(
         [] if not is_challenge_phase_public else challenge_hosts_emails
     )
 
-    challenge_host_user = is_user_a_host_of_challenge(user, challenge_obj.pk)
+    challenge_host_or_staff = is_user_a_staff_or_host(user, challenge_obj)
 
     all_banned_email_ids = challenge_obj.banned_email_ids
 
     # Check if challenge phase leaderboard is public for participant user or not
     if (
         challenge_phase_split.visibility != ChallengePhaseSplit.PUBLIC
-        and not challenge_host_user
+        and not challenge_host_or_staff
     ):
         response_data = {"error": "Sorry, the leaderboard is not public!"}
         return response_data, status.HTTP_400_BAD_REQUEST
diff --git a/apps/jobs/views.py b/apps/jobs/views.py
--- a/apps/jobs/views.py
+++ b/apps/jobs/views.py
@@ -801,7 +801,7 @@ def get_all_entries_on_public_leaderboard(request, challenge_phase_split_pk):
     challenge_obj = challenge_phase_split.challenge_phase.challenge
 
     # Allow access only to challenge host
-    if not is_user_a_staff_or_host(request.user, challenge_obj.pk):
+    if not is_user_a_staff_or_host(request.user, challenge_obj):
         response_data = {
             "error": "Sorry, you are not authorized to make this request!"
         }


Test patch: