ServiceNow-AI/gitlab-tasks huggingface dataset:
Task: gitlab.list-pr
Goal: Checkout merge requests assigned to me
Task configuration:
query: assignee_username=byteblaze

Validation code:
import re
from typing import Any, Dict, List
from urllib import parse
import os
import json
import requests

def _get_headers():
    return {
        "Content-Type": "application/json",
        "Accept": "application/json",
        **json.loads(os.getenv("GITLAB_EXTRA_HTTP_HEADERS", "{}")),
    }

def _validate_via_url(content, config):
    """Original URL-based validation for browser agents."""
    all_urls = re.findall(r"https?://\S+", content)
    if not all_urls:
        return None

    expected_page_prefix = os.getenv("GITLAB_BASE_URL", "").rstrip("/") + "/dashboard/merge_requests"

    for url in all_urls:
        page_url = parse.unquote(url)
        if not page_url.startswith(expected_page_prefix):
            continue
        page_query = parse.urlparse(page_url).query
        if all(part in page_query for part in config["query"].split("&")):
            return 1
        return 0

    return None

def _validate_via_api(content, config):
    """API-based validation for terminal agents."""
    base_url = os.getenv("GITLAB_BASE_URL", "").rstrip("/")

    # Parse query params to build API request
    query_params = parse.parse_qs(config["query"])
    params = {"per_page": 20, "scope": "all"}

    for key, values in query_params.items():
        params[key] = values[0]

    api_url = base_url + "/api/v4/merge_requests"
    response = requests.get(url=api_url, headers=_get_headers(), params=params)
    if response.status_code != 200:
        return 0

    merge_requests = response.json()
    if not merge_requests:
        return 0

    # Check if the agent's message mentions at least one MR title
    content_lower = content.lower()
    for mr in merge_requests:
        if mr["title"].lower() in content_lower:
            return 1

    # Also check for MR IIDs
    for mr in merge_requests:
        if f"!{mr['iid']}" in content:
            return 1

    return 0

def validate(messages: List[Dict[str, str]], config: Dict[str, Any]) -> int:

    if not messages:
        raise ValueError("No messages provided")

    if not "query" in config:
        raise ValueError("No query provided")

    content = messages[-1].get("content", "")

    # Try URL-based validation first
    url_result = _validate_via_url(content, config)
    if url_result is not None:
        return url_result

    # Fall back to API-based validation
    return _validate_via_api(content, config)
Task: gitlab.list-pr
Goal: Checkout merge requests requiring my review
Task configuration:
query: reviewer_username=byteblaze

Validation code:
import re
from typing import Any, Dict, List
from urllib import parse
import os
import json
import requests

def _get_headers():
    return {
        "Content-Type": "application/json",
        "Accept": "application/json",
        **json.loads(os.getenv("GITLAB_EXTRA_HTTP_HEADERS", "{}")),
    }

def _validate_via_url(content, config):
    """Original URL-based validation for browser agents."""
    all_urls = re.findall(r"https?://\S+", content)
    if not all_urls:
        return None

    expected_page_prefix = os.getenv("GITLAB_BASE_URL", "").rstrip("/") + "/dashboard/merge_requests"

    for url in all_urls:
        page_url = parse.unquote(url)
        if not page_url.startswith(expected_page_prefix):
            continue
        page_query = parse.urlparse(page_url).query
        if all(part in page_query for part in config["query"].split("&")):
            return 1
        return 0

    return None

def _validate_via_api(content, config):
    """API-based validation for terminal agents."""
    base_url = os.getenv("GITLAB_BASE_URL", "").rstrip("/")

    # Parse query params to build API request
    query_params = parse.parse_qs(config["query"])
    params = {"per_page": 20, "scope": "all"}

    for key, values in query_params.items():
        params[key] = values[0]

    api_url = base_url + "/api/v4/merge_requests"
    response = requests.get(url=api_url, headers=_get_headers(), params=params)
    if response.status_code != 200:
        return 0

    merge_requests = response.json()
    if not merge_requests:
        return 0

    # Check if the agent's message mentions at least one MR title
    content_lower = content.lower()
    for mr in merge_requests:
        if mr["title"].lower() in content_lower:
            return 1

    # Also check for MR IIDs
    for mr in merge_requests:
        if f"!{mr['iid']}" in content:
            return 1

    return 0

def validate(messages: List[Dict[str, str]], config: Dict[str, Any]) -> int:

    if not messages:
        raise ValueError("No messages provided")

    if not "query" in config:
        raise ValueError("No query provided")

    content = messages[-1].get("content", "")

    # Try URL-based validation first
    url_result = _validate_via_url(content, config)
    if url_result is not None:
        return url_result

    # Fall back to API-based validation
    return _validate_via_api(content, config)
Task: gitlab.list-repos
Goal: Tell me the full names of the repositories where I made contributions and they got more than 100 stars?
Task configuration:
description: more than 100

Validation code:
import re
from typing import Any, Dict, List
from urllib import parse
import os

def validate(messages: List[Dict[str, str]], config: Dict[str, Any]) -> int:

    if not messages:
        raise ValueError("No messages provided")
    content = messages[-1].get("content", "")

    if "query" in config:
        # the task is to match the URL
        url_match = re.search(r"https?://\S+", content)
        if not url_match:
            return 0
        page_url = parse.unquote(url_match.group(0))

        expected_page_prefix = os.getenv("GITLAB_BASE_URL", "").rstrip("/") + "/explore"
        if not page_url.startswith(expected_page_prefix):
            return 0

        # extract query params
        page_query = parse.urlparse(page_url).query
        for part in config["query"].split("&"):
            if not part in page_query:
                return 0
        return 1

    elif "repos" in config:
        # HACK: we have hardcoded the list of repos that should be there.
        # ideally we use `contributed_projects` endpoint to get real-time data
        # but it seems this is not available within out self-deployment

        # repos may be a comma-separated string or a list
        repos = config["repos"]
        if isinstance(repos, str):
            repos = [r.strip() for r in repos.split(",") if r.strip()]
        for repo in repos:
            # we don't penalize for repos that are in the message but should not be there
            if repo not in content:
                return 0
        return 1

    return 0

if __name__ == "__main__":
    # explore repos
    messages = [
        {
            "role": "assistant",
            "content": "the url is: https://gitlab.web-agent-evaluation-sites.snshared.com/explore?visibility_level=20",
        }
    ]
    config = {
        "query": "visibility_level=20",
    }
    assert validate(messages, config) == 1

    # explore repos -> wrong query
    messages = [
        {
            "role": "assistant",
            "content": "the url is: https://gitlab.web-agent-evaluation-sites.snshared.com/explore?visibility_level=20",
        }
    ]
    config = {
        "query": "visibility_level=69",
    }
    assert validate(messages, config) == 0

    # list of repos
    messages = [
        {"role": "assistant", "content": "the repos are: design, a11yproject.com"}
    ]
    config = {
        "repos": ["design", "a11yproject.com"],
    }
    assert validate(messages, config) == 1

    config = {
        "repos": ["invalid_repo"],
    }
    assert validate(messages, config) == 0
Task: gitlab.list-repos
Goal: Tell me the full names of the repositories where I made contributions and they got the most stars?
Task configuration:
description: the most
repos: a11yproject.com, design

Validation code:
import re
from typing import Any, Dict, List
from urllib import parse
import os

def validate(messages: List[Dict[str, str]], config: Dict[str, Any]) -> int:

    if not messages:
        raise ValueError("No messages provided")
    content = messages[-1].get("content", "")

    if "query" in config:
        # the task is to match the URL
        url_match = re.search(r"https?://\S+", content)
        if not url_match:
            return 0
        page_url = parse.unquote(url_match.group(0))

        expected_page_prefix = os.getenv("GITLAB_BASE_URL", "").rstrip("/") + "/explore"
        if not page_url.startswith(expected_page_prefix):
            return 0

        # extract query params
        page_query = parse.urlparse(page_url).query
        for part in config["query"].split("&"):
            if not part in page_query:
                return 0
        return 1

    elif "repos" in config:
        # HACK: we have hardcoded the list of repos that should be there.
        # ideally we use `contributed_projects` endpoint to get real-time data
        # but it seems this is not available within out self-deployment

        # repos may be a comma-separated string or a list
        repos = config["repos"]
        if isinstance(repos, str):
            repos = [r.strip() for r in repos.split(",") if r.strip()]
        for repo in repos:
            # we don't penalize for repos that are in the message but should not be there
            if repo not in content:
                return 0
        return 1

    return 0

if __name__ == "__main__":
    # explore repos
    messages = [
        {
            "role": "assistant",
            "content": "the url is: https://gitlab.web-agent-evaluation-sites.snshared.com/explore?visibility_level=20",
        }
    ]
    config = {
        "query": "visibility_level=20",
    }
    assert validate(messages, config) == 1

    # explore repos -> wrong query
    messages = [
        {
            "role": "assistant",
            "content": "the url is: https://gitlab.web-agent-evaluation-sites.snshared.com/explore?visibility_level=20",
        }
    ]
    config = {
        "query": "visibility_level=69",
    }
    assert validate(messages, config) == 0

    # list of repos
    messages = [
        {"role": "assistant", "content": "the repos are: design, a11yproject.com"}
    ]
    config = {
        "repos": ["design", "a11yproject.com"],
    }
    assert validate(messages, config) == 1

    config = {
        "repos": ["invalid_repo"],
    }
    assert validate(messages, config) == 0
Task: gitlab.list-repos
Goal: Tell me the full names of the repositories where I made contributions and they got the least stars?
Task configuration:
description: the least
repos: cloud-to-butt, dotfiles, timeit, solarized-prism-theme, gimmiethat.space, remove-board-movement-events-from-the-github-issue-timeline

Validation code:
import re
from typing import Any, Dict, List
from urllib import parse
import os

def validate(messages: List[Dict[str, str]], config: Dict[str, Any]) -> int:

    if not messages:
        raise ValueError("No messages provided")
    content = messages[-1].get("content", "")

    if "query" in config:
        # the task is to match the URL
        url_match = re.search(r"https?://\S+", content)
        if not url_match:
            return 0
        page_url = parse.unquote(url_match.group(0))

        expected_page_prefix = os.getenv("GITLAB_BASE_URL", "").rstrip("/") + "/explore"
        if not page_url.startswith(expected_page_prefix):
            return 0

        # extract query params
        page_query = parse.urlparse(page_url).query
        for part in config["query"].split("&"):
            if not part in page_query:
                return 0
        return 1

    elif "repos" in config:
        # HACK: we have hardcoded the list of repos that should be there.
        # ideally we use `contributed_projects` endpoint to get real-time data
        # but it seems this is not available within out self-deployment

        # repos may be a comma-separated string or a list
        repos = config["repos"]
        if isinstance(repos, str):
            repos = [r.strip() for r in repos.split(",") if r.strip()]
        for repo in repos:
            # we don't penalize for repos that are in the message but should not be there
            if repo not in content:
                return 0
        return 1

    return 0

if __name__ == "__main__":
    # explore repos
    messages = [
        {
            "role": "assistant",
            "content": "the url is: https://gitlab.web-agent-evaluation-sites.snshared.com/explore?visibility_level=20",
        }
    ]
    config = {
        "query": "visibility_level=20",
    }
    assert validate(messages, config) == 1

    # explore repos -> wrong query
    messages = [
        {
            "role": "assistant",
            "content": "the url is: https://gitlab.web-agent-evaluation-sites.snshared.com/explore?visibility_level=20",
        }
    ]
    config = {
        "query": "visibility_level=69",
    }
    assert validate(messages, config) == 0

    # list of repos
    messages = [
        {"role": "assistant", "content": "the repos are: design, a11yproject.com"}
    ]
    config = {
        "repos": ["design", "a11yproject.com"],
    }
    assert validate(messages, config) == 1

    config = {
        "repos": ["invalid_repo"],
    }
    assert validate(messages, config) == 0
Task: gitlab.list-repos
Goal: Tell me the full names of the repositories where I made contributions and they got less than 5 stars?
Task configuration:
description: less than 5
repos: a11y-syntax-highlighting, a11y-webring.club, accessible-html-content-patterns, ericwbailey.website, cloud-to-butt, dotfiles, timeit, solarized-prism-theme, gimmiethat.space, remove-board-movement-events-from-the-github-issue-timeline

Validation code:
import re
from typing import Any, Dict, List
from urllib import parse
import os

def validate(messages: List[Dict[str, str]], config: Dict[str, Any]) -> int:

    if not messages:
        raise ValueError("No messages provided")
    content = messages[-1].get("content", "")

    if "query" in config:
        # the task is to match the URL
        url_match = re.search(r"https?://\S+", content)
        if not url_match:
            return 0
        page_url = parse.unquote(url_match.group(0))

        expected_page_prefix = os.getenv("GITLAB_BASE_URL", "").rstrip("/") + "/explore"
        if not page_url.startswith(expected_page_prefix):
            return 0

        # extract query params
        page_query = parse.urlparse(page_url).query
        for part in config["query"].split("&"):
            if not part in page_query:
                return 0
        return 1

    elif "repos" in config:
        # HACK: we have hardcoded the list of repos that should be there.
        # ideally we use `contributed_projects` endpoint to get real-time data
        # but it seems this is not available within out self-deployment

        # repos may be a comma-separated string or a list
        repos = config["repos"]
        if isinstance(repos, str):
            repos = [r.strip() for r in repos.split(",") if r.strip()]
        for repo in repos:
            # we don't penalize for repos that are in the message but should not be there
            if repo not in content:
                return 0
        return 1

    return 0

if __name__ == "__main__":
    # explore repos
    messages = [
        {
            "role": "assistant",
            "content": "the url is: https://gitlab.web-agent-evaluation-sites.snshared.com/explore?visibility_level=20",
        }
    ]
    config = {
        "query": "visibility_level=20",
    }
    assert validate(messages, config) == 1

    # explore repos -> wrong query
    messages = [
        {
            "role": "assistant",
            "content": "the url is: https://gitlab.web-agent-evaluation-sites.snshared.com/explore?visibility_level=20",
        }
    ]
    config = {
        "query": "visibility_level=69",
    }
    assert validate(messages, config) == 0

    # list of repos
    messages = [
        {"role": "assistant", "content": "the repos are: design, a11yproject.com"}
    ]
    config = {
        "repos": ["design", "a11yproject.com"],
    }
    assert validate(messages, config) == 1

    config = {
        "repos": ["invalid_repo"],
    }
    assert validate(messages, config) == 0
Task: gitlab.list-repos
Goal: Tell me the full names of the repositories where I made contributions and they got no stars?
Task configuration:
description: no
repos: cloud-to-butt, dotfiles, timeit, solarized-prism-theme, gimmiethat.space, remove-board-movement-events-from-the-github-issue-timeline

Validation code:
import re
from typing import Any, Dict, List
from urllib import parse
import os

def validate(messages: List[Dict[str, str]], config: Dict[str, Any]) -> int:

    if not messages:
        raise ValueError("No messages provided")
    content = messages[-1].get("content", "")

    if "query" in config:
        # the task is to match the URL
        url_match = re.search(r"https?://\S+", content)
        if not url_match:
            return 0
        page_url = parse.unquote(url_match.group(0))

        expected_page_prefix = os.getenv("GITLAB_BASE_URL", "").rstrip("/") + "/explore"
        if not page_url.startswith(expected_page_prefix):
            return 0

        # extract query params
        page_query = parse.urlparse(page_url).query
        for part in config["query"].split("&"):
            if not part in page_query:
                return 0
        return 1

    elif "repos" in config:
        # HACK: we have hardcoded the list of repos that should be there.
        # ideally we use `contributed_projects` endpoint to get real-time data
        # but it seems this is not available within out self-deployment

        # repos may be a comma-separated string or a list
        repos = config["repos"]
        if isinstance(repos, str):
            repos = [r.strip() for r in repos.split(",") if r.strip()]
        for repo in repos:
            # we don't penalize for repos that are in the message but should not be there
            if repo not in content:
                return 0
        return 1

    return 0

if __name__ == "__main__":
    # explore repos
    messages = [
        {
            "role": "assistant",
            "content": "the url is: https://gitlab.web-agent-evaluation-sites.snshared.com/explore?visibility_level=20",
        }
    ]
    config = {
        "query": "visibility_level=20",
    }
    assert validate(messages, config) == 1

    # explore repos -> wrong query
    messages = [
        {
            "role": "assistant",
            "content": "the url is: https://gitlab.web-agent-evaluation-sites.snshared.com/explore?visibility_level=20",
        }
    ]
    config = {
        "query": "visibility_level=69",
    }
    assert validate(messages, config) == 0

    # list of repos
    messages = [
        {"role": "assistant", "content": "the repos are: design, a11yproject.com"}
    ]
    config = {
        "repos": ["design", "a11yproject.com"],
    }
    assert validate(messages, config) == 1

    config = {
        "repos": ["invalid_repo"],
    }
    assert validate(messages, config) == 0
Task: gitlab.list-repos
Goal: See all public projects
Task configuration:
query: visibility_level=20

Validation code:
import re
from typing import Any, Dict, List
from urllib import parse
import os

def validate(messages: List[Dict[str, str]], config: Dict[str, Any]) -> int:

    if not messages:
        raise ValueError("No messages provided")
    content = messages[-1].get("content", "")

    if "query" in config:
        # the task is to match the URL
        url_match = re.search(r"https?://\S+", content)
        if not url_match:
            return 0
        page_url = parse.unquote(url_match.group(0))

        expected_page_prefix = os.getenv("GITLAB_BASE_URL", "").rstrip("/") + "/explore"
        if not page_url.startswith(expected_page_prefix):
            return 0

        # extract query params
        page_query = parse.urlparse(page_url).query
        for part in config["query"].split("&"):
            if not part in page_query:
                return 0
        return 1

    elif "repos" in config:
        # HACK: we have hardcoded the list of repos that should be there.
        # ideally we use `contributed_projects` endpoint to get real-time data
        # but it seems this is not available within out self-deployment

        # repos may be a comma-separated string or a list
        repos = config["repos"]
        if isinstance(repos, str):
            repos = [r.strip() for r in repos.split(",") if r.strip()]
        for repo in repos:
            # we don't penalize for repos that are in the message but should not be there
            if repo not in content:
                return 0
        return 1

    return 0

if __name__ == "__main__":
    # explore repos
    messages = [
        {
            "role": "assistant",
            "content": "the url is: https://gitlab.web-agent-evaluation-sites.snshared.com/explore?visibility_level=20",
        }
    ]
    config = {
        "query": "visibility_level=20",
    }
    assert validate(messages, config) == 1

    # explore repos -> wrong query
    messages = [
        {
            "role": "assistant",
            "content": "the url is: https://gitlab.web-agent-evaluation-sites.snshared.com/explore?visibility_level=20",
        }
    ]
    config = {
        "query": "visibility_level=69",
    }
    assert validate(messages, config) == 0

    # list of repos
    messages = [
        {"role": "assistant", "content": "the repos are: design, a11yproject.com"}
    ]
    config = {
        "repos": ["design", "a11yproject.com"],
    }
    assert validate(messages, config) == 1

    config = {
        "repos": ["invalid_repo"],
    }
    assert validate(messages, config) == 0