ServiceNow-AI/gitlab-tasks huggingface dataset:
Task: gitlab.show-command
Goal: Show me the command to clone Super_Awesome_Robot with SSH.
Task configuration:
display_repo: Super_Awesome_Robot
namespace: convexegg
project_name: super_awesome_robot
search_keyword: super_awesome_robot

Validation code:
from typing import Any, Dict, List
from urllib.parse import quote

import requests
import json
import os

def validate(messages: List[Dict[str, str]], config: Dict[str, Any]) -> int:

    if not messages:
        raise ValueError("Messages cannot be empty")
    last_message_content = messages[-1]["content"]

    # Check whether we have the necessary information to validate
    if not "search_keyword" in config:
        raise ValueError("Target must contain 'search_keyword'")

    query = quote(config["search_keyword"]).replace(".", "%2E")
    # find the project by keyword
    url = os.getenv("GITLAB_BASE_URL", "").rstrip("/") + "/api/v4/projects?search=" + query

    response = requests.request(
        method="GET",
        url=url,
        headers={
            "Content-Type": "application/json",
            "Accept": "application/json",
            **json.loads(os.getenv("GITLAB_EXTRA_HTTP_HEADERS", "{}")),
        },
    )
    if not response.status_code == 200:
        return 0
    result = response.json()
    if len(result) == 0:
        return 0
    # sort result by star_count to get most popular
    result.sort(key=lambda x: x["star_count"], reverse=True)
    repo = result[0]

    command = "git clone " + repo["ssh_url_to_repo"]

    # Potential issue: READMEs mention a different command to clone sometimes
    if command.lower() in last_message_content.lower():
        return 1

    return 0

if __name__ == "__main__":

    # simple example
    messages = [
        {
            "role": "assistant",
            "content": "the command is git clone ssh://git@gitlab.web-agent-evaluation-sites.snshared.com:2222/convexegg/Super_Awesome_Robot.git",
        }
    ]
    config = {
        "search_keyword": "super_awesome_robot",
    }
    assert validate(messages, config) == 1

    # more complex example since the query is fuzzy
    messages = [
        {
            "role": "assistant",
            "content": "the command is git clone ssh://git@gitlab.web-agent-evaluation-sites.snshared.com:2222/eriklindernoren/PyTorch-GAN.git",
        }
    ]
    config = {
        "search_keyword": "gan",
    }
    assert validate(messages, config) == 1

    # repo does not exist (should be root/metaseq)
    messages = [
        {
            "role": "assistant",
            "content": "the command is git clone ssh://git@gitlab.web-agent-evaluation-sites.snshared.com:2222/facebook/metaseq.git",
        }
    ]
    config = {
        "search_keyword": "metaseq",
    }
    assert validate(messages, config) == 0
Task: gitlab.show-command
Goal: Show me the command to clone ChatGPT with SSH.
Task configuration:
display_repo: ChatGPT
namespace: convexegg
project_name: chatgpt
search_keyword: chatgpt

Validation code:
from typing import Any, Dict, List
from urllib.parse import quote

import requests
import json
import os

def validate(messages: List[Dict[str, str]], config: Dict[str, Any]) -> int:

    if not messages:
        raise ValueError("Messages cannot be empty")
    last_message_content = messages[-1]["content"]

    # Check whether we have the necessary information to validate
    if not "search_keyword" in config:
        raise ValueError("Target must contain 'search_keyword'")

    query = quote(config["search_keyword"]).replace(".", "%2E")
    # find the project by keyword
    url = os.getenv("GITLAB_BASE_URL", "").rstrip("/") + "/api/v4/projects?search=" + query

    response = requests.request(
        method="GET",
        url=url,
        headers={
            "Content-Type": "application/json",
            "Accept": "application/json",
            **json.loads(os.getenv("GITLAB_EXTRA_HTTP_HEADERS", "{}")),
        },
    )
    if not response.status_code == 200:
        return 0
    result = response.json()
    if len(result) == 0:
        return 0
    # sort result by star_count to get most popular
    result.sort(key=lambda x: x["star_count"], reverse=True)
    repo = result[0]

    command = "git clone " + repo["ssh_url_to_repo"]

    # Potential issue: READMEs mention a different command to clone sometimes
    if command.lower() in last_message_content.lower():
        return 1

    return 0

if __name__ == "__main__":

    # simple example
    messages = [
        {
            "role": "assistant",
            "content": "the command is git clone ssh://git@gitlab.web-agent-evaluation-sites.snshared.com:2222/convexegg/Super_Awesome_Robot.git",
        }
    ]
    config = {
        "search_keyword": "super_awesome_robot",
    }
    assert validate(messages, config) == 1

    # more complex example since the query is fuzzy
    messages = [
        {
            "role": "assistant",
            "content": "the command is git clone ssh://git@gitlab.web-agent-evaluation-sites.snshared.com:2222/eriklindernoren/PyTorch-GAN.git",
        }
    ]
    config = {
        "search_keyword": "gan",
    }
    assert validate(messages, config) == 1

    # repo does not exist (should be root/metaseq)
    messages = [
        {
            "role": "assistant",
            "content": "the command is git clone ssh://git@gitlab.web-agent-evaluation-sites.snshared.com:2222/facebook/metaseq.git",
        }
    ]
    config = {
        "search_keyword": "metaseq",
    }
    assert validate(messages, config) == 0
Task: gitlab.show-command
Goal: Show me the command to clone metaseq with SSH.
Task configuration:
display_repo: metaseq
namespace: root
project_name: metaseq
search_keyword: metaseq

Validation code:
from typing import Any, Dict, List
from urllib.parse import quote

import requests
import json
import os

def validate(messages: List[Dict[str, str]], config: Dict[str, Any]) -> int:

    if not messages:
        raise ValueError("Messages cannot be empty")
    last_message_content = messages[-1]["content"]

    # Check whether we have the necessary information to validate
    if not "search_keyword" in config:
        raise ValueError("Target must contain 'search_keyword'")

    query = quote(config["search_keyword"]).replace(".", "%2E")
    # find the project by keyword
    url = os.getenv("GITLAB_BASE_URL", "").rstrip("/") + "/api/v4/projects?search=" + query

    response = requests.request(
        method="GET",
        url=url,
        headers={
            "Content-Type": "application/json",
            "Accept": "application/json",
            **json.loads(os.getenv("GITLAB_EXTRA_HTTP_HEADERS", "{}")),
        },
    )
    if not response.status_code == 200:
        return 0
    result = response.json()
    if len(result) == 0:
        return 0
    # sort result by star_count to get most popular
    result.sort(key=lambda x: x["star_count"], reverse=True)
    repo = result[0]

    command = "git clone " + repo["ssh_url_to_repo"]

    # Potential issue: READMEs mention a different command to clone sometimes
    if command.lower() in last_message_content.lower():
        return 1

    return 0

if __name__ == "__main__":

    # simple example
    messages = [
        {
            "role": "assistant",
            "content": "the command is git clone ssh://git@gitlab.web-agent-evaluation-sites.snshared.com:2222/convexegg/Super_Awesome_Robot.git",
        }
    ]
    config = {
        "search_keyword": "super_awesome_robot",
    }
    assert validate(messages, config) == 1

    # more complex example since the query is fuzzy
    messages = [
        {
            "role": "assistant",
            "content": "the command is git clone ssh://git@gitlab.web-agent-evaluation-sites.snshared.com:2222/eriklindernoren/PyTorch-GAN.git",
        }
    ]
    config = {
        "search_keyword": "gan",
    }
    assert validate(messages, config) == 1

    # repo does not exist (should be root/metaseq)
    messages = [
        {
            "role": "assistant",
            "content": "the command is git clone ssh://git@gitlab.web-agent-evaluation-sites.snshared.com:2222/facebook/metaseq.git",
        }
    ]
    config = {
        "search_keyword": "metaseq",
    }
    assert validate(messages, config) == 0
Task: gitlab.show-command
Goal: Show me the command to clone the best GAN python implementation with SSH.
Task configuration:
display_repo: the best GAN python implementation
namespace: eriklindernoren
project_name: PyTorch-GAN
search_keyword: gan

Validation code:
from typing import Any, Dict, List
from urllib.parse import quote

import requests
import json
import os

def validate(messages: List[Dict[str, str]], config: Dict[str, Any]) -> int:

    if not messages:
        raise ValueError("Messages cannot be empty")
    last_message_content = messages[-1]["content"]

    # Check whether we have the necessary information to validate
    if not "search_keyword" in config:
        raise ValueError("Target must contain 'search_keyword'")

    query = quote(config["search_keyword"]).replace(".", "%2E")
    # find the project by keyword
    url = os.getenv("GITLAB_BASE_URL", "").rstrip("/") + "/api/v4/projects?search=" + query

    response = requests.request(
        method="GET",
        url=url,
        headers={
            "Content-Type": "application/json",
            "Accept": "application/json",
            **json.loads(os.getenv("GITLAB_EXTRA_HTTP_HEADERS", "{}")),
        },
    )
    if not response.status_code == 200:
        return 0
    result = response.json()
    if len(result) == 0:
        return 0
    # sort result by star_count to get most popular
    result.sort(key=lambda x: x["star_count"], reverse=True)
    repo = result[0]

    command = "git clone " + repo["ssh_url_to_repo"]

    # Potential issue: READMEs mention a different command to clone sometimes
    if command.lower() in last_message_content.lower():
        return 1

    return 0

if __name__ == "__main__":

    # simple example
    messages = [
        {
            "role": "assistant",
            "content": "the command is git clone ssh://git@gitlab.web-agent-evaluation-sites.snshared.com:2222/convexegg/Super_Awesome_Robot.git",
        }
    ]
    config = {
        "search_keyword": "super_awesome_robot",
    }
    assert validate(messages, config) == 1

    # more complex example since the query is fuzzy
    messages = [
        {
            "role": "assistant",
            "content": "the command is git clone ssh://git@gitlab.web-agent-evaluation-sites.snshared.com:2222/eriklindernoren/PyTorch-GAN.git",
        }
    ]
    config = {
        "search_keyword": "gan",
    }
    assert validate(messages, config) == 1

    # repo does not exist (should be root/metaseq)
    messages = [
        {
            "role": "assistant",
            "content": "the command is git clone ssh://git@gitlab.web-agent-evaluation-sites.snshared.com:2222/facebook/metaseq.git",
        }
    ]
    config = {
        "search_keyword": "metaseq",
    }
    assert validate(messages, config) == 0
Task: gitlab.show-command
Goal: Show me the command to clone the most stared Covid location tracker with SSH.
Task configuration:
display_repo: the most stared Covid location tracker
namespace: covid19india
project_name: covid19india-react
search_keyword: covid

Validation code:
from typing import Any, Dict, List
from urllib.parse import quote

import requests
import json
import os

def validate(messages: List[Dict[str, str]], config: Dict[str, Any]) -> int:

    if not messages:
        raise ValueError("Messages cannot be empty")
    last_message_content = messages[-1]["content"]

    # Check whether we have the necessary information to validate
    if not "search_keyword" in config:
        raise ValueError("Target must contain 'search_keyword'")

    query = quote(config["search_keyword"]).replace(".", "%2E")
    # find the project by keyword
    url = os.getenv("GITLAB_BASE_URL", "").rstrip("/") + "/api/v4/projects?search=" + query

    response = requests.request(
        method="GET",
        url=url,
        headers={
            "Content-Type": "application/json",
            "Accept": "application/json",
            **json.loads(os.getenv("GITLAB_EXTRA_HTTP_HEADERS", "{}")),
        },
    )
    if not response.status_code == 200:
        return 0
    result = response.json()
    if len(result) == 0:
        return 0
    # sort result by star_count to get most popular
    result.sort(key=lambda x: x["star_count"], reverse=True)
    repo = result[0]

    command = "git clone " + repo["ssh_url_to_repo"]

    # Potential issue: READMEs mention a different command to clone sometimes
    if command.lower() in last_message_content.lower():
        return 1

    return 0

if __name__ == "__main__":

    # simple example
    messages = [
        {
            "role": "assistant",
            "content": "the command is git clone ssh://git@gitlab.web-agent-evaluation-sites.snshared.com:2222/convexegg/Super_Awesome_Robot.git",
        }
    ]
    config = {
        "search_keyword": "super_awesome_robot",
    }
    assert validate(messages, config) == 1

    # more complex example since the query is fuzzy
    messages = [
        {
            "role": "assistant",
            "content": "the command is git clone ssh://git@gitlab.web-agent-evaluation-sites.snshared.com:2222/eriklindernoren/PyTorch-GAN.git",
        }
    ]
    config = {
        "search_keyword": "gan",
    }
    assert validate(messages, config) == 1

    # repo does not exist (should be root/metaseq)
    messages = [
        {
            "role": "assistant",
            "content": "the command is git clone ssh://git@gitlab.web-agent-evaluation-sites.snshared.com:2222/facebook/metaseq.git",
        }
    ]
    config = {
        "search_keyword": "metaseq",
    }
    assert validate(messages, config) == 0
Task: gitlab.star-repo
Goal: Star the top five most stared repos in Gitlab
Task configuration:
number: five

Validation code:
from typing import Any, Dict, List

import requests
import json
import os

MOST_STARRED_REPOS = [
    "AndroidSlidingUpPanel",
    "create-react-app",
    "ffmpeg-python",
    "PHP_XLSXWriter",
    "AndroidAsync",
    "Pytorch-GAN",
    "administrate",
    "keycloak",
]

def validate(messages: List[Dict[str, str]], config: Dict[str, Any]) -> int:

    if not messages:
        raise ValueError("Messages cannot be empty")

    # Check whether we have the necessary information to validate
    if not "number" in config:
        raise ValueError("Target must contain 'number'")

    # map number (one, two, three, etc.) to int
    number = config["number"]
    if number == "one":
        number = 1
    elif number == "two":
        number = 2
    elif number == "three":
        number = 3
    elif number == "four":
        number = 4
    elif number == "five":
        number = 5
    elif number == "six":
        number = 6
    elif number == "seven":
        number = 7
    elif number == "eight":
        number = 8
    elif number == "nine":
        number = 9
    elif number == "ten":
        number = 10
    else:
        # TODO: better way of scaling this to *arbitrary* number?
        raise ValueError("Invalid number")

    # Build the url
    # TODO: maybe support all users?
    # TODO: maybe support pagination? or sort by stars to only get the most recent repos
    url = os.getenv("GITLAB_BASE_URL", "").rstrip("/") + "/api/v4/projects?starred=true"

    response = requests.request(
        method="GET",
        url=url,
        headers={
            "Content-Type": "application/json",
            "Accept": "application/json",
            **json.loads(os.getenv("GITLAB_EXTRA_HTTP_HEADERS", "{}")),
        },
    )

    if not response.status_code == 200:
        return 0
    result = response.json()

    # TODO: validate fields you need here
    repo_names = [repo["name"] for repo in result]
    for repo_name in MOST_STARRED_REPOS[:number]:
        if repo_name not in repo_names:
            return 0
    return 1
Task: gitlab.star-repo
Goal: Star the top eight most stared repos in Gitlab
Task configuration:
number: eight

Validation code:
from typing import Any, Dict, List

import requests
import json
import os

MOST_STARRED_REPOS = [
    "AndroidSlidingUpPanel",
    "create-react-app",
    "ffmpeg-python",
    "PHP_XLSXWriter",
    "AndroidAsync",
    "Pytorch-GAN",
    "administrate",
    "keycloak",
]

def validate(messages: List[Dict[str, str]], config: Dict[str, Any]) -> int:

    if not messages:
        raise ValueError("Messages cannot be empty")

    # Check whether we have the necessary information to validate
    if not "number" in config:
        raise ValueError("Target must contain 'number'")

    # map number (one, two, three, etc.) to int
    number = config["number"]
    if number == "one":
        number = 1
    elif number == "two":
        number = 2
    elif number == "three":
        number = 3
    elif number == "four":
        number = 4
    elif number == "five":
        number = 5
    elif number == "six":
        number = 6
    elif number == "seven":
        number = 7
    elif number == "eight":
        number = 8
    elif number == "nine":
        number = 9
    elif number == "ten":
        number = 10
    else:
        # TODO: better way of scaling this to *arbitrary* number?
        raise ValueError("Invalid number")

    # Build the url
    # TODO: maybe support all users?
    # TODO: maybe support pagination? or sort by stars to only get the most recent repos
    url = os.getenv("GITLAB_BASE_URL", "").rstrip("/") + "/api/v4/projects?starred=true"

    response = requests.request(
        method="GET",
        url=url,
        headers={
            "Content-Type": "application/json",
            "Accept": "application/json",
            **json.loads(os.getenv("GITLAB_EXTRA_HTTP_HEADERS", "{}")),
        },
    )

    if not response.status_code == 200:
        return 0
    result = response.json()

    # TODO: validate fields you need here
    repo_names = [repo["name"] for repo in result]
    for repo_name in MOST_STARRED_REPOS[:number]:
        if repo_name not in repo_names:
            return 0
    return 1
Task: gitlab.star-repo
Goal: Star the top four most stared repos in Gitlab
Task configuration:
number: four

Validation code:
from typing import Any, Dict, List

import requests
import json
import os

MOST_STARRED_REPOS = [
    "AndroidSlidingUpPanel",
    "create-react-app",
    "ffmpeg-python",
    "PHP_XLSXWriter",
    "AndroidAsync",
    "Pytorch-GAN",
    "administrate",
    "keycloak",
]

def validate(messages: List[Dict[str, str]], config: Dict[str, Any]) -> int:

    if not messages:
        raise ValueError("Messages cannot be empty")

    # Check whether we have the necessary information to validate
    if not "number" in config:
        raise ValueError("Target must contain 'number'")

    # map number (one, two, three, etc.) to int
    number = config["number"]
    if number == "one":
        number = 1
    elif number == "two":
        number = 2
    elif number == "three":
        number = 3
    elif number == "four":
        number = 4
    elif number == "five":
        number = 5
    elif number == "six":
        number = 6
    elif number == "seven":
        number = 7
    elif number == "eight":
        number = 8
    elif number == "nine":
        number = 9
    elif number == "ten":
        number = 10
    else:
        # TODO: better way of scaling this to *arbitrary* number?
        raise ValueError("Invalid number")

    # Build the url
    # TODO: maybe support all users?
    # TODO: maybe support pagination? or sort by stars to only get the most recent repos
    url = os.getenv("GITLAB_BASE_URL", "").rstrip("/") + "/api/v4/projects?starred=true"

    response = requests.request(
        method="GET",
        url=url,
        headers={
            "Content-Type": "application/json",
            "Accept": "application/json",
            **json.loads(os.getenv("GITLAB_EXTRA_HTTP_HEADERS", "{}")),
        },
    )

    if not response.status_code == 200:
        return 0
    result = response.json()

    # TODO: validate fields you need here
    repo_names = [repo["name"] for repo in result]
    for repo_name in MOST_STARRED_REPOS[:number]:
        if repo_name not in repo_names:
            return 0
    return 1
Task: gitlab.star-repo
Goal: Star the top three most stared repos in Gitlab
Task configuration:
number: three

Validation code:
from typing import Any, Dict, List

import requests
import json
import os

MOST_STARRED_REPOS = [
    "AndroidSlidingUpPanel",
    "create-react-app",
    "ffmpeg-python",
    "PHP_XLSXWriter",
    "AndroidAsync",
    "Pytorch-GAN",
    "administrate",
    "keycloak",
]

def validate(messages: List[Dict[str, str]], config: Dict[str, Any]) -> int:

    if not messages:
        raise ValueError("Messages cannot be empty")

    # Check whether we have the necessary information to validate
    if not "number" in config:
        raise ValueError("Target must contain 'number'")

    # map number (one, two, three, etc.) to int
    number = config["number"]
    if number == "one":
        number = 1
    elif number == "two":
        number = 2
    elif number == "three":
        number = 3
    elif number == "four":
        number = 4
    elif number == "five":
        number = 5
    elif number == "six":
        number = 6
    elif number == "seven":
        number = 7
    elif number == "eight":
        number = 8
    elif number == "nine":
        number = 9
    elif number == "ten":
        number = 10
    else:
        # TODO: better way of scaling this to *arbitrary* number?
        raise ValueError("Invalid number")

    # Build the url
    # TODO: maybe support all users?
    # TODO: maybe support pagination? or sort by stars to only get the most recent repos
    url = os.getenv("GITLAB_BASE_URL", "").rstrip("/") + "/api/v4/projects?starred=true"

    response = requests.request(
        method="GET",
        url=url,
        headers={
            "Content-Type": "application/json",
            "Accept": "application/json",
            **json.loads(os.getenv("GITLAB_EXTRA_HTTP_HEADERS", "{}")),
        },
    )

    if not response.status_code == 200:
        return 0
    result = response.json()

    # TODO: validate fields you need here
    repo_names = [repo["name"] for repo in result]
    for repo_name in MOST_STARRED_REPOS[:number]:
        if repo_name not in repo_names:
            return 0
    return 1