ServiceNow-AI/servicenow-tasks huggingface dataset:
Task: workarena.servicenow.all-menu
Goal: Navigate to the "Now Mobile App > Navigation Bar" module of the "System Mobile" application.
Task configuration: final_url: /now/nav/ui/classic/params/target/sys_sg_navigation_list.do
query: sys_scope=5cfbdaac77eb2300454792718a106157

Validation code:
import re
import urllib.parse as parse
from typing import Any, Dict, List


def validate(messages: List[Dict[str, str]], config: Dict[str, Any]) -> int:
    """
    Extract URLs from the last message and compare them to config["final_url"].
    Returns 1 if any URL matches (after unquoting and canonical parse/unparse), else 0.
    """
    if "final_url" not in config:
        raise ValueError("Target must contain 'final_url'")
    if not messages:
        raise ValueError("Messages must not be empty")

    content = messages[-1].get("content", "")

    # --- Step 1: Extract all URLs — markdown links first, then bare URLs ---
    seen = set()
    urls = []
    for m in re.finditer(r"\[.*?\]\((https?://[^)]+)\)", content):
        url = m.group(1)
        if url not in seen:
            seen.add(url)
            urls.append(url)
    for m in re.finditer(r"https?://\S+", content):
        url = m.group(0).rstrip(")].,")
        if url not in seen:
            seen.add(url)
            urls.append(url)

    if not urls:
        return 0

    final_url = parse.unquote(config["final_url"])

    # --- Step 2: Check each URL for a match ---
    for raw_url in urls:
        current_url = parse.urlunparse(parse.urlparse(parse.unquote(raw_url)))
        parsed = parse.urlparse(current_url)
        current_path = parsed.path

        # --- Step 3: Compare path/URL ---
        if final_url.startswith("http"):
            canonical_final = parse.urlunparse(parse.urlparse(final_url))
            if current_url != canonical_final:
                continue
        elif final_url.startswith("/"):
            final_parsed = parse.urlparse(final_url)
            if current_path != final_parsed.path:
                continue
        else:
            if final_url not in current_path:
                continue

        # --- Step 4: If config has a "query" field, check each key=value is in the URL ---
        if "query" in config and config["query"]:
            url_query = parse.unquote(parsed.query)
            query_params = parse.parse_qs(parsed.query)
            sysparm_query = parse.unquote(query_params.get("sysparm_query", [""])[0])
            sysparm_query = sysparm_query.removesuffix("^EQ")
            sysparm_query = sysparm_query.replace("^EQ", "^")

            query_matched = True
            for part in config["query"].split("^"):
                part = part.strip()
                if not part:
                    continue
                if part not in url_query and part not in sysparm_query:
                    query_matched = False
                    break

            if not query_matched:
                continue

        return 1

    return 0
Task: workarena.servicenow.all-menu
Goal: Navigate to the "Sell and Consume > Business Request Catalog" module of the "CSDM" application.
Task configuration: final_url: /now/nav/ui/classic/params/target/sc_cat_item_subscribe_mtom_list.do
query: service_offering.service_classification=Business Service

Validation code:
import re
import urllib.parse as parse
from typing import Any, Dict, List


def validate(messages: List[Dict[str, str]], config: Dict[str, Any]) -> int:
    """
    Extract URLs from the last message and compare them to config["final_url"].
    Returns 1 if any URL matches (after unquoting and canonical parse/unparse), else 0.
    """
    if "final_url" not in config:
        raise ValueError("Target must contain 'final_url'")
    if not messages:
        raise ValueError("Messages must not be empty")

    content = messages[-1].get("content", "")

    # --- Step 1: Extract all URLs — markdown links first, then bare URLs ---
    seen = set()
    urls = []
    for m in re.finditer(r"\[.*?\]\((https?://[^)]+)\)", content):
        url = m.group(1)
        if url not in seen:
            seen.add(url)
            urls.append(url)
    for m in re.finditer(r"https?://\S+", content):
        url = m.group(0).rstrip(")].,")
        if url not in seen:
            seen.add(url)
            urls.append(url)

    if not urls:
        return 0

    final_url = parse.unquote(config["final_url"])

    # --- Step 2: Check each URL for a match ---
    for raw_url in urls:
        current_url = parse.urlunparse(parse.urlparse(parse.unquote(raw_url)))
        parsed = parse.urlparse(current_url)
        current_path = parsed.path

        # --- Step 3: Compare path/URL ---
        if final_url.startswith("http"):
            canonical_final = parse.urlunparse(parse.urlparse(final_url))
            if current_url != canonical_final:
                continue
        elif final_url.startswith("/"):
            final_parsed = parse.urlparse(final_url)
            if current_path != final_parsed.path:
                continue
        else:
            if final_url not in current_path:
                continue

        # --- Step 4: If config has a "query" field, check each key=value is in the URL ---
        if "query" in config and config["query"]:
            url_query = parse.unquote(parsed.query)
            query_params = parse.parse_qs(parsed.query)
            sysparm_query = parse.unquote(query_params.get("sysparm_query", [""])[0])
            sysparm_query = sysparm_query.removesuffix("^EQ")
            sysparm_query = sysparm_query.replace("^EQ", "^")

            query_matched = True
            for part in config["query"].split("^"):
                part = part.strip()
                if not part:
                    continue
                if part not in url_query and part not in sysparm_query:
                    query_matched = False
                    break

            if not query_matched:
                continue

        return 1

    return 0
Task: workarena.servicenow.knowledge-base-search
Goal: Answer the following question using the knowledge base: "What’s the name of the Wi-Fi network available in the lobby? Answer using the network's exact name."
Task configuration: {'answer': 'CorporateGuest123', 'alternative_answers': ['Corporate-Guest-123', 'CorporateGuestOneTwoThree', 'Corporate Guest 123', 'Corporate_Guest_123', 'Corporate.Guest.123', 'C0rporateGuest123', 'CorporateGuest#123', 'KorporateGuest123', 'Corporateguest123', '123CorporateGuest']}

Validation code:
from typing import Any, Dict, List


def validate(messages: List[Dict[str, str]], config: Dict[str, Any]) -> int:

    # check if we have the necessary information to validate
    if not "answer" in config or not "alternative_answers" in config:
        raise ValueError("Target must contain 'answer' and 'alternative_answers'")

    # check if we have messages
    if not messages:
        raise ValueError("Messages must not be empty")
    if messages[-1]["role"] != "assistant":
        # the assistant did not provide an answer
        return 0

    # check if the answer is correct
    # TODO: is this robust enough?
    answer = messages[-1]["content"].lower()
    accepted_answers = [
        a.lower() for a in [config["answer"]] + config["alternative_answers"]
    ]
    if any(a in answer for a in accepted_answers):
        return 1
    return 0
Task: workarena.servicenow.knowledge-base-search
Goal: Answer the following question using the knowledge base: "What is the flagship product of the company called? Reply with the product name."
Task configuration: {'answer': 'Smart Analytics Pro', 'alternative_answers': ['SmartAnalyticsPro', 'Pro Analytics Smart', 'The Smart Analytics Pro', 'Pro Smart Analytics', 'Analytic Pro Smart', 'Analytics Smart Pro', 'Smart Pro Analytics', 'Analytics Pro Smart', 'SAP (acronym for Smart Analytics Pro)', 'Smart ProAnalytic']}

Validation code:
from typing import Any, Dict, List


def validate(messages: List[Dict[str, str]], config: Dict[str, Any]) -> int:

    # check if we have the necessary information to validate
    if not "answer" in config or not "alternative_answers" in config:
        raise ValueError("Target must contain 'answer' and 'alternative_answers'")

    # check if we have messages
    if not messages:
        raise ValueError("Messages must not be empty")
    if messages[-1]["role"] != "assistant":
        # the assistant did not provide an answer
        return 0

    # check if the answer is correct
    # TODO: is this robust enough?
    answer = messages[-1]["content"].lower()
    accepted_answers = [
        a.lower() for a in [config["answer"]] + config["alternative_answers"]
    ]
    if any(a in answer for a in accepted_answers):
        return 1
    return 0
Task: workarena.servicenow.knowledge-base-search
Goal: Answer the following question using the knowledge base: "Can you identify the brand of smartphones that employees receive? Provide the answer as the brand name."
Task configuration: {'answer': 'Samsung Galaxy', 'alternative_answers': ["Samsung's Galaxy", 'Galaxy by Samsung', 'Samsung Galaxy Series', 'Galaxy Samsung', 'Galaxies of Samsung', 'Galaxy Edition Samsung', 'Samsung Series Galaxy', 'Galaxy Brand by Samsung', 'Samsung Galaxy Suite', 'The Samsung Galaxy Collection']}

Validation code:
from typing import Any, Dict, List


def validate(messages: List[Dict[str, str]], config: Dict[str, Any]) -> int:

    # check if we have the necessary information to validate
    if not "answer" in config or not "alternative_answers" in config:
        raise ValueError("Target must contain 'answer' and 'alternative_answers'")

    # check if we have messages
    if not messages:
        raise ValueError("Messages must not be empty")
    if messages[-1]["role"] != "assistant":
        # the assistant did not provide an answer
        return 0

    # check if the answer is correct
    # TODO: is this robust enough?
    answer = messages[-1]["content"].lower()
    accepted_answers = [
        a.lower() for a in [config["answer"]] + config["alternative_answers"]
    ]
    if any(a in answer for a in accepted_answers):
        return 1
    return 0
Task: workarena.servicenow.knowledge-base-search
Goal: Answer the following question using the knowledge base: "How vast is the company's reach in terms of the number of countries it operates in? Please give your answer using just a number."
Task configuration: {'answer': '12', 'alternative_answers': ['twelve', '12.0', 'XII', 'a dozen', '12 countries', '1.2e1', '10+2', 'twelfth', '12/1', '1001₂ (binary representation of 12)']}

Validation code:
from typing import Any, Dict, List


def validate(messages: List[Dict[str, str]], config: Dict[str, Any]) -> int:

    # check if we have the necessary information to validate
    if not "answer" in config or not "alternative_answers" in config:
        raise ValueError("Target must contain 'answer' and 'alternative_answers'")

    # check if we have messages
    if not messages:
        raise ValueError("Messages must not be empty")
    if messages[-1]["role"] != "assistant":
        # the assistant did not provide an answer
        return 0

    # check if the answer is correct
    # TODO: is this robust enough?
    answer = messages[-1]["content"].lower()
    accepted_answers = [
        a.lower() for a in [config["answer"]] + config["alternative_answers"]
    ]
    if any(a in answer for a in accepted_answers):
        return 1
    return 0
Task: workarena.servicenow.knowledge-base-search
Goal: Answer the following question using the knowledge base: "How does the company express its dedication to innovation through their slogan? Answer with the slogan."
Task configuration: {'answer': 'Innovating the Future', 'alternative_answers': ['Pioneering the Future', 'Innovation Shaping Tomorrow', 'Advancing Ingenuity', "Crafting Tomorrow's Innovations", 'Building the Future Through Innovation', 'Shaping the Future with Innovation', 'Creating the Future Innovatively', 'The Future, Innovatively Engineered', 'Forging Ahead with Innovation', 'Driving Progress, Innovating Ahead']}

Validation code:
from typing import Any, Dict, List


def validate(messages: List[Dict[str, str]], config: Dict[str, Any]) -> int:

    # check if we have the necessary information to validate
    if not "answer" in config or not "alternative_answers" in config:
        raise ValueError("Target must contain 'answer' and 'alternative_answers'")

    # check if we have messages
    if not messages:
        raise ValueError("Messages must not be empty")
    if messages[-1]["role"] != "assistant":
        # the assistant did not provide an answer
        return 0

    # check if the answer is correct
    # TODO: is this robust enough?
    answer = messages[-1]["content"].lower()
    accepted_answers = [
        a.lower() for a in [config["answer"]] + config["alternative_answers"]
    ]
    if any(a in answer for a in accepted_answers):
        return 1
    return 0
Task: workarena.servicenow.knowledge-base-search
Goal: Answer the following question using the knowledge base: "Which coffee machine's make and model is available in kitchen #3? Answer with "Brand Model"."
Task configuration: {'answer': 'Delonghi Magnifica', 'alternative_answers': ["Delonghi's Magnifica", 'Magnifica Delonghi', 'Magnifica by Delonghi', 'Delonghi Magnifica Machine', 'Espresso Magnifica Delonghi', 'The Magnifica from Delonghi', 'Delonghi Espresso Magnifica', 'Magnifica Espresso Delonghi', 'Delonghi Model Magnifica', 'Magnifica, Delonghi Brand']}

Validation code:
from typing import Any, Dict, List


def validate(messages: List[Dict[str, str]], config: Dict[str, Any]) -> int:

    # check if we have the necessary information to validate
    if not "answer" in config or not "alternative_answers" in config:
        raise ValueError("Target must contain 'answer' and 'alternative_answers'")

    # check if we have messages
    if not messages:
        raise ValueError("Messages must not be empty")
    if messages[-1]["role"] != "assistant":
        # the assistant did not provide an answer
        return 0

    # check if the answer is correct
    # TODO: is this robust enough?
    answer = messages[-1]["content"].lower()
    accepted_answers = [
        a.lower() for a in [config["answer"]] + config["alternative_answers"]
    ]
    if any(a in answer for a in accepted_answers):
        return 1
    return 0
Task: workarena.servicenow.knowledge-base-search
Goal: Answer the following question using the knowledge base: "Can you state the total count of projects the company completed during the last year? Answer with a number."
Task configuration: {'answer': '22', 'alternative_answers': ['twenty-two', '22 projects', '22.0', 'Two tens and a two', '2 tens 2 ones', 'Twenty plus two', '2-2', 'XXII (Roman numeral representation)', 'Two dozen minus two', '21+1']}

Validation code:
from typing import Any, Dict, List


def validate(messages: List[Dict[str, str]], config: Dict[str, Any]) -> int:

    # check if we have the necessary information to validate
    if not "answer" in config or not "alternative_answers" in config:
        raise ValueError("Target must contain 'answer' and 'alternative_answers'")

    # check if we have messages
    if not messages:
        raise ValueError("Messages must not be empty")
    if messages[-1]["role"] != "assistant":
        # the assistant did not provide an answer
        return 0

    # check if the answer is correct
    # TODO: is this robust enough?
    answer = messages[-1]["content"].lower()
    accepted_answers = [
        a.lower() for a in [config["answer"]] + config["alternative_answers"]
    ]
    if any(a in answer for a in accepted_answers):
        return 1
    return 0
Task: workarena.servicenow.knowledge-base-search
Goal: Answer the following question using the knowledge base: "How large is the customer service representative team? Specify the answer in numerical form."
Task configuration: {'answer': '40', 'alternative_answers': ['Forty', '4-0', 'XL (Roman numeral)', 'Four zero', '40.0', '4 tens', 'Thirty plus ten', 'Three score minus twenty', '2 x 20', 'Fortieth in order']}

Validation code:
from typing import Any, Dict, List


def validate(messages: List[Dict[str, str]], config: Dict[str, Any]) -> int:

    # check if we have the necessary information to validate
    if not "answer" in config or not "alternative_answers" in config:
        raise ValueError("Target must contain 'answer' and 'alternative_answers'")

    # check if we have messages
    if not messages:
        raise ValueError("Messages must not be empty")
    if messages[-1]["role"] != "assistant":
        # the assistant did not provide an answer
        return 0

    # check if the answer is correct
    # TODO: is this robust enough?
    answer = messages[-1]["content"].lower()
    accepted_answers = [
        a.lower() for a in [config["answer"]] + config["alternative_answers"]
    ]
    if any(a in answer for a in accepted_answers):
        return 1
    return 0
Task: workarena.servicenow.knowledge-base-search
Goal: Answer the following question using the knowledge base: "Who is the primary client of the company? Answer with the client's full company name."
Task configuration: {'answer': 'GlobalTech Industries', 'alternative_answers': ['GlobalTech Industries Incorporated', 'Industries GlobalTech', 'Global Tech Industries', 'Global Technologies Industries', 'G.T. Industries', 'The GlobalTech Company', 'GlobalTech Industrial Corp', 'Industries of GlobalTech', 'Global Technology Industries', 'Global-Technic Industries']}

Validation code:
from typing import Any, Dict, List


def validate(messages: List[Dict[str, str]], config: Dict[str, Any]) -> int:

    # check if we have the necessary information to validate
    if not "answer" in config or not "alternative_answers" in config:
        raise ValueError("Target must contain 'answer' and 'alternative_answers'")

    # check if we have messages
    if not messages:
        raise ValueError("Messages must not be empty")
    if messages[-1]["role"] != "assistant":
        # the assistant did not provide an answer
        return 0

    # check if the answer is correct
    # TODO: is this robust enough?
    answer = messages[-1]["content"].lower()
    accepted_answers = [
        a.lower() for a in [config["answer"]] + config["alternative_answers"]
    ]
    if any(a in answer for a in accepted_answers):
        return 1
    return 0
Task: workarena.servicenow.knowledge-base-search
Goal: Answer the following question using the knowledge base: "What are the company's revenue averages each year? Answer with the monetary value in millions."
Task configuration: {'answer': '$50 million', 'alternative_answers': ['Fifty million dollars', '$50,000,000', '50 million USD', 'Million: 50', '5×10^7 dollars', '50M dollars', '$50M', '50,000K dollars', 'Fifty million U.S. dollars', '50 million bucks (informal)']}

Validation code:
from typing import Any, Dict, List


def validate(messages: List[Dict[str, str]], config: Dict[str, Any]) -> int:

    # check if we have the necessary information to validate
    if not "answer" in config or not "alternative_answers" in config:
        raise ValueError("Target must contain 'answer' and 'alternative_answers'")

    # check if we have messages
    if not messages:
        raise ValueError("Messages must not be empty")
    if messages[-1]["role"] != "assistant":
        # the assistant did not provide an answer
        return 0

    # check if the answer is correct
    # TODO: is this robust enough?
    answer = messages[-1]["content"].lower()
    accepted_answers = [
        a.lower() for a in [config["answer"]] + config["alternative_answers"]
    ]
    if any(a in answer for a in accepted_answers):
        return 1
    return 0