Repository: instructlab/instructlab

Issue:
Unit tests for knowledge cli changes 



Discussion:

Patch:
diff --git a/cli/generator/generate_data.py b/cli/generator/generate_data.py
index 3c54f8ef42..1ffed19bda 100755
--- a/cli/generator/generate_data.py
+++ b/cli/generator/generate_data.py
@@ -1,10 +1,9 @@
 # Standard
 from datetime import datetime
-from functools import cache, partial
-from logging import Logger
+from functools import partial
 from multiprocessing import Pool
 from pathlib import Path
-from typing import Any, Mapping, Optional
+from typing import Optional
 import json
 import os
 import random
@@ -17,11 +16,11 @@
 from rouge_score import rouge_scorer
 import click
 import tqdm
-import yaml
 
 # Local
-from ..utils import TAXONOMY_FOLDERS, chunk_document, get_documents, get_taxonomy_diff
+from ..utils import chunk_document, read_taxonomy
 from . import utils
+from .utils import GenerateException
 
 DEFAULT_PROMPT_TEMPLATE = """\
 You are asked to come up with a set of 5 diverse task instructions under {{taxonomy}}{{" for the task \\"%s\\""|format(task_description)  if task_description}}. These task instructions will be given to a GPT model and we will evaluate the GPT model for completing the instructions.
@@ -57,14 +56,6 @@
 {% endif -%}
 """
 
-DEFAULT_YAML_RULES = """\
-extends: relaxed
-
-rules:
-  line-length:
-    max: 120
-"""
-
 _WORD_DENYLIST = [
     "image",
     "images",
@@ -87,10 +78,6 @@
 ]
 
 
-class GenerateException(Exception):
-    """An exception raised during generate step."""
-
-
 def check_prompt_file(prompt_file_path):
     """Check for prompt file."""
     try:
@@ -230,17 +217,6 @@ def find_word_in_string(w, s):
     return re.compile(r"\b({0})\b".format(w), flags=re.IGNORECASE).search(s)
 
 
-def get_version(contents: Mapping) -> int:
-    version = contents.get("version", 1)
-    if not isinstance(version, int):
-        # schema validation will complain about the type
-        try:
-            version = int(version)
-        except ValueError:
-            version = 1  # fallback to version 1
-    return version
-
-
 def get_instructions_from_model(
     logger,
     request_idx,
@@ -547,226 +523,3 @@ def unescape(s):
         )
     generate_duration = time.time() - generate_start
     logger.info(f"Generation took {generate_duration:.2f}s")
-
-
-@cache
-def _load_schema(path: "importlib.resources.abc.Traversable") -> "referencing.Resource":
-    """Load the schema from the path into a Resource object.
-
-    Args:
-        path (Traversable): Path to the schema to be loaded.
-
-    Raises:
-        NoSuchResource: If the resource cannot be loaded.
-
-    Returns:
-        Resource: A Resource containing the requested schema.
-    """
-    # pylint: disable=C0415
-    # Third Party
-    from referencing import Resource
-    from referencing.exceptions import NoSuchResource
-    from referencing.jsonschema import DRAFT202012
-
-    try:
-        contents = json.loads(path.read_text(encoding="utf-8"))
-        resource = Resource.from_contents(
-            contents=contents, default_specification=DRAFT202012
-        )
-    except Exception as e:
-        raise NoSuchResource(ref=str(path)) from e
-    return resource
-
-
-def validate_yaml(
-    logger: Logger, contents: Mapping[str, Any], taxonomy_path: Path
-) -> int:
-    """Validate the parsed yaml document using the taxonomy path to
-    determine the proper schema.
-
-    Args:
-        logger (Logger): The logger for errors/warnings.
-        contents (Mapping): The parsed yaml document to validate against the schema.
-        taxonomy_path (Path): Relative path of the taxonomy yaml document where the
-        first element is the schema to use.
-
-    Returns:
-        int: The number of errors found during validation.
-        Messages for each error have been logged.
-    """
-    # pylint: disable=C0415
-    # Standard
-    from importlib import resources
-
-    # Third Party
-    from jsonschema.protocols import Validator
-    from jsonschema.validators import validator_for
-    from referencing import Registry, Resource
-    from referencing.exceptions import NoSuchResource
-    from referencing.typing import URI
-
-    errors = 0
-    version = get_version(contents)
-    schemas_path = resources.files("cli").joinpath(f"schema/v{version}")
-
-    def retrieve(uri: URI) -> Resource:
-        path = schemas_path.joinpath(uri)
-        return _load_schema(path)
-
-    schema_name = taxonomy_path.parts[0]
-    if schema_name not in TAXONOMY_FOLDERS:
-        schema_name = "knowledge" if "document" in contents else "compositional_skills"
-        logger.info(
-            f"Cannot determine schema name from path {taxonomy_path}. Using {schema_name} schema."
-        )
-
-    try:
-        schema_resource = retrieve(f"{schema_name}.json")
-        schema = schema_resource.contents
-        validator_cls = validator_for(schema)
-        validator: Validator = validator_cls(
-            schema, registry=Registry(retrieve=retrieve)
-        )
-
-        for error in validator.iter_errors(contents):
-            errors += 1
-            error_path = error.json_path[1:]
-            if not error_path:
-                error_path = "."
-            logger.error(
-                f"Validation error in {taxonomy_path} on {error_path}: {error.message}"
-            )
-    except NoSuchResource as e:
-        cause = e.__cause__ if e.__cause__ is not None else e
-        errors += 1
-        logger.error(f"Cannot load schema file {e.ref}. {cause}")
-
-    return errors
-
-
-# pylint: disable=broad-exception-caught
-def read_taxonomy_file(logger, file_path, yaml_rules: Optional[str] = None):
-    # pylint: disable=C0415
-    # Third Party
-    from yamllint import linter
-    from yamllint.config import YamlLintConfig
-
-    seed_instruction_data = []
-    warnings = 0
-    errors = 0
-    file_path = Path(file_path).resolve()
-    # file should end with ".yaml" explicitly
-    if file_path.suffix != ".yaml":
-        logger.warn(f"Skipping {file_path}! Use lowercase '.yaml' extension instead.")
-        warnings += 1
-        return None, warnings, errors
-    for i in range(len(file_path.parts) - 1, -1, -1):
-        if file_path.parts[i] in TAXONOMY_FOLDERS:
-            taxonomy_path = Path(*file_path.parts[i:])
-            break
-    else:
-        taxonomy_path = file_path
-    # read file if extension is correct
-    try:
-        with open(file_path, "r", encoding="utf-8") as file:
-            # do general YAML linting if specified
-            if yaml_rules is not None:
-                try:
-                    yaml_config = YamlLintConfig(file=yaml_rules)
-                except FileNotFoundError:
-                    logger.debug(f"Cannot find {yaml_rules}. Using default rules.")
-                    yaml_config = YamlLintConfig(DEFAULT_YAML_RULES)
-            else:
-                yaml_config = YamlLintConfig(DEFAULT_YAML_RULES)
-            linter_problems = list(linter.run(file, yaml_config))
-            if linter_problems:
-                lint_messages = [f"Problems found in file {file.name}"]
-                for p in linter_problems:
-                    errors += 1
-                    loc = f"{p.line}:{p.column}"
-                    desc = f"{p.desc} ({p.rule})" if p.rule else p.desc
-                    lint_messages.append(f"  {loc:<9} {p.level:<9} {desc}")
-                logger.error("\n".join(lint_messages))
-                return None, warnings, errors
-        # do more explict checking of file contents
-        with open(file_path, "r", encoding="utf-8") as file:
-            contents = yaml.safe_load(file)
-            if not contents:
-                logger.warn(f"Skipping {file_path} because it is empty!")
-                warnings += 1
-                return None, warnings, errors
-            validation_errors = validate_yaml(logger, contents, taxonomy_path)
-            if validation_errors:
-                errors += validation_errors
-                return None, warnings, errors
-
-            # get seed instruction data
-            tax_path = "->".join(taxonomy_path.parent.parts)
-            task_description = contents.get("task_description")
-            documents = contents.get("document")
-            if documents:
-                documents = get_documents(documents)
-                logger.debug("Content from git repo fetched")
-
-            for seed_example in contents.get("seed_examples"):
-                question = seed_example.get("question")
-                answer = seed_example.get("answer")
-                context = seed_example.get("context", "")
-                seed_instruction_data.append(
-                    {
-                        "instruction": question,
-                        "input": context,
-                        "output": answer,
-                        "taxonomy_path": tax_path,
-                        "task_description": task_description,
-                        "document": documents,
-                    }
-                )
-    except Exception as e:
-        errors += 1
-        raise GenerateException(f"Exception {e} raised in {file_path}") from e
-
-    return seed_instruction_data, warnings, errors
-
-
-def read_taxonomy(logger, taxonomy, taxonomy_base, yaml_rules):
-    seed_instruction_data = []
-    is_file = os.path.isfile(taxonomy)
-    if is_file:  # taxonomy is file
-        seed_instruction_data, warnings, errors = read_taxonomy_file(
-            logger, taxonomy, yaml_rules
-        )
-        if warnings:
-            logger.warn(
-                f"{warnings} warnings (see above) due to taxonomy file not (fully) usable."
-            )
-        if errors:
-            raise SystemExit(yaml.YAMLError("Taxonomy file with errors! Exiting."))
-    else:  # taxonomy is dir
-        # Gather the new or changed YAMLs using git diff
-        try:
-            updated_taxonomy_files = get_taxonomy_diff(taxonomy, taxonomy_base)
-        except NameError as exc:
-            raise utils.GenerateException("`git` binary not found") from exc
-        total_errors = 0
-        total_warnings = 0
-        if updated_taxonomy_files:
-            logger.debug("Found new taxonomy files:")
-            for e in updated_taxonomy_files:
-                logger.debug(f"* {e}")
-        for f in updated_taxonomy_files:
-            file_path = os.path.join(taxonomy, f)
-            data, warnings, errors = read_taxonomy_file(logger, file_path, yaml_rules)
-            total_warnings += warnings
-            total_errors += errors
-            if data:
-                seed_instruction_data.extend(data)
-        if total_warnings:
-            logger.warn(
-                f"{total_warnings} warnings (see above) due to taxonomy files that were not (fully) usable."
-            )
-        if total_errors:
-            raise SystemExit(
-                yaml.YAMLError(f"{total_errors} taxonomy files with errors! Exiting.")
-            )
-    return seed_instruction_data
diff --git a/cli/lab.py b/cli/lab.py
index ec678a4442..4be1899126 100755
--- a/cli/lab.py
+++ b/cli/lab.py
@@ -224,7 +224,7 @@ def diff(ctx, taxonomy_path, taxonomy_base, yaml_rules, quiet):
     """
     # pylint: disable=C0415
     # Local
-    from .generator.generate_data import get_taxonomy_diff, read_taxonomy
+    from .utils import get_taxonomy_diff, read_taxonomy
 
     if not taxonomy_base:
         taxonomy_base = ctx.obj.config.generate.taxonomy_base
diff --git a/cli/utils.py b/cli/utils.py
index 50be1f1d23..1cc24412d6 100644
--- a/cli/utils.py
+++ b/cli/utils.py
@@ -1,18 +1,20 @@
 # Standard
+from functools import cache, wraps
+from logging import Logger
 from pathlib import Path
-from typing import Dict, List, Union
+from typing import Any, Dict, List, Mapping, Optional, Union
 import copy
-import functools
 import glob
+import json
 import logging
 import os
 import platform
 import re
-import shutil
 import subprocess
-import sys
+import tempfile
 
 # Third Party
+from git import Repo, exc
 from langchain_text_splitters import RecursiveCharacterTextSplitter
 import click
 import git
@@ -22,12 +24,22 @@
 # Local
 from . import common
 
+DEFAULT_YAML_RULES = """\
+extends: relaxed
+
+rules:
+  line-length:
+    max: 120
+"""
+
 logging.basicConfig(
     level=logging.INFO,
     format="%(levelname)s %(asctime)s %(filename)s:%(lineno)d %(message)s",
 )
 
-logger = logging.getLogger(__name__)
+
+class TaxonomyReadingException(Exception):
+    """An exception raised during reading of the taxonomy."""
 
 
 def macos_requirement(echo_func, exit_exception):
@@ -38,7 +50,7 @@ def macos_requirement(echo_func, exit_exception):
     """
 
     def decorator(func):
-        @functools.wraps(func)
+        @wraps(func)
         def wrapper(*args, **kwargs):
             if not is_macos_with_m_chip():
                 echo_func(
@@ -139,12 +151,12 @@ def get_taxonomy_diff(repo="taxonomy", base="origin/main"):
     else:
         try:
             head_commit = repo.commit(base)
-        except gitdb.exc.BadName as exc:
+        except gitdb.exc.BadName as e:
             raise SystemExit(
                 yaml.YAMLError(
                     f'Couldn\'t find the taxonomy git ref "{base}" from the current HEAD'
                 )
-            ) from exc
+            ) from e
 
     # Move backwards from HEAD until we find the first commit that is part of base
     # then we can take our diff from there
@@ -156,12 +168,12 @@ def get_taxonomy_diff(repo="taxonomy", base="origin/main"):
             break
         try:
             current_commit = current_commit.parents[0]
-        except IndexError as exc:
+        except IndexError as e:
             raise SystemExit(
                 yaml.YAMLError(
                     f'Couldn\'t find the taxonomy base branch "{base}" from the current HEAD'
                 )
-            ) from exc
+            ) from e
 
     modified_files = [
         d.b_path
@@ -173,55 +185,65 @@ def get_taxonomy_diff(repo="taxonomy", base="origin/main"):
     return updated_taxonomy_files
 
 
-def get_documents(input_pattern: Dict[str, Union[str, List[str]]]) -> List[str]:
+def get_documents(
+    logger,
+    source: Dict[str, Union[str, List[str]]],
+    skip_checkout: Optional[bool] = False,
+) -> List[str]:
     """
     Retrieve the content of files from a Git repository.
 
     Args:
-        input_pattern (dict): Input dictionary containing repository URL, commit hash, and list of file patterns.
+        source (dict): Source info containing repository URL, commit hash, and list of file patterns.
 
     Returns:
          List[str]: List of document contents.
     """ ""
-
-    # Extract input parameters
-
-    repo_url = input_pattern.get("repo")
-    commit_hash = input_pattern.get("commit")
-    file_patterns = input_pattern.get("patterns")
-    temp_dir = os.path.join(os.getcwd(), "temp_repo")
-    if os.path.exists(temp_dir):
-        shutil.rmtree(temp_dir)
-    try:
-        # Create a temporary directory to clone the repository
-        os.makedirs(temp_dir, exist_ok=True)
-
-        # Clone the repository to the temporary directory
-        repo = git.Repo.clone_from(repo_url, temp_dir)
-
-        # Checkout the specified commit
+    repo_url = source.get("repo")
+    commit_hash = source.get("commit")
+    file_patterns = source.get("patterns")
+    with tempfile.TemporaryDirectory() as temp_dir:
+        try:
+            skip_checkout = False
+            if skip_checkout:
+                skip_checkout = True
+            repo = git_clone_checkout(
+                repo_url=repo_url,
+                commit_hash=commit_hash,
+                temp_dir=temp_dir,
+                skip_checkout=skip_checkout,
+            )
+            file_contents = []
+
+            # hack for unit tests. Unit tests use temp dirs in tests/testdata
+            # that needs to be processed instead of the temp_dir created
+            working_dir = temp_dir
+            if not skip_checkout:
+                working_dir = repo
+
+            logger.debug("Processing files...")
+            for pattern in file_patterns:
+                for file_path in glob.glob(os.path.join(working_dir, pattern)):
+                    if os.path.isfile(file_path) and file_path.endswith(".md"):
+                        with open(file_path, "r", encoding="utf-8") as file:
+                            file_contents.append(file.read())
+            # hack for unit tests. Unit tests use temp dirs in tests/testdata
+            # and patches return string values that are paths to those directories.
+            # str object don't have close() method
+            if not isinstance(repo, str):
+                repo.close()
+            return file_contents
+        except (OSError, exc.GitCommandError, FileNotFoundError) as e:
+            raise e
+
+
+def git_clone_checkout(
+    repo_url: str, temp_dir: str, commit_hash: str, skip_checkout: bool
+) -> Repo:
+    repo = Repo.clone_from(repo_url, temp_dir)
+    if not skip_checkout:
         repo.git.checkout(commit_hash)
-
-        file_contents = []
-
-        logger.debug("Processing files...")
-        for pattern in file_patterns:
-            for file_path in glob.glob(os.path.join(temp_dir, pattern)):
-                if os.path.isfile(file_path) and file_path.endswith(".md"):
-                    with open(file_path, "r", encoding="utf-8") as file:
-                        file_contents.append(file.read())
-        repo.close()
-        shutil.rmtree(temp_dir)
-        return file_contents
-
-    except (OSError, git.exc.GitCommandError, FileNotFoundError) as e:
-        logger.error("Error: {}".format(str(e)))
-        return [f"Error: {str(e)}"]
-
-    finally:
-        # Cleanup: Remove the temporary directory if it exists
-        if os.path.exists(temp_dir):
-            shutil.rmtree(temp_dir)
+    return repo
 
 
 def chunk_document(documents: List, server_ctx_size, chunk_word_count) -> List[str]:
@@ -236,14 +258,13 @@ def chunk_document(documents: List, server_ctx_size, chunk_word_count) -> List[s
     """
     no_tokens_per_doc = int(chunk_word_count * 1.3)  # 1 word ~ 1.3 token
     if no_tokens_per_doc > int(server_ctx_size - 1024):
-        logger.error(
+        raise ValueError(
             "Error: {}".format(
                 str(
-                    f"Given word count per doc will exceed the server context window size {server_ctx_size}"
+                    f"Given word count ({chunk_word_count}) per doc will exceed the server context window size ({server_ctx_size})"
                 )
             )
         )
-        sys.exit()
     content = []
     text_splitter = RecursiveCharacterTextSplitter(
         separators=["\n\n", "\n"],
@@ -268,3 +289,234 @@ def get_sysprompt(model=None):
         str: The system prompt for the model being used
     """
     return common.SYS_PROMPT
+
+
+@cache
+def _load_schema(path: "importlib.resources.abc.Traversable") -> "referencing.Resource":
+    """Load the schema from the path into a Resource object.
+
+    Args:
+        path (Traversable): Path to the schema to be loaded.
+
+    Raises:
+        NoSuchResource: If the resource cannot be loaded.
+
+    Returns:
+        Resource: A Resource containing the requested schema.
+    """
+    # pylint: disable=C0415
+    # Third Party
+    from referencing import Resource
+    from referencing.exceptions import NoSuchResource
+    from referencing.jsonschema import DRAFT202012
+
+    try:
+        contents = json.loads(path.read_text(encoding="utf-8"))
+        resource = Resource.from_contents(
+            contents=contents, default_specification=DRAFT202012
+        )
+    except Exception as e:
+        raise NoSuchResource(ref=str(path)) from e
+    return resource
+
+
+def validate_yaml(
+    logger: Logger, contents: Mapping[str, Any], taxonomy_path: Path
+) -> int:
+    """Validate the parsed yaml document using the taxonomy path to
+    determine the proper schema.
+
+    Args:
+        logger (Logger): The logger for errors/warnings.
+        contents (Mapping): The parsed yaml document to validate against the schema.
+        taxonomy_path (Path): Relative path of the taxonomy yaml document where the
+        first element is the schema to use.
+
+    Returns:
+        int: The number of errors found during validation.
+        Messages for each error have been logged.
+    """
+    # pylint: disable=C0415
+    # Standard
+    from importlib import resources
+
+    # Third Party
+    from jsonschema.protocols import Validator
+    from jsonschema.validators import validator_for
+    from referencing import Registry, Resource
+    from referencing.exceptions import NoSuchResource
+    from referencing.typing import URI
+
+    errors = 0
+    version = get_version(contents)
+    schemas_path = resources.files("cli").joinpath(f"schema/v{version}")
+
+    def retrieve(uri: URI) -> Resource:
+        path = schemas_path.joinpath(uri)
+        return _load_schema(path)
+
+    schema_name = taxonomy_path.parts[0]
+    if schema_name not in TAXONOMY_FOLDERS:
+        schema_name = "knowledge" if "document" in contents else "compositional_skills"
+        logger.info(
+            f"Cannot determine schema name from path {taxonomy_path}. Using {schema_name} schema."
+        )
+
+    try:
+        schema_resource = retrieve(f"{schema_name}.json")
+        schema = schema_resource.contents
+        validator_cls = validator_for(schema)
+        validator: Validator = validator_cls(
+            schema, registry=Registry(retrieve=retrieve)
+        )
+
+        for error in validator.iter_errors(contents):
+            errors += 1
+            error_path = error.json_path[1:]
+            if not error_path:
+                error_path = "."
+            logger.error(
+                f"Validation error in {taxonomy_path} on {error_path}: {error.message}"
+            )
+    except NoSuchResource as e:
+        cause = e.__cause__ if e.__cause__ is not None else e
+        errors += 1
+        logger.error(f"Cannot load schema file {e.ref}. {cause}")
+
+    return errors
+
+
+def get_version(contents: Mapping) -> int:
+    version = contents.get("version", 1)
+    if not isinstance(version, int):
+        # schema validation will complain about the type
+        try:
+            version = int(version)
+        except ValueError:
+            version = 1  # fallback to version 1
+    return version
+
+
+# pylint: disable=broad-exception-caught
+def read_taxonomy_file(logger, file_path, yaml_rules: Optional[str] = None):
+    # pylint: disable=C0415
+    # Third Party
+    from yamllint import linter
+    from yamllint.config import YamlLintConfig
+
+    seed_instruction_data = []
+    warnings = 0
+    errors = 0
+    file_path = Path(file_path).resolve()
+    # file should end with ".yaml" explicitly
+    if file_path.suffix != ".yaml":
+        logger.warn(f"Skipping {file_path}! Use lowercase '.yaml' extension instead.")
+        warnings += 1
+        return None, warnings, errors
+    for i in range(len(file_path.parts) - 1, -1, -1):
+        if file_path.parts[i] in TAXONOMY_FOLDERS:
+            taxonomy_path = Path(*file_path.parts[i:])
+            break
+    else:
+        taxonomy_path = file_path
+    # read file if extension is correct
+    try:
+        with open(file_path, "r", encoding="utf-8") as file:
+            # do general YAML linting if specified
+            if yaml_rules is not None:
+                try:
+                    yaml_config = YamlLintConfig(file=yaml_rules)
+                except FileNotFoundError:
+                    logger.debug(f"Cannot find {yaml_rules}. Using default rules.")
+                    yaml_config = YamlLintConfig(DEFAULT_YAML_RULES)
+            else:
+                yaml_config = YamlLintConfig(DEFAULT_YAML_RULES)
+            linter_problems = list(linter.run(file, yaml_config))
+            if linter_problems:
+                lint_messages = [f"Problems found in file {file.name}"]
+                for p in linter_problems:
+                    errors += 1
+                    loc = f"{p.line}:{p.column}"
+                    desc = f"{p.desc} ({p.rule})" if p.rule else p.desc
+                    lint_messages.append(f"  {loc:<9} {p.level:<9} {desc}")
+                logger.error("\n".join(lint_messages))
+                return None, warnings, errors
+        # do more explict checking of file contents
+        with open(file_path, "r", encoding="utf-8") as file:
+            contents = yaml.safe_load(file)
+            if not contents:
+                logger.warn(f"Skipping {file_path} because it is empty!")
+                warnings += 1
+                return None, warnings, errors
+            validation_errors = validate_yaml(logger, contents, taxonomy_path)
+            if validation_errors:
+                errors += validation_errors
+                return None, warnings, errors
+
+            # get seed instruction data
+            tax_path = "->".join(taxonomy_path.parent.parts)
+            task_description = contents.get("task_description")
+            documents = contents.get("document")
+            if documents:
+                documents = get_documents(source=documents, logger=logger)
+                logger.debug("Content from git repo fetched")
+
+            for seed_example in contents.get("seed_examples"):
+                question = seed_example.get("question")
+                answer = seed_example.get("answer")
+                context = seed_example.get("context", "")
+                seed_instruction_data.append(
+                    {
+                        "instruction": question,
+                        "input": context,
+                        "output": answer,
+                        "taxonomy_path": tax_path,
+                        "task_description": task_description,
+                        "document": documents,
+                    }
+                )
+    except Exception as e:
+        errors += 1
+        raise TaxonomyReadingException(f"Exception {e} raised in {file_path}") from e
+
+    return seed_instruction_data, warnings, errors
+
+
+def read_taxonomy(logger, taxonomy, taxonomy_base, yaml_rules):
+    seed_instruction_data = []
+    is_file = os.path.isfile(taxonomy)
+    if is_file:  # taxonomy is file
+        seed_instruction_data, warnings, errors = read_taxonomy_file(
+            logger, taxonomy, yaml_rules
+        )
+        if warnings:
+            logger.warn(
+                f"{warnings} warnings (see above) due to taxonomy file not (fully) usable."
+            )
+        if errors:
+            raise SystemExit(yaml.YAMLError("Taxonomy file with errors! Exiting."))
+    else:  # taxonomy is dir
+        # Gather the new or changed YAMLs using git diff
+        updated_taxonomy_files = get_taxonomy_diff(taxonomy, taxonomy_base)
+        total_errors = 0
+        total_warnings = 0
+        if updated_taxonomy_files:
+            logger.debug("Found new taxonomy files:")
+            for e in updated_taxonomy_files:
+                logger.debug(f"* {e}")
+        for f in updated_taxonomy_files:
+            file_path = os.path.join(taxonomy, f)
+            data, warnings, errors = read_taxonomy_file(logger, file_path, yaml_rules)
+            total_warnings += warnings
+            total_errors += errors
+            if data:
+                seed_instruction_data.extend(data)
+        if total_warnings:
+            logger.warn(
+                f"{total_warnings} warnings (see above) due to taxonomy files that were not (fully) usable."
+            )
+        if total_errors:
+            raise SystemExit(
+                yaml.YAMLError(f"{total_errors} taxonomy files with errors! Exiting.")
+            )
+    return seed_instruction_data


Test patch:
diff --git a/tests/test_lab_diff.py b/tests/test_lab_diff.py
index 153b58c7b5..7149297c2a 100644
--- a/tests/test_lab_diff.py
+++ b/tests/test_lab_diff.py
@@ -10,47 +10,6 @@
 from cli import lab
 
 TAXONOMY_BASE = "main"
-TEST_VALID_YAML = b"""created_by: nathan-weinberg
-seed_examples:
-- answer: "Yes I can!"
-  context: "This unit test is valid!"
-  question: "Could you write a unit test?"
-- answer: answer 2
-  context: context 2
-  question: question 2
-- answer: answer 3
-  context: context 3
-  question: question 3
-- answer: answer 4
-  context: context 4
-  question: question 4
-- answer: answer 5
-  context: context 5
-  question: question 5
-task_description: 'This is a task'
-"""
-TEST_INVALID_YAML = b"""created_by: nathan-weinberg
-seed_examples:
-- answer: "Yes I can!"
-  context: "This unit test has a line with 124 characters! It is too long for the default rules but not too long for the customer rules!"
-  question: "Could you write a unit test?"
-- answer: "answer2"
-  question: "question2"
-- answer: "answer3"
-  question: "question3"
-- answer: "answer4"
-  question: "question4"
-- answer: "answer5"
-  question: "question5"
-task_description: 'This is a task'
-"""
-TEST_FAILING_COMPOSITIONAL_SKILL_YAML = b"""created_by: test
-version: 1
-seed_examples:
-- question: "Does this yaml pass schema validation?"
-  answer: "No, it does not! It should have 5 examples."
-task_description: 'This yaml does not conform to the schema'
-"""
 
 TEST_CUSTOM_YAML_RULES = b"""extends: relaxed
 
@@ -166,108 +125,116 @@ def test_diff_invalid_path(self):
             self.assertEqual(result.exit_code, 0)
 
     def test_diff_valid_yaml(self):
-        valid_yaml_file = "compositional_skills/qna_valid.yaml"
-        self.taxonomy.create_untracked(valid_yaml_file, TEST_VALID_YAML)
-        runner = CliRunner()
-        result = runner.invoke(
-            lab.diff,
-            [
-                "--taxonomy-base",
-                TAXONOMY_BASE,
-                "--taxonomy-path",
-                self.taxonomy.root,
-            ],
-        )
-        self.assertIn(f"Taxonomy in /{self.taxonomy.root}/ is valid :)", result.output)
-        self.assertEqual(result.exit_code, 0)
+        with open("tests/testdata/skill_valid_answer.yaml", "rb") as qnafile:
+            valid_yaml_file = "compositional_skills/qna_valid.yaml"
+            self.taxonomy.create_untracked(valid_yaml_file, qnafile.read())
+            runner = CliRunner()
+            result = runner.invoke(
+                lab.diff,
+                [
+                    "--taxonomy-base",
+                    TAXONOMY_BASE,
+                    "--taxonomy-path",
+                    self.taxonomy.root,
+                ],
+            )
+            self.assertIn(
+                f"Taxonomy in /{self.taxonomy.root}/ is valid :)", result.output
+            )
+            self.assertEqual(result.exit_code, 0)
 
     def test_diff_valid_yaml_quiet(self):
-        valid_yaml_file = "compositional_skills/qna_valid.yaml"
-        self.taxonomy.create_untracked(valid_yaml_file, TEST_VALID_YAML)
-        runner = CliRunner()
-        result = runner.invoke(
-            lab.diff,
-            [
-                "--taxonomy-base",
-                TAXONOMY_BASE,
-                "--taxonomy-path",
-                self.taxonomy.root,
-                "--quiet",
-            ],
-        )
-        self.assertEqual(result.output, "")
-        self.assertEqual(result.exit_code, 0)
+        with open("tests/testdata/skill_valid_answer.yaml", "rb") as qnafile:
+            valid_yaml_file = "compositional_skills/qna_valid.yaml"
+            self.taxonomy.create_untracked(valid_yaml_file, qnafile.read())
+            runner = CliRunner()
+            result = runner.invoke(
+                lab.diff,
+                [
+                    "--taxonomy-base",
+                    TAXONOMY_BASE,
+                    "--taxonomy-path",
+                    self.taxonomy.root,
+                    "--quiet",
+                ],
+            )
+            self.assertEqual(result.output, "")
+            self.assertEqual(result.exit_code, 0)
 
     def test_diff_invalid_yaml(self):
-        invalid_yaml_file = "compositional_skills/qna_invalid.yaml"
-        self.taxonomy.create_untracked(invalid_yaml_file, TEST_INVALID_YAML)
-        runner = CliRunner()
-        result = runner.invoke(
-            lab.diff,
-            [
-                "--taxonomy-base",
-                TAXONOMY_BASE,
-                "--taxonomy-path",
-                self.taxonomy.root,
-            ],
-        )
-        self.assertIn("Reading taxonomy failed", result.output)
-        self.assertEqual(result.exit_code, 1)
+        with open("tests/testdata/invalid_yaml.yaml", "rb") as qnafile:
+            self.taxonomy.create_untracked(
+                "compositional_skills/qna_invalid.yaml", qnafile.read()
+            )
+            runner = CliRunner()
+            result = runner.invoke(
+                lab.diff,
+                [
+                    "--taxonomy-base",
+                    TAXONOMY_BASE,
+                    "--taxonomy-path",
+                    self.taxonomy.root,
+                ],
+            )
+            self.assertIn("Reading taxonomy failed", result.output)
+            self.assertEqual(result.exit_code, 1)
 
     def test_diff_invalid_yaml_quiet(self):
-        invalid_yaml_file = "compositional_skills/qna_invalid.yaml"
-        self.taxonomy.create_untracked(invalid_yaml_file, TEST_INVALID_YAML)
-        runner = CliRunner()
-        result = runner.invoke(
-            lab.diff,
-            [
-                "--taxonomy-base",
-                TAXONOMY_BASE,
-                "--taxonomy-path",
-                self.taxonomy.root,
-                "--quiet",
-            ],
-        )
-        self.assertIsNotNone(result.exception)
-        self.assertEqual(result.exit_code, 1)
+        with open("tests/testdata/invalid_yaml.yaml", "rb") as qnafile:
+            self.taxonomy.create_untracked(
+                "compositional_skills/qna_invalid.yaml", qnafile.read()
+            )
+            runner = CliRunner()
+            result = runner.invoke(
+                lab.diff,
+                [
+                    "--taxonomy-base",
+                    TAXONOMY_BASE,
+                    "--taxonomy-path",
+                    self.taxonomy.root,
+                    "--quiet",
+                ],
+            )
+            self.assertIsNotNone(result.exception)
+            self.assertEqual(result.exit_code, 1)
 
     def test_diff_custom_yaml(self):
-        custom_rules_file = Path("custom_rules.yaml")
-        custom_rules_file.write_bytes(TEST_CUSTOM_YAML_RULES)
-        invalid_yaml_file = "compositional_skills/qna_invalid.yaml"
-        self.taxonomy.create_untracked(invalid_yaml_file, TEST_INVALID_YAML)
-        runner = CliRunner()
-        result = runner.invoke(
-            lab.diff,
-            [
-                "--taxonomy-base",
-                TAXONOMY_BASE,
-                "--taxonomy-path",
-                self.taxonomy.root,
-                "--yaml-rules",
-                custom_rules_file,
-                "--quiet",
-            ],
-        )
-        # custom yaml rules mean "invalid" yaml file should pass
-        self.assertEqual(result.output, "")
-        self.assertEqual(result.exit_code, 0)
-        Path.unlink(custom_rules_file)
+        with open("tests/testdata/invalid_yaml.yaml", "rb") as qnafile:
+            custom_rules_file = Path("custom_rules.yaml")
+            custom_rules_file.write_bytes(TEST_CUSTOM_YAML_RULES)
+            invalid_yaml_file = "compositional_skills/qna_invalid.yaml"
+            self.taxonomy.create_untracked(invalid_yaml_file, qnafile.read())
+            runner = CliRunner()
+            result = runner.invoke(
+                lab.diff,
+                [
+                    "--taxonomy-base",
+                    TAXONOMY_BASE,
+                    "--taxonomy-path",
+                    self.taxonomy.root,
+                    "--yaml-rules",
+                    custom_rules_file,
+                    "--quiet",
+                ],
+            )
+            # custom yaml rules mean "invalid" yaml file should pass
+            self.assertEqual(result.output, "")
+            self.assertEqual(result.exit_code, 0)
+            Path.unlink(custom_rules_file)
 
     def test_diff_failing_schema_yaml(self):
-        failing_yaml_file = "compositional_skills/failing/qna.yaml"
-        self.taxonomy.create_untracked(
-            failing_yaml_file, TEST_FAILING_COMPOSITIONAL_SKILL_YAML
-        )
-        runner = CliRunner()
-        result = runner.invoke(
-            lab.diff,
-            [
-                "--taxonomy-base",
-                TAXONOMY_BASE,
-                "--taxonomy-path",
-                self.taxonomy.root,
-            ],
-        )
-        self.assertIn("Reading taxonomy failed", result.output)
-        self.assertEqual(result.exit_code, 1)
+        with open("tests/testdata/skill_incomplete.yaml", "rb") as qnafile:
+            failing_yaml_file = "compositional_skills/failing/qna.yaml"
+            self.taxonomy.create_untracked(failing_yaml_file, qnafile.read())
+            runner = CliRunner()
+            result = runner.invoke(
+                lab.diff,
+                [
+                    "--taxonomy-base",
+                    TAXONOMY_BASE,
+                    "--taxonomy-path",
+                    self.taxonomy.root,
+                ],
+            )
+            self.assertIn("Reading taxonomy failed", result.output)
+            self.assertEqual(result.exit_code, 1)
diff --git a/tests/test_lab_generate.py b/tests/test_lab_generate.py
index 227f20f463..77162d570e 100644
--- a/tests/test_lab_generate.py
+++ b/tests/test_lab_generate.py
@@ -16,57 +16,7 @@
 
 # Local
 from .taxonomy import MockTaxonomy
-
-test_skill_valid_answer = b"""created_by: test-bot
-version: 1
-seed_examples:
-- answer: Yes, it is.
-  question: Is this for a test?
-- answer: Yes I am very sure.
-  question: Are you sure it's for a test?
-- answer: "answer3"
-  question: "question3"
-- answer: "answer4"
-  question: "question4"
-- answer: "answer5"
-  question: "question5"
-task_description: for testing
-"""
-
-
-test_skill_invalid_answer = b"""created_by: test-bot
-seed_examples:
-- answer: Yes
-  question: Is this for a test?
-- answer: "answer2"
-  question: "question2"
-- answer: "answer3"
-  question: "question3"
-- answer: "answer4"
-  question: "question4"
-- answer: "answer5"
-  question: "question5"
-task_description: for testing
-"""
-
-generate_data_return_value = [
-    {
-        "instruction": "3. Tell me a pun about water.",
-        "input": "",
-        "output": "Why did the scarecrow win an award?\nBecause he was outstanding in his field!",
-        "taxonomy_path": "compositional_skills->writing->freeform->jokes->puns-copy->general",
-        "task_description": "to teach a large language model to come up with puns",
-        "document": None,
-    },
-    {
-        "instruction": "4. Give me a pun about books.",
-        "input": "",
-        "output": "Why don't books ever get lost on the shelf?\nBecause they are always on the cover!",
-        "taxonomy_path": "compositional_skills->writing->freeform->jokes->puns-copy->general",
-        "task_description": "to teach a large language model to come up with puns",
-        "document": None,
-    },
-]
+from .testdata import testdata
 
 
 class TestLabGenerate(unittest.TestCase):
@@ -99,6 +49,7 @@ def test_generate_exception_error(self, generate_data_mock):
                 "Generating dataset failed with the following error: Connection Error",
                 result.output,
             )
+            mt.teardown()
 
     def test_taxonomy_not_found(self):
         runner = CliRunner()
@@ -140,32 +91,35 @@ def test_no_new_data(self):
                 "Nothing to generate. Exiting.",
                 result.output,
             )
+            mt.teardown()
 
     def test_new_data_invalid_answer(self):
         runner = CliRunner()
-        with runner.isolated_filesystem():
-            mt = MockTaxonomy(pathlib.Path("taxonomy"))
-            mt.create_untracked(
-                "compositional_skills/tracked/qna.yaml", test_skill_invalid_answer
-            )
-            result = runner.invoke(
-                lab.generate,
-                [
-                    "--taxonomy-base",
-                    "main",
-                    "--taxonomy-path",
-                    mt.root,
-                    "--endpoint-url",
-                    "localhost:8000",
-                ],
-            )
-            self.assertEqual(
-                result.exit_code, 1, "command finished with an unexpected exit code"
-            )
-            self.assertIn(
-                "taxonomy files with errors",
-                result.output,
-            )
+        with open("tests/testdata/skill_invalid_answer.yaml", "rb") as qnafile:
+            with runner.isolated_filesystem():
+                mt = MockTaxonomy(pathlib.Path("taxonomy"))
+                mt.create_untracked(
+                    "compositional_skills/tracked/qna.yaml", qnafile.read()
+                )
+                result = runner.invoke(
+                    lab.generate,
+                    [
+                        "--taxonomy-base",
+                        "main",
+                        "--taxonomy-path",
+                        mt.root,
+                        "--endpoint-url",
+                        "localhost:8000",
+                    ],
+                )
+                self.assertEqual(
+                    result.exit_code, 1, "command finished with an unexpected exit code"
+                )
+                self.assertIn(
+                    "taxonomy files with errors",
+                    result.output,
+                )
+                mt.teardown()
 
     @patch(
         "cli.generator.generate_data.get_instructions_from_model",
@@ -174,19 +128,55 @@ def test_new_data_invalid_answer(self):
         ),
     )
     def test_OpenAI_server_error(self, get_instructions_from_model):
-        with CliRunner().isolated_filesystem():
-            mt = MockTaxonomy(pathlib.Path("taxonomy"))
-            mt.create_untracked(
-                "compositional_skills/tracked/qna.yaml", test_skill_valid_answer
-            )
-            with self.assertRaises(GenerateException) as exc:
+        with open("tests/testdata/skill_valid_answer.yaml", "rb") as qnafile:
+            with CliRunner().isolated_filesystem():
+                mt = MockTaxonomy(pathlib.Path("taxonomy"))
+                mt.create_untracked(
+                    "compositional_skills/tracked/qna.yaml", qnafile.read()
+                )
+                with self.assertRaises(GenerateException) as exc:
+                    generate_data(
+                        logger=logging.getLogger("test_logger"),
+                        api_base="localhost:8000",
+                        api_key="",
+                        model_name="test-model",
+                        num_cpus=10,
+                        num_instructions_to_generate=100,
+                        taxonomy=mt.root,
+                        taxonomy_base="main",
+                        output_dir="generated",
+                        prompt_file_path="prompt.txt",
+                        rouge_threshold=0.9,
+                        console_output=True,
+                        chunk_word_count=1000,
+                        server_ctx_size=4096,
+                        tls_insecure=False,
+                    )
+                self.assertIn(
+                    "There was a problem connecting to the OpenAI server",
+                    f"{exc.exception}",
+                )
+                get_instructions_from_model.assert_called_once()
+                mt.teardown()
+
+    @patch(
+        "cli.generator.generate_data.get_instructions_from_model",
+        return_value=(testdata.generate_data_return_value, 0),
+    )
+    def test_generate_no_error(self, get_instructions_from_model):
+        with open("tests/testdata/skill_valid_answer.yaml", "rb") as qnafile:
+            with CliRunner().isolated_filesystem():
+                mt = MockTaxonomy(pathlib.Path("taxonomy"))
+                mt.create_untracked(
+                    "compositional_skills/tracked/qna.yaml", qnafile.read()
+                )
                 generate_data(
                     logger=logging.getLogger("test_logger"),
                     api_base="localhost:8000",
                     api_key="",
-                    model_name="test-model",
+                    model_name="my-model",
                     num_cpus=10,
-                    num_instructions_to_generate=100,
+                    num_instructions_to_generate=1,
                     taxonomy=mt.root,
                     taxonomy_base="main",
                     output_dir="generated",
@@ -197,21 +187,31 @@ def test_OpenAI_server_error(self, get_instructions_from_model):
                     server_ctx_size=4096,
                     tls_insecure=False,
                 )
-            self.assertIn(
-                "There was a problem connecting to the OpenAI server",
-                f"{exc.exception}",
-            )
-            get_instructions_from_model.assert_called_once()
+                get_instructions_from_model.assert_called_once()
+                expected_files = [
+                    "generated_my-model*.json",
+                    "train_my-model*.jsonl",
+                    "test_my-model*.jsonl",
+                ]
+                for f in os.listdir("generated"):
+                    self.assertTrue(
+                        any(fnmatch.fnmatch(f, pattern) for pattern in expected_files)
+                    )
+                mt.teardown()
 
     @patch(
         "cli.generator.generate_data.get_instructions_from_model",
-        return_value=(generate_data_return_value, 0),
+        return_value=(testdata.generate_data_return_value, 0),
+    )
+    @patch(
+        "cli.generator.generate_data.read_taxonomy",
+        return_value=testdata.knowledge_seed_instruction,
     )
-    def test_no_error(self, get_instructions_from_model):
-        with CliRunner().isolated_filesystem():
+    def test_knowledge_docs_no_error(self, read_taxonomy, get_instructions_from_model):
+        with open("tests/testdata/knowledge_valid.yaml", "rb") as qnafile:
             mt = MockTaxonomy(pathlib.Path("taxonomy"))
             mt.create_untracked(
-                "compositional_skills/tracked/qna.yaml", test_skill_valid_answer
+                "knowledge/technical-manual/test/qna.yaml", qnafile.read()
             )
             generate_data(
                 logger=logging.getLogger("test_logger"),
@@ -231,6 +231,7 @@ def test_no_error(self, get_instructions_from_model):
                 tls_insecure=False,
             )
             get_instructions_from_model.assert_called_once()
+            read_taxonomy.assert_called_once()
             expected_files = [
                 "generated_my-model*.json",
                 "train_my-model*.jsonl",
@@ -240,3 +241,4 @@ def test_no_error(self, get_instructions_from_model):
                 self.assertTrue(
                     any(fnmatch.fnmatch(f, pattern) for pattern in expected_files)
                 )
+            mt.teardown()
diff --git a/tests/test_utils.py b/tests/test_utils.py
new file mode 100644
index 0000000000..13a0f4f4b5
--- /dev/null
+++ b/tests/test_utils.py
@@ -0,0 +1,55 @@
+# Standard
+from unittest.mock import patch
+import logging
+import unittest
+
+# Third Party
+# Third party
+import yaml
+
+# First Party
+from cli import utils
+
+# Local
+from .testdata import testdata
+
+
+class TestUtils(unittest.TestCase):
+    """Test collection in cli.utils."""
+
+    def test_chunk_docs_wc_exeeds_ctx_window(self):
+        with self.assertRaises(ValueError) as exc:
+            utils.chunk_document(
+                documents=testdata.documents,
+                chunk_word_count=1000,
+                server_ctx_size=1034,
+            )
+        self.assertIn(
+            "Given word count (1000) per doc will exceed the server context window size (1034)",
+            f"{exc.exception}",
+        )
+
+    def test_chunk_docs_chunk_overlap_error(self):
+        with self.assertRaises(ValueError) as exc:
+            utils.chunk_document(
+                documents=testdata.documents,
+                chunk_word_count=5,
+                server_ctx_size=1034,
+            )
+        self.assertIn(
+            "Got a larger chunk overlap (100) than chunk size (24), should be smaller",
+            f"{exc.exception}",
+        )
+
+    @patch("cli.utils.git_clone_checkout", return_value="tests/testdata/temp_repo")
+    def test_get_document(self, git_actions):
+        with open(
+            "tests/testdata/knowledge_valid.yaml", "r", encoding="utf-8"
+        ) as qnafile:
+            documents = utils.get_documents(
+                source=yaml.safe_load(qnafile).get("document"),
+                skip_checkout=True,
+                logger=logging.getLogger("_test_"),
+            )
+            git_actions.assert_called_once()
+            self.assertEqual(len(documents), 2)
diff --git a/tests/testdata/invalid_yaml.yaml b/tests/testdata/invalid_yaml.yaml
new file mode 100644
index 0000000000..b46a09344b
--- /dev/null
+++ b/tests/testdata/invalid_yaml.yaml
@@ -0,0 +1,15 @@
+created_by: test-bot
+seed_examples:
+- question: What is this skill about?
+  answer: It's a skill that makes the tests more skillful
+- answer: "answer2"
+  question: "question2"
+- answer: "answer6"
+  question: "This is for a unit test and has a line with 124 characters! It is too long for the default rules but not too long for the customer rules!"
+- answer: "answer3"
+  question: "question3"
+- answer: "answer4"
+  question: "question4"
+- answer: "answer5"
+  question: "question5"
+task_description: For invalid yaml tests
diff --git a/tests/testdata/knowledge_valid.yaml b/tests/testdata/knowledge_valid.yaml
new file mode 100644
index 0000000000..0027cedba2
--- /dev/null
+++ b/tests/testdata/knowledge_valid.yaml
@@ -0,0 +1,20 @@
+created_by: test-bot
+domain: test-domain
+seed_examples:
+- question: What is this knowledge about?
+  answer: It's a knowledge that makes the tests more knowledgeable
+- answer: "answer2"
+  question: "question2"
+- answer: "answer3"
+  question: "question3"
+- answer: "answer4"
+  question: "question4"
+- answer: "answer5"
+  question: "question5"
+task_description: For knowledge tests
+document:
+  repo: https://github.com/example-org/example-repo
+  commit: a0c3c8e
+  patterns:
+  - "*.md"
+  - docs/*.md
\ No newline at end of file
diff --git a/tests/testdata/skill_incomplete.yaml b/tests/testdata/skill_incomplete.yaml
new file mode 100644
index 0000000000..d1aa753c32
--- /dev/null
+++ b/tests/testdata/skill_incomplete.yaml
@@ -0,0 +1,6 @@
+created_by: test
+version: 1
+seed_examples:
+- question: "Does this yaml pass schema validation?"
+  answer: "No, it does not! It should have 5 examples."
+task_description: 'This yaml does not conform to the schema'
diff --git a/tests/testdata/skill_invalid_answer.yaml b/tests/testdata/skill_invalid_answer.yaml
new file mode 100644
index 0000000000..bd6c9e22a9
--- /dev/null
+++ b/tests/testdata/skill_invalid_answer.yaml
@@ -0,0 +1,13 @@
+created_by: test-bot
+seed_examples:
+- answer: Yes
+  question: Is this for a test?
+- answer: "answer2"
+  question: "question2"
+- answer: "answer3"
+  question: "question3"
+- answer: "answer4"
+  question: "question4"
+- answer: "answer5"
+  question: "question5"
+task_description: for testing
diff --git a/tests/testdata/skill_valid_answer.yaml b/tests/testdata/skill_valid_answer.yaml
new file mode 100644
index 0000000000..30c02ad1de
--- /dev/null
+++ b/tests/testdata/skill_valid_answer.yaml
@@ -0,0 +1,14 @@
+created_by: test-bot
+version: 1
+seed_examples:
+- answer: Yes, it is.
+  question: Is this for a test?
+- answer: Yes I am very sure.
+  question: Are you sure it's for a test?
+- answer: "answer3"
+  question: "question3"
+- answer: "answer4"
+  question: "question4"
+- answer: "answer5"
+  question: "question5"
+task_description: for testing
diff --git a/tests/testdata/temp_repo/docs/skill-wiki.md b/tests/testdata/temp_repo/docs/skill-wiki.md
new file mode 100644
index 0000000000..ce53550a71
--- /dev/null
+++ b/tests/testdata/temp_repo/docs/skill-wiki.md
@@ -0,0 +1,7 @@
+<!-- source: https://en.wikipedia.org/wiki/Skill -->
+
+# Skill
+
+A skill is the learned ability to act with determined results with good execution often within a given amount of time, energy, or both. Skills can often be divided into domain-general and domain-specific skills. For example, in the domain of work, some general skills would include time management, teamwork and leadership, self-motivation and others, whereas domain-specific skills would be used only for a certain job. Skill usually requires certain environmental stimuli and situations to assess the level of skill being shown and used.[citation needed]
+A skill may be called an art when it represents a body of knowledge or branch of learning, as in the art of medicine or the art of war.[1] Although the arts are also skills, there are many skills that form an art but have no connection to the fine arts.
+People need a broad range of skills to contribute to the modern economy. A U.S. Department of Labor study showed that through technology, the workplace is changing, and identified 16 basic skills that employees must have to be able to change with it.[2] Three broad categories of skills are suggested and these are technical, human, and conceptual.[3] The first two can be substituted with hard and soft skills, respectively.[4]
\ No newline at end of file
diff --git a/tests/testdata/temp_repo/knowledge-wiki.md b/tests/testdata/temp_repo/knowledge-wiki.md
new file mode 100644
index 0000000000..466ca58424
--- /dev/null
+++ b/tests/testdata/temp_repo/knowledge-wiki.md
@@ -0,0 +1,12 @@
+<!-- source: https://en.wikipedia.org/wiki/Knowledge#A_priori_and_a_posteriori -->
+
+# Knowledge 
+
+Knowledge is an awareness of facts, a familiarity with individuals and situations, 
+or a practical skill. Knowledge of facts, also called propositional knowledge, is often characterized 
+as true belief that is distinct from opinion or guesswork by virtue of justification. 
+While there is wide agreement among philosophers that propositional knowledge is a form of true belief, 
+many controversies focus on justification. This includes questions like how to understand justification, 
+whether it is needed at all, and whether something else besides it is needed. 
+These controversies intensified in the latter half of the 20 century due to a series of thought experiments 
+called Get tier cases that provoked alternative definitions.
diff --git a/tests/testdata/testdata.py b/tests/testdata/testdata.py
new file mode 100644
index 0000000000..caea400302
--- /dev/null
+++ b/tests/testdata/testdata.py
@@ -0,0 +1,72 @@
+documents = [
+    """Knowledge is an awareness of facts, a familiarity with individuals and situations, 
+      or a practical skill. Knowledge of facts, also called propositional knowledge, is often characterized 
+      as true belief that is distinct from opinion or guesswork by virtue of justification. 
+      While there is wide agreement among philosophers that propositional knowledge is a form of true belief, 
+      many controversies focus on justification. This includes questions like how to understand justification, 
+      whether it is needed at all, and whether something else besides it is needed. 
+      These controversies intensified in the latter half of the 20th century due to a series of thought experiments 
+      called Gettier cases that provoked alternative definitions."""
+]
+
+knowledge_seed_instruction = [
+    {
+        "instruction": "What is this knowledge about?",
+        "input": "",
+        "output": "It's a knowledge that makes the tests more knowledgeable",
+        "taxonomy_path": "knowledge->textbook->puns-copy->general",
+        "task_description": "For knowledge tests",
+        "document": documents,
+    },
+    {
+        "instruction": "question2",
+        "input": "",
+        "output": "answer2",
+        "taxonomy_path": "knowledge->textbook->puns-copy->general",
+        "task_description": "For knowledge tests",
+        "document": documents,
+    },
+    {
+        "instruction": "question3",
+        "input": "",
+        "output": "answer3",
+        "taxonomy_path": "knowledge->textbook->puns-copy->general",
+        "task_description": "For knowledge tests",
+        "document": documents,
+    },
+    {
+        "instruction": "question4",
+        "input": "",
+        "output": "answer4",
+        "taxonomy_path": "knowledge->textbook->puns-copy->general",
+        "task_description": "For knowledge tests",
+        "document": documents,
+    },
+    {
+        "instruction": "Question5",
+        "input": "",
+        "output": "Answer5",
+        "taxonomy_path": "knowledge->textbook->puns-copy->general",
+        "task_description": "For knowledge tests",
+        "document": documents,
+    },
+]
+
+generate_data_return_value = [
+    {
+        "instruction": "3. Tell me a pun about water.",
+        "input": "",
+        "output": "Why did the scarecrow win an award?\nBecause he was outstanding in his field!",
+        "taxonomy_path": "compositional_skills->writing->freeform->jokes->puns-copy->general",
+        "task_description": "to teach a large language model to come up with puns",
+        "document": None,
+    },
+    {
+        "instruction": "4. Give me a pun about books.",
+        "input": "",
+        "output": "Why don't books ever get lost on the shelf?\nBecause they are always on the cover!",
+        "taxonomy_path": "compositional_skills->writing->freeform->jokes->puns-copy->general",
+        "task_description": "to teach a large language model to come up with puns",
+        "document": None,
+    },
+]