diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/.gitignore b/Prism/Dream/Dream_Baseline/eval_instruct/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..18b56efe2b0943aea0e6dc856f57f3a18f9eceb2 --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/.gitignore @@ -0,0 +1,26 @@ +env +*.pyc +output/ +output5/ +data/ +lm_cache +.idea +build +dist +*.egg-info +venv +.venv/ +.vscode/ +temp +__pycache__ +.ipynb_checkpoints +temp +test_logs/ +# IPython +profile_default/ +ipython_config.py +# don't track (the default location of) the cached requests +lm_eval/caching/.cache +# don't track files created by wandb +wandb +examples/wandb diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/README.md b/Prism/Dream/Dream_Baseline/eval_instruct/README.md new file mode 100644 index 0000000000000000000000000000000000000000..aebe1795bdb18fb0cbea231d2262fcb91fe9304c --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/README.md @@ -0,0 +1,16 @@ +# Dream-Instruct Evaluation Toolkit +This toolkit contains the code Dream-Instruct models make use of for evaluation. + +## Quickstart +To install the toolkit, run: +``` +pip install -e ".[ifeval,math]" +``` + +We provide a script to evaluate [Dream-org/Dream-v0-Instruct-7B](https://huggingface.co/Dream-org/Dream-v0-Instruct-7B): +``` +bash eval.sh +``` + +## Acknowledgement +This is a fork of [EleutherAI/lm-evaluation-harness](https://github.com/EleutherAI/lm-evaluation-harness/tree/main). diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/__init__.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..fece9162482d018c2969a9e67603096d0ad21713 --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/__init__.py @@ -0,0 +1,7 @@ +import logging +import os + +from .evaluator import evaluate, simple_evaluate + + +__version__ = "0.4.8" diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/__main__.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..dda41ebe43d39522c4961fc7bfdf2d730e2eec26 --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/__main__.py @@ -0,0 +1,512 @@ +import argparse +import json +import logging +import os +import sys +from functools import partial +from typing import Union + +from lm_eval import evaluator, utils +from lm_eval.evaluator import request_caching_arg_to_dict +from lm_eval.loggers import EvaluationTracker, WandbLogger +from lm_eval.tasks import TaskManager +from lm_eval.utils import ( + handle_non_serializable, + make_table, + simple_parse_args_string, +) + + +def try_parse_json(value: str) -> Union[str, dict, None]: + if value is None: + return None + try: + return json.loads(value) + except json.JSONDecodeError: + if "{" in value: + raise argparse.ArgumentTypeError( + f"Invalid JSON: {value}. Hint: Use double quotes for JSON strings." + ) + return value + + +def _int_or_none_list_arg_type( + min_len: int, max_len: int, defaults: str, value: str, split_char: str = "," +): + def parse_value(item): + item = item.strip().lower() + if item == "none": + return None + try: + return int(item) + except ValueError: + raise argparse.ArgumentTypeError(f"{item} is not an integer or None") + + items = [parse_value(v) for v in value.split(split_char)] + num_items = len(items) + + if num_items == 1: + # Makes downstream handling the same for single and multiple values + items = items * max_len + elif num_items < min_len or num_items > max_len: + raise argparse.ArgumentTypeError( + f"Argument requires {max_len} integers or None, separated by '{split_char}'" + ) + elif num_items != max_len: + logging.warning( + f"Argument requires {max_len} integers or None, separated by '{split_char}'. " + "Missing values will be filled with defaults." + ) + default_items = [parse_value(v) for v in defaults.split(split_char)] + items.extend( + default_items[num_items:] + ) # extend items list with missing defaults + + return items + + +def check_argument_types(parser: argparse.ArgumentParser): + """ + Check to make sure all CLI args are typed, raises error if not + """ + for action in parser._actions: + if action.dest != "help" and not action.const: + if action.type is None: + raise ValueError( + f"Argument '{action.dest}' doesn't have a type specified." + ) + else: + continue + + +def setup_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument( + "--model", "-m", type=str, default="hf", help="Name of model e.g. `hf`" + ) + parser.add_argument( + "--tasks", + "-t", + default=None, + type=str, + metavar="task1,task2", + help="Comma-separated list of task names or task groupings to evaluate on.\nTo get full list of tasks, use one of the commands `lm-eval --tasks {{list_groups,list_subtasks,list_tags,list}}` to list out all available names for task groupings; only (sub)tasks; tags; or all of the above", + ) + parser.add_argument( + "--model_args", + "-a", + default="", + type=try_parse_json, + help="""Comma separated string or JSON formatted arguments for model, e.g. `pretrained=EleutherAI/pythia-160m,dtype=float32` or '{"pretrained":"EleutherAI/pythia-160m","dtype":"float32"}'""", + ) + parser.add_argument( + "--num_fewshot", + "-f", + type=int, + default=None, + metavar="N", + help="Number of examples in few-shot context", + ) + parser.add_argument( + "--batch_size", + "-b", + type=str, + default=1, + metavar="auto|auto:N|N", + help="Acceptable values are 'auto', 'auto:N' or N, where N is an integer. Default 1.", + ) + parser.add_argument( + "--max_batch_size", + type=int, + default=None, + metavar="N", + help="Maximal batch size to try with --batch_size auto.", + ) + parser.add_argument( + "--device", + type=str, + default=None, + help="Device to use (e.g. cuda, cuda:0, cpu).", + ) + parser.add_argument( + "--output_path", + "-o", + default=None, + type=str, + metavar="DIR|DIR/file.json", + help="The path to the output file where the result metrics will be saved. If the path is a directory and log_samples is true, the results will be saved in the directory. Else the parent directory will be used.", + ) + parser.add_argument( + "--limit", + "-L", + type=float, + default=None, + metavar="N|0 argparse.Namespace: + check_argument_types(parser) + return parser.parse_args() + + +def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None: + if not args: + # we allow for args to be passed externally, else we parse them ourselves + parser = setup_parser() + args = parse_eval_args(parser) + + if args.wandb_args: + wandb_args_dict = simple_parse_args_string(args.wandb_args) + wandb_config_args_dict = simple_parse_args_string(args.wandb_config_args) + wandb_logger = WandbLogger(wandb_args_dict, wandb_config_args_dict) + + utils.setup_logging(args.verbosity) + eval_logger = logging.getLogger(__name__) + os.environ["TOKENIZERS_PARALLELISM"] = "false" + + # update the evaluation tracker args with the output path and the HF token + if args.output_path: + args.hf_hub_log_args += f",output_path={args.output_path}" + if os.environ.get("HF_TOKEN", None): + args.hf_hub_log_args += f",token={os.environ.get('HF_TOKEN')}" + evaluation_tracker_args = simple_parse_args_string(args.hf_hub_log_args) + evaluation_tracker = EvaluationTracker(**evaluation_tracker_args) + + if args.predict_only: + args.log_samples = True + if (args.log_samples or args.predict_only) and not args.output_path: + raise ValueError( + "Specify --output_path if providing --log_samples or --predict_only" + ) + + if args.fewshot_as_multiturn and args.apply_chat_template is False: + raise ValueError( + "When `fewshot_as_multiturn` is selected, `apply_chat_template` must be set (either to `True` or to the chosen template name)." + ) + + if args.include_path is not None: + eval_logger.info(f"Including path: {args.include_path}") + metadata = ( + simple_parse_args_string(args.model_args) + if isinstance(args.model_args, str) + else args.model_args + if isinstance(args.model_args, dict) + else {} + ) | ( + args.metadata + if isinstance(args.metadata, dict) + else simple_parse_args_string(args.metadata) + ) + + task_manager = TaskManager(include_path=args.include_path, metadata=metadata) + + if "push_samples_to_hub" in evaluation_tracker_args and not args.log_samples: + eval_logger.warning( + "Pushing samples to the Hub requires --log_samples to be set. Samples will not be pushed to the Hub." + ) + + if args.limit: + eval_logger.warning( + " --limit SHOULD ONLY BE USED FOR TESTING." + "REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT." + ) + + if args.tasks is None: + eval_logger.error("Need to specify task to evaluate.") + sys.exit() + elif args.tasks == "list": + print(task_manager.list_all_tasks()) + sys.exit() + elif args.tasks == "list_groups": + print(task_manager.list_all_tasks(list_subtasks=False, list_tags=False)) + sys.exit() + elif args.tasks == "list_tags": + print(task_manager.list_all_tasks(list_groups=False, list_subtasks=False)) + sys.exit() + elif args.tasks == "list_subtasks": + print(task_manager.list_all_tasks(list_groups=False, list_tags=False)) + sys.exit() + else: + if os.path.isdir(args.tasks): + import glob + + task_names = [] + yaml_path = os.path.join(args.tasks, "*.yaml") + for yaml_file in glob.glob(yaml_path): + config = utils.load_yaml_config(yaml_file) + task_names.append(config) + else: + task_list = args.tasks.split(",") + task_names = task_manager.match_tasks(task_list) + for task in [task for task in task_list if task not in task_names]: + if os.path.isfile(task): + config = utils.load_yaml_config(task) + task_names.append(config) + task_missing = [ + task for task in task_list if task not in task_names and "*" not in task + ] # we don't want errors if a wildcard ("*") task name was used + + if task_missing: + missing = ", ".join(task_missing) + eval_logger.error( + f"Tasks were not found: {missing}\n" + f"{utils.SPACING}Try `lm-eval --tasks list` for list of available tasks", + ) + raise ValueError( + f"Tasks not found: {missing}. Try `lm-eval --tasks {{list_groups,list_subtasks,list_tags,list}}` to list out all available names for task groupings; only (sub)tasks; tags; or all of the above, or pass '--verbosity DEBUG' to troubleshoot task registration issues." + ) + + # Respect user's value passed in via CLI, otherwise default to True and add to comma-separated model args + if args.trust_remote_code: + eval_logger.info( + "Passed `--trust_remote_code`, setting environment variable `HF_DATASETS_TRUST_REMOTE_CODE=true`" + ) + # HACK: import datasets and override its HF_DATASETS_TRUST_REMOTE_CODE value internally, + # because it's already been determined based on the prior env var before launching our + # script--`datasets` gets imported by lm_eval internally before these lines can update the env. + import datasets + + datasets.config.HF_DATASETS_TRUST_REMOTE_CODE = True + + args.model_args = args.model_args + ",trust_remote_code=True" + eval_logger.info( + f"Selected Tasks: {task_names}" + ) if eval_logger.getEffectiveLevel() >= logging.INFO else print( + f"Selected Tasks: {task_names}" + ) + + request_caching_args = request_caching_arg_to_dict( + cache_requests=args.cache_requests + ) + + results = evaluator.simple_evaluate( + model=args.model, + model_args=args.model_args, + tasks=task_names, + num_fewshot=args.num_fewshot, + batch_size=args.batch_size, + max_batch_size=args.max_batch_size, + device=args.device, + use_cache=args.use_cache, + limit=args.limit, + check_integrity=args.check_integrity, + write_out=args.write_out, + log_samples=args.log_samples, + evaluation_tracker=evaluation_tracker, + system_instruction=args.system_instruction, + apply_chat_template=args.apply_chat_template, + fewshot_as_multiturn=args.fewshot_as_multiturn, + gen_kwargs=args.gen_kwargs, + task_manager=task_manager, + predict_only=args.predict_only, + random_seed=args.seed[0], + numpy_random_seed=args.seed[1], + torch_random_seed=args.seed[2], + fewshot_random_seed=args.seed[3], + confirm_run_unsafe_code=args.confirm_run_unsafe_code, + metadata=metadata, + **request_caching_args, + ) + + if results is not None: + if args.log_samples: + samples = results.pop("samples") + dumped = json.dumps( + results, indent=2, default=handle_non_serializable, ensure_ascii=False + ) + if args.show_config: + print(dumped) + + batch_sizes = ",".join(map(str, results["config"]["batch_sizes"])) + + # Add W&B logging + if args.wandb_args: + try: + wandb_logger.post_init(results) + wandb_logger.log_eval_result() + if args.log_samples: + wandb_logger.log_eval_samples(samples) + except Exception as e: + eval_logger.info(f"Logging to Weights and Biases failed due to {e}") + + evaluation_tracker.save_results_aggregated( + results=results, samples=samples if args.log_samples else None + ) + + if args.log_samples: + for task_name, config in results["configs"].items(): + evaluation_tracker.save_results_samples( + task_name=task_name, samples=samples[task_name] + ) + + if ( + evaluation_tracker.push_results_to_hub + or evaluation_tracker.push_samples_to_hub + ): + evaluation_tracker.recreate_metadata_card() + + print( + f"{args.model} ({args.model_args}), gen_kwargs: ({args.gen_kwargs}), limit: {args.limit}, num_fewshot: {args.num_fewshot}, " + f"batch_size: {args.batch_size}{f' ({batch_sizes})' if batch_sizes else ''}" + ) + print(make_table(results)) + if "groups" in results: + print(make_table(results, "groups")) + + if args.wandb_args: + # Tear down wandb run once all the logging is done. + wandb_logger.run.finish() + + +if __name__ == "__main__": + cli_evaluate() diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/api/__init__.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/api/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/api/filter.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/api/filter.py new file mode 100644 index 0000000000000000000000000000000000000000..8d9db6821724c497c4a27116a1238e3b8d32ae29 --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/api/filter.py @@ -0,0 +1,56 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Callable, Iterable, List, Union + +from lm_eval.api.instance import Instance + + +class Filter(ABC): + """ + Filter classes operate on a per-task level. + They take all model outputs (`instance.resps` for all `task.instances`) + across all instances of a task, and perform operations. + In a single run, one can configure any number of separate filters or lists of filters. + + """ + + def __init__(self, **kwargs) -> None: + """ + Can define custom behavior here, if an individual instantiation of a Filter class should have state. + """ + + @abstractmethod + def apply(self, resps: Union[List, Iterable], docs: List[dict]) -> Iterable: + """ + Defines the operation to perform on a list of the `inst.resps` properties of `Instance` objects. + Should return the list of (filtered) response lists *in the same order as they were input*, e.g. + if pass in [, ] should return + [, ] + """ + return resps + + +@dataclass +class FilterEnsemble: + """ + FilterEnsemble creates a pipeline applying multiple filters. + Its intended usage is to stack multiple post-processing steps in order. + `task.apply_filters` should use a list of FilterEnsemble classes that it stores, to apply each + pipeline separately. + """ + + name: str + filters: List[Callable[[], Filter]] + + def apply(self, instances: List[Instance]) -> None: + resps, docs = zip(*((inst.resps, inst.doc) for inst in instances)) + resps, docs = list(resps), list(docs) + + for f in self.filters: + # apply filters in sequence + resps = f().apply(resps, docs) + + # add the end results after filtering to filtered_requests of their respective source instances. + # has key `self.name`: each FilterEnsemble applied in a given run should use a different name. + for inst, resp in zip(instances, resps): + inst.filtered_resps[self.name] = resp diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/api/group.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/api/group.py new file mode 100644 index 0000000000000000000000000000000000000000..0c60739bbd26c79ecab91f54240798b2ae9e3313 --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/api/group.py @@ -0,0 +1,115 @@ +import abc +from dataclasses import asdict, dataclass +from inspect import getsource +from typing import Any, Callable, List, Optional, Union + + +@dataclass +class AggMetricConfig(dict): + metric: Optional[str] = None + aggregation: Optional[str] = "mean" + weight_by_size: Optional[str] = False + # list of filter names which should be incorporated into the aggregated metric. + filter_list: Optional[Union[str, list]] = "none" + + def __post_init__(self): + if self.aggregation != "mean" and not callable(self.aggregation): + raise ValueError( + f"Currently, 'mean' is the only pre-defined aggregation across groups' subtasks. Got '{self.aggregation}'." + ) + + if isinstance(self.filter_list, str): + self.filter_list = [self.filter_list] + + +@dataclass +class GroupConfig(dict): + group: Optional[str] = None + group_alias: Optional[str] = None + task: Optional[Union[str, list]] = None + aggregate_metric_list: Optional[ + Union[List[AggMetricConfig], AggMetricConfig, dict] + ] = None + metadata: Optional[dict] = ( + None # by default, not used in the code. allows for users to pass arbitrary info to tasks + ) + + def __getitem__(self, item): + return getattr(self, item) + + def __setitem__(self, item, value): + return setattr(self, item, value) + + def __post_init__(self): + if self.aggregate_metric_list is not None: + if isinstance(self.aggregate_metric_list, dict): + self.aggregate_metric_list = [self.aggregate_metric_list] + + self.aggregate_metric_list = [ + AggMetricConfig(**item) if isinstance(item, dict) else item + for item in self.aggregate_metric_list + ] + + def to_dict(self, keep_callable: bool = False) -> dict: + """dumps the current config as a dictionary object, as a printable format. + null fields will not be printed. + Used for dumping results alongside full task configuration + + :return: dict + A printable dictionary version of the TaskConfig object. + + # TODO: should any default value in the TaskConfig not be printed? + """ + cfg_dict = asdict(self) + # remove values that are `None` + for k, v in list(cfg_dict.items()): + if callable(v): + cfg_dict[k] = self.serialize_function(v, keep_callable=keep_callable) + return cfg_dict + + def serialize_function( + self, value: Union[Callable, str], keep_callable=False + ) -> Union[Callable, str]: + """Serializes a given function or string. + + If 'keep_callable' is True, the original callable is returned. + Otherwise, attempts to return the source code of the callable using 'getsource'. + """ + if keep_callable: + return value + else: + try: + return getsource(value) + except (TypeError, OSError): + return str(value) + + +class ConfigurableGroup(abc.ABC): + def __init__( + self, + config: Optional[dict] = None, + ) -> None: + self._config = GroupConfig(**config) + + @property + def group(self): + return self._config.group + + @property + def group_alias(self): + return self._config.group_alias + + @property + def version(self): + return self._config.version + + @property + def config(self): + return self._config.to_dict() + + @property + def group_name(self) -> Any: + return self._config.group + + def __repr__(self): + return f"ConfigurableGroup(group={self.group},group_alias={self.group_alias})" diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/api/instance.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/api/instance.py new file mode 100644 index 0000000000000000000000000000000000000000..d3c6afa0644e729ba441728c72a2469fdad07b8f --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/api/instance.py @@ -0,0 +1,38 @@ +from dataclasses import dataclass, field +from typing import Literal, Optional, Tuple + + +OutputType = Literal[ + "loglikelihood", "loglikelihood_rolling", "generate_until", "multiple_choice" +] + + +@dataclass +class Instance: + request_type: OutputType + doc: dict + arguments: tuple + idx: int + metadata: Tuple[Optional[str], Optional[int], Optional[int]] = field( + default_factory=lambda: (None, None, None) + ) + resps: list = field(default_factory=list) + filtered_resps: dict = field(default_factory=dict) + + # initialized after init + task_name: Optional[str] = None + doc_id: Optional[int] = None + repeats: Optional[int] = None + + def __post_init__(self) -> None: + # unpack metadata field + self.task_name, self.doc_id, self.repeats = self.metadata + + @property + def args(self): + """ + Returns (string,) where `string` is the string to calculate loglikelihood over + """ + return ( + self.arguments if isinstance(self.arguments, tuple) else (self.arguments,) + ) diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/api/metrics.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/api/metrics.py new file mode 100644 index 0000000000000000000000000000000000000000..61fca5e19d376502f3b75aa5328045cee6ee5454 --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/api/metrics.py @@ -0,0 +1,578 @@ +import logging +import math +import random +import re +import string +from collections.abc import Iterable +from typing import List + +import numpy as np +import sacrebleu + +from lm_eval.api.registry import register_aggregation, register_metric + + +eval_logger = logging.getLogger(__name__) + + +# Register Aggregations First +@register_aggregation("bypass") +def bypass_agg(arr): + return 999 + + +@register_aggregation("nanmean") +def nanmean(arr): + if len(arr) == 0 or all(np.isnan(arr)): + return np.nan + return np.nanmean(arr) + + +@register_aggregation("mean") +def mean(arr): + return sum(arr) / len(arr) + + +@register_aggregation("median") +def median(arr): + return arr[len(arr) // 2] + + +# Certain metrics must be calculated across all documents in a benchmark. +# We use them as aggregation metrics, paired with no-op passthrough metric fns. +@register_aggregation("perplexity") +def perplexity(items): + return math.exp(-mean(items)) + + +@register_aggregation("weighted_perplexity") +def weighted_perplexity(items): + return math.exp(-weighted_mean(items)) + + +@register_aggregation("bits_per_byte") +def bits_per_byte(items): + return -weighted_mean(items) / math.log(2) + + +@register_aggregation("f1") +def f1_score(items): + from sklearn.metrics import f1_score + + unzipped_list = list(zip(*items)) + golds = unzipped_list[0] + preds = unzipped_list[1] + fscore = f1_score(golds, preds) + + return np.max(fscore) + + +@register_aggregation("matthews_corrcoef") +def matthews_corrcoef(items): + from sklearn.metrics import matthews_corrcoef + + unzipped_list = list(zip(*items)) + golds = unzipped_list[0] + preds = unzipped_list[1] + return matthews_corrcoef(golds, preds) + + +@register_aggregation("bleu") +def bleu(items): + """The Bilingual Evaluation Understudy Score, or BLEU for short, is a metric + for evaluating a generated sentence to a reference sentence. It counts matching + n-grams in the candidate translation to n-grams in the reference text, where + 1-gram or unigram would be each token and a bigram comparison would be each + word pair. The comparison is made regardless of word order + Source: https://machinelearningmastery.com/calculate-bleu-score-for-text-python/ + Paper: https://www.aclweb.org/anthology/P02-1040/ + + Higher is better + """ + refs = list(zip(*items))[0] + preds = list(zip(*items))[1] + refs, preds = _sacreformat(refs, preds) + return sacrebleu.corpus_bleu(preds, refs).score + + +@register_aggregation("chrf") +def chrf(items): + """chrF++ is a tool for automatic evaluation of machine translation output + based on character n-gram precision and recall enhanced with word n-grams. + Source: https://github.com/m-popovic/chrF + Paper: https://www.aclweb.org/anthology/W15-3049.pdf + + Higher is better # TODO I think + """ + refs = list(zip(*items))[0] + preds = list(zip(*items))[1] + refs, preds = _sacreformat(refs, preds) + return sacrebleu.corpus_chrf(preds, refs).score + + +@register_aggregation("ter") +def ter(items): + """Translation Error Rate is an error metric for machine translation that + measures the number of edits required to change a system output into one + of the references + Source: http://www.cs.umd.edu/~snover/tercom/ + Paper: http://mt-archive.info/AMTA-2006-Snover.pdf + + Lower is better + """ + refs = list(zip(*items))[0] + preds = list(zip(*items))[1] + refs, preds = _sacreformat(refs, preds) + return sacrebleu.corpus_ter(preds, refs).score + + +@register_aggregation("brier_score") +def brier_score(items): # This is a passthrough function + gold, predictions = list(zip(*items)) + bs, num_class = np.array(predictions).shape + + gold = list(gold) + gold_one_hot = np.eye(num_class)[gold] + return np.mean(np.sum((predictions - gold_one_hot) ** 2, axis=1)) + + +@register_metric( + metric="brier_score", + higher_is_better=False, + output_type=["multiple_choice"], + aggregation="brier_score", +) +def brier_score_fn(items): # This is a passthrough function + return items + + +@register_metric( + metric="acc", + higher_is_better=True, + output_type=["loglikelihood", "multiple_choice"], + aggregation="mean", +) +def acc_fn(items): # This is a passthrough function + return items + + +@register_metric( + metric="acc_norm", + higher_is_better=True, + output_type=["loglikelihood", "multiple_choice"], + aggregation="mean", +) +def acc_norm_fn(items): # This is a passthrough function + return items + + +@register_metric( + metric="acc_mutual_info", + higher_is_better=True, + output_type="multiple_choice", + aggregation="mean", +) +def acc_mutual_info_fn(items): # This is a passthrough function + return items + + +### the code used in the `exact_match_hf_evaluate` function is ported from +### https://github.com/huggingface/evaluate/blob/main/metrics/exact_match/exact_match.py +### which is under the apache license. + +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +def exact_match_hf_evaluate( + predictions, + references, + regexes_to_ignore=None, + ignore_case=False, + ignore_punctuation=False, + ignore_numbers=False, +): + if regexes_to_ignore is not None: + for s in regexes_to_ignore: + predictions = np.array([re.sub(s, "", x) for x in predictions]) + references = np.array([re.sub(s, "", x) for x in references]) + else: + predictions = np.asarray(predictions) + references = np.asarray(references) + + if ignore_case: + predictions = np.char.lower(predictions) + references = np.char.lower(references) + + if ignore_punctuation: + repl_table = string.punctuation.maketrans("", "", string.punctuation) + predictions = np.char.translate(predictions, table=repl_table) + references = np.char.translate(references, table=repl_table) + + if ignore_numbers: + repl_table = string.digits.maketrans("", "", string.digits) + predictions = np.char.translate(predictions, table=repl_table) + references = np.char.translate(references, table=repl_table) + + score_list = predictions == references + + return {"exact_match": np.mean(score_list)} + + +### + + +@register_metric( + metric="exact_match", + higher_is_better=True, + output_type="generate_until", + aggregation="mean", +) +def exact_match_fn(**kwargs): + return exact_match_hf_evaluate(**kwargs) + + +@register_metric( + metric="perplexity", + higher_is_better=False, + output_type="loglikelihood", + aggregation="perplexity", +) +def perplexity_fn(items): # This is a passthrough function + return items + + +@register_metric( + metric="word_perplexity", + higher_is_better=False, + output_type="loglikelihood_rolling", + aggregation="weighted_perplexity", +) +def word_perplexity_fn(items): # This is a passthrough function + return items + + +@register_metric( + metric="byte_perplexity", + higher_is_better=False, + output_type="loglikelihood_rolling", + aggregation="weighted_perplexity", +) +def byte_perplexity_fn(items): # This is a passthrough function + return items + + +@register_metric( + metric="bits_per_byte", + higher_is_better=False, + output_type="loglikelihood_rolling", + aggregation="bits_per_byte", +) +def bits_per_byte_fn(items): # This is a passthrough function + return items + + +def pop_stddev(arr): + mu = mean(arr) + return math.sqrt(sum([(x - mu) ** 2 for x in arr]) / len(arr)) + + +def sample_stddev(arr): + mu = mean(arr) + return math.sqrt(sum([(x - mu) ** 2 for x in arr]) / (len(arr) - 1)) + + +def mean_stderr(arr): + return sample_stddev(arr) / math.sqrt(len(arr)) + + +@register_metric( + metric="bypass", + higher_is_better=True, + output_type=["loglikelihood", "multiple_choice", "generate_until"], + aggregation="bypass", +) +def bypass(items): + return None + + +@register_metric( + metric="mcc", + higher_is_better=True, + output_type="multiple_choice", + aggregation="matthews_corrcoef", +) +def mcc_fn(items): # This is a passthrough function + return items + + +@register_metric( + metric="f1", + higher_is_better=True, + output_type="multiple_choice", + aggregation="f1", +) +def f1_fn(items): # This is a passthrough function + return items + + +@register_metric( + metric="bleu", + higher_is_better=True, + output_type="generate_until", + aggregation="bleu", +) +def bleu_fn(items): # This is a passthrough function + return items + + +@register_metric( + metric="chrf", + higher_is_better=True, + output_type="generate_until", + aggregation="chrf", +) +def chrf_fn(items): # This is a passthrough function + return items + + +@register_metric( + metric="ter", + higher_is_better=True, + output_type="generate_until", + aggregation="ter", +) +def ter_fn(items): # This is a passthrough function + return items + + +@register_metric( + metric="acc_all", + higher_is_better=True, + output_type="loglikelihood", + aggregation="mean", +) +def acc_all(items): + # Only count as correct if all answers are labeled correctly for each question + question_scoring_dict = {} + preds = list(zip(*items))[0] + docs = list(zip(*items))[1] + + for doc, pred in zip(docs, preds): + paragraph_id = doc["idx"]["paragraph"] + question_id = doc["idx"]["question"] + if (paragraph_id, question_id) not in question_scoring_dict: + question_scoring_dict[(paragraph_id, question_id)] = [] + + gold_label = doc["label"] == 1 + + question_scoring_dict[(paragraph_id, question_id)].append(gold_label == pred) + acc = np.mean([int(all(x)) for x in question_scoring_dict.values()]) + return acc + + +def acc_all_stderr(items): + # Only count as correct if all answers are labeled correctly for each question + question_scoring_dict = {} + preds = list(zip(*items))[0] + docs = list(zip(*items))[1] + + for doc, pred in zip(docs, preds): + question_id = doc["idx"]["question"] + if question_id not in question_scoring_dict: + question_scoring_dict[question_id] = [] + + gold_label = doc["label"] == 1 + question_scoring_dict[question_id].append(gold_label == pred) + + acc = mean_stderr([int(all(x)) for x in question_scoring_dict.values()]) + return acc + + +def metric_max_over_ground_truths(metric_fn, prediction, ground_truths): + """Compute max metric between prediction and each ground truth.""" + scores_for_ground_truths = [] + for ground_truth in ground_truths: + score = metric_fn(prediction, ground_truth) + scores_for_ground_truths.append(score) + return max(scores_for_ground_truths) + + +def weighted_mean(items): + a, b = zip(*items) + return sum(a) / sum(b) + + +def is_non_str_iterable(obj): + return isinstance(obj, Iterable) and not isinstance(obj, str) + + +def _sacreformat(refs, preds): + """Format refs and preds for sacrebleu corpus calculation. It is very particular""" + # Sacrebleu expects (List[str], List[List[str]) + # e.g. sacrebleu.corpus_bleu([pred_t], [[ref1_stream], [ref2_stream], ...]) + + # Note [ref1_stream] is the first reference for each pred. + # So lists are size N and (M, N) for N preds and M possible refs for each pred + # This is a different order of dimensions that I would expect + + # We expect refs to be List[str] or List[List[str]], the outer list corresponding to preds + # Must become List[List[str]] with the inner list corresponding to preds + if not is_non_str_iterable(refs): + refs = list(refs) + if not is_non_str_iterable(refs[0]): + refs = [[ref] for ref in refs] + refs = list(zip(*refs)) + # Note the number of refs in each ref list much match the number of preds + + # We expect preds to be List[str] or List[List[str]]. Must become List[str] + if not is_non_str_iterable(preds): + preds = list(preds) + if is_non_str_iterable(preds[0]): + assert len(preds[0]) == 1, f"Pred must be a str, was {preds[0]}" + preds = [pred[0] for pred in preds] + + return refs, preds + + +# stderr stuff + + +class _bootstrap_internal: + def __init__(self, f, n) -> None: + self.f = f + self.n = n + + def __call__(self, v): + i, xs = v + rnd = random.Random() + rnd.seed(i) + res = [] + for _ in range(self.n): + res.append(self.f(rnd.choices(xs, k=len(xs)))) + return res + + +def bootstrap_stderr(f, xs, iters): + import multiprocessing as mp + + pool = mp.Pool(mp.cpu_count()) + # this gives a biased estimate of the stderr (i.e w/ the mean, it gives something + # equivalent to stderr calculated without Bessel's correction in the stddev. + # Unfortunately, I haven't been able to figure out what the right correction is + # to make the bootstrap unbiased - i considered multiplying by sqrt(n/(n-1)) but + # that would be ad-hoc and I can't prove that that would actually be an unbiased estimator) + # Thankfully, shouldn't matter because our samples are pretty big usually anyways + res = [] + chunk_size = min(1000, iters) + from tqdm import tqdm + + print("bootstrapping for stddev:", f.__name__) + for bootstrap in tqdm( + pool.imap( + _bootstrap_internal(f, chunk_size), + [(i, xs) for i in range(iters // chunk_size)], + ), + total=iters // chunk_size, + ): + # sample w replacement + res.extend(bootstrap) + + pool.close() + return sample_stddev(res) + + +def stderr_for_metric(metric, bootstrap_iters: int): + if bootstrap_iters <= 0: + # return no function (don't compute stderr) if bootstrap iters = 0 + return None + + bootstrappable = [ + median, + matthews_corrcoef, + f1_score, + perplexity, + bleu, + chrf, + ter, + nanmean, + ] + + if metric in bootstrappable: + return lambda x: bootstrap_stderr(metric, x, iters=bootstrap_iters) + + stderr = {mean: mean_stderr, acc_all: acc_all_stderr} + + return stderr.get(metric, None) + + +def pooled_sample_stderr(stderrs: List[float], sizes: List[int]): + # Used to aggregate bootstrapped stderrs across subtasks in a group, + # when we are weighting by the size of each subtask. + # + + assert len(stderrs) == len(sizes) + + # formula source: https://en.wikipedia.org/wiki/Pooled_variance + # and: https://stats.stackexchange.com/a/4841331 + # this empirically seems to match running `stderr_for_metric` on all instances + # from the subtasks concatenated with each other. + pooled_sample_var = ( + sum([(size - 1) * stderr**2 * size for size, stderr in zip(sizes, stderrs)]) + ) / (sum(sizes) - len(sizes)) + + return np.sqrt(pooled_sample_var / sum(sizes)) + + +def combined_sample_stderr(stderrs: List[float], sizes: List[int], metrics=None): + assert metrics is not None, ( + "Need to pass a list of each subtask's metric for this stderr aggregation" + ) + assert len(stderrs) == len(sizes) and len(sizes) == len(metrics) + + # See https://github.com/EleutherAI/lm-evaluation-harness/pull/1390 for more documentation. + # This formula depends on sample means. + # removed because it seems to give erroneously huge stderrs for groupings of tasks + # and does not seem to match up with bootstrap-calculated stderrs for groups. + + ### don't use this unless a statistician has told you it's the right thing to do ### + + # accumulators: we'll aggregate pairwise N - 1 times + variance = stderrs[0] ** 2 + curr_size = sizes[0] + curr_score = metrics[0] + + for stderr, size, score in zip(stderrs[1:], sizes[1:], metrics[1:]): + curr_score = ((curr_score * curr_size) + (score * size)) / ( + curr_size + size + ) # NOTE: this assumes our aggregation fn is "mean" + + variance = ((curr_size - 1) * variance + (size - 1) * (stderr**2)) / ( + curr_size + size - 1 + ) + curr_size * size / ((curr_size + size) * (curr_size + size - 1)) * ( + curr_score - score + ) ** 2 + + return np.sqrt(variance) + + +def aggregate_subtask_metrics(metrics, sizes, weight_by_size=True): + # A helper function that is used to aggregate + # subtask scores cross-task. + # TODO: does not hold for non-mean aggregations + if not weight_by_size: + sizes = [1] * len(sizes) + + assert len(metrics) == len(sizes) + + return sum([metric * size for metric, size in zip(metrics, sizes)]) / sum(sizes) diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/api/model.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/api/model.py new file mode 100644 index 0000000000000000000000000000000000000000..1debc9b4da2ed307951e50bf2351af971937e59a --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/api/model.py @@ -0,0 +1,493 @@ +import abc +import hashlib +import json +import logging +import os +from typing import Dict, List, Optional, Tuple, Type, TypeVar, Union + +import transformers +from sqlitedict import SqliteDict +from tqdm import tqdm + +from lm_eval import utils + + +eval_logger = logging.getLogger(__name__) + +T = TypeVar("T", bound="LM") + + +class LM(abc.ABC): + def __init__(self) -> None: + """Defines the interface that should be implemented by all LM subclasses. + LMs are assumed to take text (strings) as input and yield strings as output + (inputs/outputs should be tokenization-agnostic.) + + """ + # set rank and world size to a single process, by default. + self._rank = 0 + self._world_size = 1 + self.cache_hook = CacheHook(None) + + @abc.abstractmethod + def loglikelihood(self, requests) -> List[Tuple[float, bool]]: + """Compute log-likelihood of generating a continuation from a context. + Downstream tasks should attempt to use loglikelihood instead of other + LM calls whenever possible. + + :param requests: list[Instance] + A list of Instance objects, with property `args` which returns a tuple (context, continuation). + `context: str` + Context string. Implementations of LM must be able to handle an + empty context string. + `continuation: str` + The continuation over which log likelihood will be calculated. If + there is a word boundary, the space should be in the continuation. + For example, context="hello" continuation=" world" is correct. + + :return: list[tuple[float, bool]] + A list of pairs (logprob, isgreedy) + `logprob: float` + The log probability of `continuation`. + `isgreedy`: + Whether `continuation` would be generated by greedy sampling from `context`. + """ + pass + + @abc.abstractmethod + def loglikelihood_rolling(self, requests) -> List[float]: + """Compute full log-likelihood of a string, with no truncation, for perplexity computation + - We will use the full max context length of the model. + - For inputs that exceed the max context length, we divide the tokenized string into chunks of up to + the max context length. + - IMPORTANT: Each document's loglikelihood/perplexity is computed *separately*, unlike other implementations + which may simply concatenate multiple documents together. + - IMPORTANT: We maximize the amount of context for each prediction. Specifically, for inputs that we break into + multiple chunks, the last input will still a full-sized context. + Example: + Input tokens: [ 0 1 2 3 4 5 6 7 8 9 ] + Prefix: BOS/EOS + Max context length: 4 + Resulting input/prediction pairs: + + INPUT: BOS 0 1 2 + PRED: 0 1 2 3 + + INPUT: 3 4 5 6 + PRED: 4 5 6 7 + + INPUT: 5 6 7 8 + PRED: 8 9 + + Observe that: + 1. Each token is predicted exactly once + 2. For the last pair, we provide the full context, but only score the last two tokens + + :param requests: list[Instance] + A list of Instance objects with property `args` which returns a tuple (context,). + string: str + String for which we are computing overall loglikelihood + :return: list[tuple[float]] + A list of tuples (logprob,) + logprob: float + The log probability of `context` conditioned on the BOS/EOS token. + Can also be overridden for custom cases by `prefix_token_id`. + """ + pass + + # TODO: Add an optional max length + @abc.abstractmethod + def generate_until(self, requests) -> List[str]: + """Generate greedily until a stopping sequence + + :param requests: list[Instance] + A list of Instance objects with property `args` which returns a tuple (context, gen_kwargs). + context: str + Context string + gen_kwargs: dict + A dictionary of keyword arguments to pass to the generation function e.g. top_k, until, etc. + :return: list[str] + A list of model generated continuations. + continuation: str + The generated continuation. + """ + pass + + def apply_chat_template( + self, chat_history: List[Dict[str, str]], add_generation_prompt=True + ) -> str: + """ + Defines how to transform few-shot examples provided as chat history into a format that can be used as input to the LM. + + :param chat_history: list[dict[str, str]] + A list of dictionaries with keys 'role' and 'content'. + Values are strings representing the role name and the content of the message, respectively. + :param add_generation_prompt: bool + Whether to append an assistant gen prefix (for e.g. <|assistant|>) to the assistant messages in the chat history. False if prefilling an assistant message. + :return: str + A string representing the chat history in a format that can be used as input to the LM. + """ + raise NotImplementedError( + "To use this model with chat templates, please implement the 'apply_chat_template' method for your model type." + ) + + @classmethod + def create_from_arg_string( + cls: Type[T], arg_string: str, additional_config: Optional[dict] = None + ) -> T: + """ + Creates an instance of the LM class using the given argument string and additional config. + + Parameters: + - arg_string: A string containing arguments in the format key1=value1,key2=value2. + - additional_config: Optional dictionary containing additional configuration parameters. + + Returns: + - Instance of the LM class. + """ + additional_config = {} if additional_config is None else additional_config + args = utils.simple_parse_args_string(arg_string) + args2 = {k: v for k, v in additional_config.items() if v is not None} + return cls(**args, **args2) + + @classmethod + def create_from_arg_obj( + cls: Type[T], arg_dict: dict, additional_config: Optional[dict] = None + ) -> T: + """ + Creates an instance of the LM class using the given arg_obj + + Parameters: + - arg_obj: A dict containing arguments in the format key1=value1,key2=value2. + - additional_config: Optional dictionary containing additional configuration parameters. + + Returns: + - Instance of the LM class. + """ + + additional_config = {} if additional_config is None else additional_config + additional_config = { + k: v for k, v in additional_config.items() if v is not None + } + + return cls(**arg_dict, **additional_config) + + @property + def rank(self): + # used in the case of parallelism. Hardcoded to + # ensure no errors arise using API models which do + # not support multi-device parallelism nor expect it. + return self._rank + + @property + def world_size(self): + # used in the case of parallelism. Hardcoded to + # ensure no errors arise using API models which do + # not support multi-device parallelism nor expect it. + return self._world_size + + @property + def tokenizer_name(self) -> str: + """Must be defined for LM subclasses which implement Chat Templating. + Should return the name of the tokenizer or chat template used. + Used only to properly fingerprint caches when requests are being cached with `--cache_requests`, otherwise not used. + """ + raise NotImplementedError( + "To use this model with chat templates, please implement the 'tokenizer_name' property." + ) + + def chat_template(self, chat_template: Union[bool, str] = False) -> Optional[str]: + """Returns the chat template structure for user/assistant messages if a template is provided. + This method is intended to be overridden in a subclass to define a specific chat template format. + For models that do not support chat templates, this method returns None by default. + """ + + return "" + + def set_cache_hook(self, cache_hook) -> None: + self.cache_hook = cache_hook + + +### SQLite-based caching of LM responses +def hash_args(attr, args): + dat = json.dumps([attr] + list(args)) + return hashlib.sha256(dat.encode("utf-8")).hexdigest() + + +class CacheHook: + def __init__(self, cachinglm) -> None: + if cachinglm is None: + self.dbdict = None + return + + self.dbdict = cachinglm.dbdict + + def add_partial(self, attr, req, res) -> None: + if self.dbdict is None: + return + hsh = hash_args(attr, req) + self.dbdict[hsh] = res + + +class CachingLM: + def __init__(self, lm, cache_db) -> None: + """LM wrapper that returns cached results if they exist, and uses the underlying LM if not. + + :param lm: LM + Underlying LM + :param cache_db: str + Path to cache db + """ + self.lm = lm + self.cache_db = cache_db + if os.path.dirname(cache_db): + os.makedirs(os.path.dirname(cache_db), exist_ok=True) + self.dbdict = SqliteDict(cache_db, autocommit=True) + + # add hook to lm + lm.set_cache_hook(self.get_cache_hook()) + + def __getattr__(self, attr: str): + lm_attr = getattr(self.lm, attr) + if attr not in ["loglikelihood", "loglikelihood_rolling", "generate_until"]: + eval_logger.debug(f"Passing through attribute '{attr}' to underlying LM") + return lm_attr + + def fn(requests): + res = [] + remaining_reqs = [] + warned = False + # figure out which ones are cached and which ones are new + eval_logger.info( + f"Loading '{attr}' responses from cache '{self.cache_db}' where possible..." + ) + for req in tqdm(requests, desc="Checking cached requests"): + hsh = hash_args(attr, req.args) + if attr == "generate_until" and req.args[1].get("do_sample", False): + # when we are doing non-greedy generation, don't use the cache + # (else every "randomly sampled" generation would be identical for repeats > 1). + if not warned: + eval_logger.warning( + f"Arguments to lm.generate_until() '{req.args[1]}' include non-deterministic sampling. Caching will not be performed for such requests." + ) + warned = True + res.append(None) + remaining_reqs.append(req) + elif hsh in self.dbdict: + ob = self.dbdict[hsh] + + assert ob is not None + + res.append(ob) + else: + res.append(None) + remaining_reqs.append(req) + eval_logger.info( + f"Cached requests: {len(requests) - len(remaining_reqs)}, Requests remaining: {len(remaining_reqs)}" + ) + if remaining_reqs: + # actually run the LM on the requests that do not have cached results + rem_res = getattr(self.lm, attr)(remaining_reqs) + else: + rem_res = [] + + # stick the new ones back into the list and also cache any of the new ones + resptr = 0 + for req, r in zip(remaining_reqs, rem_res): + while res[resptr] is not None: + resptr += 1 + + res[resptr] = r + + # caching + hsh = hash_args(attr, req.args) + self.dbdict[hsh] = r + self.dbdict.commit() + + return res + + return fn + + def get_cache_hook(self): + return CacheHook(self) + + +class TemplateLM(LM): + """ + A class acting as intermediary between the LM base class + and boilerplate often included in other LM subclasses. + """ + + tokenizer = None + + @property + @abc.abstractmethod + def eot_token_id(self): + pass + + @property + def prefix_token_id(self): + # it is used as prefix for loglikelihood + return self.eot_token_id + + @abc.abstractmethod + def tok_encode(self, string: str, **kwargs) -> List[int]: + """ + Tokenize a string using the model's tokenizer and return a list of token IDs. + """ + pass + + @abc.abstractmethod + def _loglikelihood_tokens(self, requests, **kwargs) -> List[Tuple[float, bool]]: + pass + + def _encode_pair( + self, context: str, continuation: str + ) -> Tuple[List[int], List[int]]: + n_spaces = len(context) - len(context.rstrip()) + if n_spaces > 0: + continuation = context[-n_spaces:] + continuation + context = context[:-n_spaces] + + model_class = getattr(self, "AUTO_MODEL_CLASS", None) + + if model_class == transformers.AutoModelForSeq2SeqLM: + context_enc = self.tok_encode(context) + continuation_enc = self.tok_encode(continuation, add_special_tokens=False) + else: + whole_enc = self.tok_encode(context + continuation) + context_enc = self.tok_encode(context) + + context_enc_len = len(context_enc) + continuation_enc = whole_enc[context_enc_len:] + + return context_enc, continuation_enc + + def loglikelihood( + self, requests, disable_tqdm: bool = False + ) -> List[Tuple[float, bool]]: + new_reqs = [] + for context, continuation in [req.args for req in requests]: + if context == "": + # BOS or EOS as context + context_enc, continuation_enc = ( + [self.prefix_token_id], + self.tok_encode(continuation), + ) + else: + context_enc, continuation_enc = self._encode_pair(context, continuation) + + new_reqs.append(((context, continuation), context_enc, continuation_enc)) + + return self._loglikelihood_tokens(new_reqs, disable_tqdm=disable_tqdm) + + @abc.abstractmethod + def loglikelihood_rolling( + self, requests, disable_tqdm: bool = False + ) -> List[float]: + pass + + @abc.abstractmethod + def generate_until(self, requests, disable_tqdm: bool = False) -> List[str]: + pass + + def chat_template(self, chat_template: Union[bool, str] = False) -> Optional[str]: + """ + Set and get the appropriate chat template for the model. + This method sets the tokenizer's chat_template and returns the template string for reproducibility. + + The template selection logic is adapted from the Transformers library's `apply_chat_template` + method in the Tokenizer class. The original implementation can be found at: + https://github.com/huggingface/transformers/blob/fc35907f95459d7a6c5281dfadd680b6f7b620e3/src/transformers/tokenization_utils_base.py#L1687 + + This method ensures that the right template is chosen based on the following: + 0. If the model has no 'tokenizer' attribute: assumes that there is only a single possible chat template, handled on the model provider side internally. Returns the empty string. + 1. If the model's tokenizer has multiple templates: + a. Use the specified template if it exists in the dictionary. + b. Use the default template from the list if no specific template is provided. + c. Raise an error if no default template exists and no specific template is provided. + 2. If the model's tokenizer has a single template or no template: + a. Use the tokenizer's chat template if available. + b. Fall back to the default chat template if no tokenizer chat template exists. + + Args: + chat_template (Union[bool, str]): Specifies the chat template to use. + - If False or None, no template is applied. + - If True, the default or only available template is used. + - If a string, the template with the matching name is used. + + Returns: + Optional[str]: The selected chat template, or None if no template is applied. + """ + if self.tokenizer is None: + return "" + + if chat_template is False or chat_template is None: + eval_logger.warning( + "model.chat_template was called with the chat_template set to False or None. " + "Therefore no chat template will be applied. Make sure this is an intended behavior." + ) + return None + + # Convert boolean chat_template to None to ensure compatibility with the adapted logic + if isinstance(chat_template, bool): + chat_template = None + using_default_template = False + + # First, handle the cases when the model has a dict of multiple templates + try: + template = ( + self.tokenizer.chat_template or self.tokenizer.default_chat_template + ) + except AttributeError: + return None + + if isinstance(template, dict): + using_default_dict = self.tokenizer.chat_template is None + + if chat_template is not None: + if chat_template in template: + selected_template = template[chat_template] + if using_default_dict: + using_default_template = True + else: + raise ValueError( + f"The specified chat template '{chat_template}' is not available. " + f"Available template names are {sorted(template.keys())}." + ) + else: + # If user didn't pass a chat template, use the default template from the dict + if "default" in template: + selected_template = template["default"] + using_default_template = True + else: + raise ValueError( + "This model has multiple chat templates with no default specified! Please either pass a chat " + "template or the name of the template you wish to use to the `chat_template` argument. Available " + f"template names are {sorted(template.keys())}." + ) + + # Cases when the model has a single template or no template + else: + # priority: `chat_template` argument > `tokenizer.chat_template` > `tokenizer.default_chat_template + if isinstance(chat_template, str): + eval_logger.warning( + "Chat template name provided, but the tokenizer's chat template is not a dictionary. " + "Using the tokenizer's chat template or the default template instead." + ) + if self.tokenizer.chat_template is not None: + selected_template = self.tokenizer.chat_template + else: + selected_template = self.tokenizer.default_chat_template + using_default_template = True + + if using_default_template: + eval_logger.warning( + "No chat template is set for this tokenizer, falling back to a default class-level template. This is " + "very error-prone, because models are often trained with templates different from the class default! " + "Default chat templates are a legacy feature and will be removed in Transformers v4.43, at which " + "point any code depending on them will stop working. We recommend setting a valid chat template before " + "then to ensure that this model continues working without issues." + ) + + return selected_template diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/api/registry.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/api/registry.py new file mode 100644 index 0000000000000000000000000000000000000000..4673b157b1fc1eaed2eb40e7a1ad527ce1fcb595 --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/api/registry.py @@ -0,0 +1,196 @@ +import logging +from typing import Callable, Dict, Union + +import evaluate as hf_evaluate + +from lm_eval.api.model import LM + + +eval_logger = logging.getLogger(__name__) + +MODEL_REGISTRY = {} + + +def register_model(*names): + # either pass a list or a single alias. + # function receives them as a tuple of strings + + def decorate(cls): + for name in names: + assert issubclass(cls, LM), ( + f"Model '{name}' ({cls.__name__}) must extend LM class" + ) + + assert name not in MODEL_REGISTRY, ( + f"Model named '{name}' conflicts with existing model! Please register with a non-conflicting alias instead." + ) + + MODEL_REGISTRY[name] = cls + return cls + + return decorate + + +def get_model(model_name): + try: + return MODEL_REGISTRY[model_name] + except KeyError: + raise ValueError( + f"Attempted to load model '{model_name}', but no model for this name found! Supported model names: {', '.join(MODEL_REGISTRY.keys())}" + ) + + +TASK_REGISTRY = {} +GROUP_REGISTRY = {} +ALL_TASKS = set() +func2task_index = {} + + +def register_task(name): + def decorate(fn): + assert name not in TASK_REGISTRY, ( + f"task named '{name}' conflicts with existing registered task!" + ) + + TASK_REGISTRY[name] = fn + ALL_TASKS.add(name) + func2task_index[fn.__name__] = name + return fn + + return decorate + + +def register_group(name): + def decorate(fn): + func_name = func2task_index[fn.__name__] + if name in GROUP_REGISTRY: + GROUP_REGISTRY[name].append(func_name) + else: + GROUP_REGISTRY[name] = [func_name] + ALL_TASKS.add(name) + return fn + + return decorate + + +OUTPUT_TYPE_REGISTRY = {} +METRIC_REGISTRY = {} +METRIC_AGGREGATION_REGISTRY = {} +AGGREGATION_REGISTRY: Dict[str, Callable[[], Dict[str, Callable]]] = {} +HIGHER_IS_BETTER_REGISTRY = {} +FILTER_REGISTRY = {} + +DEFAULT_METRIC_REGISTRY = { + "loglikelihood": [ + "perplexity", + "acc", + ], + "loglikelihood_rolling": ["word_perplexity", "byte_perplexity", "bits_per_byte"], + "multiple_choice": ["acc", "acc_norm"], + "generate_until": ["exact_match"], +} + + +def register_metric(**args): + # TODO: do we want to enforce a certain interface to registered metrics? + def decorate(fn): + assert "metric" in args + name = args["metric"] + + for key, registry in [ + ("metric", METRIC_REGISTRY), + ("higher_is_better", HIGHER_IS_BETTER_REGISTRY), + ("aggregation", METRIC_AGGREGATION_REGISTRY), + ]: + if key in args: + value = args[key] + assert value not in registry, ( + f"{key} named '{value}' conflicts with existing registered {key}!" + ) + + if key == "metric": + registry[name] = fn + elif key == "aggregation": + registry[name] = AGGREGATION_REGISTRY[value] + else: + registry[name] = value + + return fn + + return decorate + + +def get_metric(name: str, hf_evaluate_metric=False) -> Callable: + if not hf_evaluate_metric: + if name in METRIC_REGISTRY: + return METRIC_REGISTRY[name] + else: + eval_logger.warning( + f"Could not find registered metric '{name}' in lm-eval, searching in HF Evaluate library..." + ) + + try: + metric_object = hf_evaluate.load(name) + return metric_object.compute + except Exception: + eval_logger.error( + f"{name} not found in the evaluate library! Please check https://huggingface.co/evaluate-metric", + ) + + +def register_aggregation(name: str): + def decorate(fn): + assert name not in AGGREGATION_REGISTRY, ( + f"aggregation named '{name}' conflicts with existing registered aggregation!" + ) + + AGGREGATION_REGISTRY[name] = fn + return fn + + return decorate + + +def get_aggregation(name: str) -> Callable[[], Dict[str, Callable]]: + try: + return AGGREGATION_REGISTRY[name] + except KeyError: + eval_logger.warning(f"{name} not a registered aggregation metric!") + + +def get_metric_aggregation(name: str) -> Callable[[], Dict[str, Callable]]: + try: + return METRIC_AGGREGATION_REGISTRY[name] + except KeyError: + eval_logger.warning(f"{name} metric is not assigned a default aggregation!") + + +def is_higher_better(metric_name) -> bool: + try: + return HIGHER_IS_BETTER_REGISTRY[metric_name] + except KeyError: + eval_logger.warning( + f"higher_is_better not specified for metric '{metric_name}'!" + ) + + +def register_filter(name): + def decorate(cls): + if name in FILTER_REGISTRY: + eval_logger.info( + f"Registering filter `{name}` that is already in Registry {FILTER_REGISTRY}" + ) + FILTER_REGISTRY[name] = cls + return cls + + return decorate + + +def get_filter(filter_name: Union[str, Callable]) -> Callable: + try: + return FILTER_REGISTRY[filter_name] + except KeyError as e: + if callable(filter_name): + return filter_name + else: + eval_logger.warning(f"filter `{filter_name}` is not registered!") + raise e diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/api/samplers.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/api/samplers.py new file mode 100644 index 0000000000000000000000000000000000000000..5d1791bdb4f8ae06cf4168dcdfa4c6a5a9bbc823 --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/api/samplers.py @@ -0,0 +1,232 @@ +import logging +import warnings +from functools import partial +from typing import TYPE_CHECKING, Iterable, Optional, Union + +import datasets + + +if TYPE_CHECKING: + from random import Random + + from lm_eval.api.task import ConfigurableTask, Task + +eval_logger = logging.getLogger("lm-eval") + + +class ContextSampler: + def __init__( + self, + docs: list[dict], + task: Union["Task", "ConfigurableTask"], + fewshot_indices: Optional[Iterable] = None, + rnd: Optional["Random"] = None, + ) -> None: + self.rnd = rnd + if not self.rnd: + raise ValueError( + "A `random.Random` generator argument must be provided to `rnd` of FewShotSampler!" + ) + + self.task = task + self.config = task._config + + self.target_delimiter = self.config.target_delimiter + self.fewshot_delimiter = self.config.fewshot_delimiter + + if ( + self.config.fewshot_config is not None + and self.config.fewshot_config.get("doc_to_text", None) is not None + ): + self.doc_to_text = partial( + self.task.doc_to_text, + doc_to_text=self.config.fewshot_config.get("doc_to_text", None), + ) + else: + self.doc_to_text = self.task.doc_to_text + + if ( + self.config.fewshot_config is not None + and self.config.fewshot_config.get("doc_to_target", None) is not None + ): + self.doc_to_target = partial( + self.task.doc_to_target, + doc_to_target=self.config.fewshot_config.get("doc_to_target", None), + ) + else: + self.doc_to_target = self.task.doc_to_target + + if ( + self.config.fewshot_config is not None + and self.config.fewshot_config.get("doc_to_choice", None) is not None + ): + self.doc_to_choice = partial( + self.task.doc_to_choice, + doc_to_choice=self.config.fewshot_config.get("doc_to_choice", None), + ) + else: + self.doc_to_choice = self.task.doc_to_choice + + self.docs = docs # HF dataset split, provided by task._fewshot_docs() + if fewshot_indices: # subset few-shot docs from + if not isinstance(self.docs, datasets.Dataset): + raise ValueError( + "Got `fewshot_indices` but fewshot_docs are not a HF dataset. Don't use both `fewshot_indices` and a user-defined few-shot sample list simultaneously" + ) + self.docs = self.docs.select(fewshot_indices) + + def get_context(self, doc: dict, num_fewshot: int, gen_prefix: str = None): + # draw an extra fewshot sample if using same split as evaluating on + prefix = gen_prefix + " " if gen_prefix else "" + n_samples = ( + num_fewshot + 1 + if self.config.fewshot_split == self.config.test_split + else num_fewshot + ) + + # draw `n_samples` docs from fewshot_docs + fewshotex = self.sample(n_samples) + + # get rid of the doc that's the one we're evaluating, if it's in the fewshot + # TODO: should we just stop people from using fewshot from same split as evaluating? + selected_docs = [x for x in fewshotex if x != doc][:num_fewshot] + + labeled_examples = "" + for doc in selected_docs: + doc_content = self.doc_to_text(doc) + doc_target = self.doc_to_target(doc) + if self.config.doc_to_choice is None or isinstance(doc_content, str): + labeled_examples += doc_content + else: + labeled_examples += self.doc_to_choice(doc)[doc_content] + + if doc_target != "": + if self.target_delimiter.isspace() and str(doc_target)[0].isspace(): + # TODO: add logger warn once here. + warnings.warn( + "Both target_delimiter and target start with a space. This may cause issues.", + Warning, + stacklevel=2, + ) + labeled_examples += self.target_delimiter + labeled_examples += prefix + labeled_examples += ( + str(doc_target[0]) + if isinstance(doc_target, list) + else doc_target + if self.config.doc_to_choice is None or isinstance(doc_target, str) + else str(self.doc_to_choice(doc)[doc_target]) + ) + labeled_examples += self.fewshot_delimiter + + return labeled_examples + + def get_chat_context( + self, + doc: dict, + num_fewshot: int, + fewshot_as_multiturn: bool = False, + gen_prefix: Optional[str] = None, + ): + # TODO: Do we need any other delimiter + prefix = gen_prefix + " " if gen_prefix else "" + chat_history = [] + # draw an extra fewshot sample if using same split as evaluating on + n_samples = ( + num_fewshot + 1 + if self.config.fewshot_split == self.config.test_split + else num_fewshot + ) + # draw `n_samples` docs from fewshot_docs + fewshotex = self.sample(n_samples) + + # get rid of the doc that's the one we're evaluating, if it's in the fewshot + # TODO: should we just stop people from using fewshot from same split as evaluating? + selected_docs = [x for x in fewshotex if x != doc][:num_fewshot] + + if fewshot_as_multiturn: + for doc in selected_docs: + doc_content = self.doc_to_text(doc) + doc_target = self.doc_to_target(doc) + chat_history.append( + { + "role": "user", + "content": doc_content + if self.config.doc_to_choice is None + or isinstance(doc_content, str) + else self.doc_to_choice(doc)[doc_content], + } + ) + chat_history.append( + { + "role": "assistant", + "content": prefix + str(doc_target[0]) + if isinstance(doc_target, list) + else prefix + doc_target + if self.config.doc_to_choice is None + or isinstance(doc_target, str) + else prefix + str(self.doc_to_choice(doc)[doc_target]), + } + ) + else: + # get fewshot context as one user turn + chat_history.append( + { + "role": "user", + "content": self.get_context( + doc, num_fewshot, gen_prefix=gen_prefix + ), + } + ) + + return chat_history + + def sample(self, n: int): + """ + Draw `n` samples from our fewshot docs. This method should be overridden by subclasses. + """ + + return self.rnd.sample(self.docs, n) + + +class FirstNSampler(ContextSampler): + def sample(self, n: int) -> None: + """ + Draw the first `n` samples in order from the specified split. + Used for tasks with "canonical" ordered fewshot examples, such as MMLU and CMMLU. + """ + assert n <= len(self.docs), ( + f"Error: number of fewshot samples requested exceeds the {len(self.docs)} that are available." + ) + return self.docs[:n] + + +class BalancedSampler(ContextSampler): + def sample(self, n: int) -> None: + """ + TODO: this should return approximately class-balanced samples from our fewshot examples. + TODO: what order should they be in? maybe random? + """ + + pass + + +class ManualSampler(ContextSampler): + def sample(self, n: int) -> None: + """ """ + pass + + +SAMPLER_REGISTRY = { + "default": ContextSampler, + "first_n": FirstNSampler, +} + + +def get_sampler(name: str): + try: + return SAMPLER_REGISTRY[name] + except KeyError: + raise ValueError( + f"Attempted to use contextsampler '{name}', but no sampling strategy for this name found! Supported model names: {', '.join(SAMPLER_REGISTRY.keys())}" + ) diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/api/task.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/api/task.py new file mode 100644 index 0000000000000000000000000000000000000000..cf865a167725fb1da7dcdd229aa1b75bc38362b1 --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/api/task.py @@ -0,0 +1,1839 @@ +import abc +import ast +import logging +import random +import re +from collections.abc import Callable +from copy import deepcopy +from dataclasses import asdict, dataclass +from inspect import getsource +from typing import ( + Any, + Dict, + Iterable, + Iterator, + List, + Literal, + Mapping, + Optional, + Tuple, + Union, +) + +import datasets +import numpy as np +from tqdm import tqdm + +from lm_eval import utils +from lm_eval.api import samplers +from lm_eval.api.instance import Instance, OutputType +from lm_eval.api.metrics import bits_per_byte, mean, weighted_perplexity +from lm_eval.api.registry import ( + AGGREGATION_REGISTRY, + DEFAULT_METRIC_REGISTRY, + get_aggregation, + get_metric, + get_metric_aggregation, + is_higher_better, +) +from lm_eval.caching.cache import load_from_cache, save_to_cache +from lm_eval.filters import build_filter_ensemble +from lm_eval.prompts import get_prompt + + +ALL_OUTPUT_TYPES = [ + "loglikelihood", + "multiple_choice", + "loglikelihood_rolling", + "generate_until", +] + +eval_logger = logging.getLogger(__name__) + + +@dataclass +class TaskConfig(dict): + # task naming/registry + task: Optional[str] = None + task_alias: Optional[str] = None + tag: Optional[Union[str, list]] = None + # HF dataset options. + # which dataset to use, + # and what splits for what purpose + custom_dataset: Optional[Callable] = None + dataset_path: Optional[str] = None + dataset_name: Optional[str] = None + dataset_kwargs: Optional[dict] = None + training_split: Optional[str] = None + validation_split: Optional[str] = None + test_split: Optional[str] = None + fewshot_split: Optional[str] = ( + None # TODO: assert that this not None if num_fewshot > 0. (?) assert if this is same split as one evaluating (?) + ) + # formatting / prompting options. + # see docs/advanced_task_guide.md for more info + process_docs: Optional[Callable] = None + doc_to_text: Optional[Union[Callable, str]] = None + doc_to_target: Optional[Union[Callable, str]] = None + doc_to_image: Union[Callable, str] = None + doc_to_audio: Union[Callable, str] = None + unsafe_code: bool = False + doc_to_choice: Optional[Union[Callable, str, dict, list]] = None + process_results: Optional[Union[Callable, str]] = None + use_prompt: Optional[str] = None + description: str = "" + target_delimiter: str = " " + fewshot_delimiter: str = "\n\n" + fewshot_config: Optional[dict] = None + # runtime configuration options + num_fewshot: Optional[int] = None + # scoring options + metric_list: Optional[list] = None + output_type: OutputType = "generate_until" + generation_kwargs: Optional[dict] = None + repeats: int = 1 + filter_list: Optional[Union[str, list]] = None + should_decontaminate: bool = False + doc_to_decontamination_query: Optional[str] = None + gen_prefix: Optional[str] = None + metadata: Optional[dict] = ( + None # by default, not used in the code. allows for users to pass arbitrary info to tasks + ) + + def __post_init__(self) -> None: + if self.generation_kwargs is not None: + if self.output_type != "generate_until": + eval_logger.warning( + f"[{self.task}] passed `generation_kwargs`, but not using `output_type: generate_until`!" + ) + + if "temperature" in self.generation_kwargs: + self.generation_kwargs["temperature"] = float( + self.generation_kwargs["temperature"] + ) + + if "until" not in self.generation_kwargs: + self.generation_kwargs["until"] = [self.fewshot_delimiter] + else: + if self.output_type == "generate_until": + # ensure that we greedily generate in absence of explicit arguments otherwise + self.generation_kwargs = { + "until": ( + None + if self.fewshot_delimiter is None + else [self.fewshot_delimiter] + ), + "do_sample": False, + } + + def __getitem__(self, item): + return getattr(self, item) + + def __setitem__(self, item, value): + return setattr(self, item, value) + + def to_dict(self, keep_callable: bool = False) -> dict: + """dumps the current config as a dictionary object, as a printable format. + null fields will not be printed. + Used for dumping results alongside full task configuration + + :return: dict + A printable dictionary version of the TaskConfig object. + + # TODO: should any default value in the TaskConfig not be printed? + """ + cfg_dict = asdict(self) + # remove values that are `None` + for k, v in list(cfg_dict.items()): + if v is None: + cfg_dict.pop(k) + elif k == "metric_list": + for metric_dict in v: + for metric_key, metric_value in metric_dict.items(): + if callable(metric_value): + metric_dict[metric_key] = self.serialize_function( + metric_value, keep_callable=keep_callable + ) + cfg_dict[k] = v + elif callable(v): + cfg_dict[k] = self.serialize_function(v, keep_callable=keep_callable) + return cfg_dict + + def serialize_function( + self, value: Union[Callable, str], keep_callable=False + ) -> Union[Callable, str]: + """Serializes a given function or string. + + If 'keep_callable' is True, the original callable is returned. + Otherwise, attempts to return the source code of the callable using 'getsource'. + """ + if keep_callable: + return value + else: + try: + return getsource(value) + except (TypeError, OSError): + return str(value) + + +class Task(abc.ABC): + """A task represents an entire benchmark including its dataset, problems, + answers, and evaluation methods. See BoolQ for a simple example implementation + + A `doc` can be any python object which represents one instance of evaluation. + This is usually a dictionary e.g. + {"question": ..., "answer": ...} or + {"question": ..., question, answer) + """ + + VERSION: Optional[Union[int, str]] = None + + # The name of the `Task` benchmark as denoted in the HuggingFace datasets Hub + # or a path to a custom `datasets` loading script. + DATASET_PATH: Optional[str] = None + + # The name of a subset within `DATASET_PATH`. + DATASET_NAME: Optional[str] = None + + OUTPUT_TYPE: Optional[OutputType] = None + + def __init__( + self, + data_dir: Optional[str] = None, + cache_dir: Optional[str] = None, + download_mode: Optional[datasets.DownloadMode] = None, + config: Optional[Mapping] = None, # Union[dict, TaskConfig] + ) -> None: + """ + :param data_dir: str + Stores the path to a local folder containing the `Task`'s data files. + Use this to specify the path to manually downloaded data (usually when + the dataset is not publicly accessible). + :param cache_dir: str + The directory to read/write the `Task` dataset. This follows the + HuggingFace `datasets` API with the default cache directory located at: + `~/.cache/huggingface/datasets` + NOTE: You can change the cache location globally for a given process + to another directory: + `export HF_DATASETS_CACHE="/path/to/another/directory"` + :param download_mode: datasets.DownloadMode + How to treat pre-existing `Task` downloads and data. + - `datasets.DownloadMode.REUSE_DATASET_IF_EXISTS` + Reuse download and reuse dataset. + - `datasets.DownloadMode.REUSE_CACHE_IF_EXISTS` + Reuse download with fresh dataset. + - `datasets.DownloadMode.FORCE_REDOWNLOAD` + Fresh download and fresh dataset. + """ + self.download(data_dir, cache_dir, download_mode) + self._training_docs: Optional[list] = None + self._fewshot_docs: Optional[list] = None + self._instances: Optional[List[Instance]] = None + + self._config: TaskConfig = TaskConfig({**config}) if config else TaskConfig() + + self._filters = [build_filter_ensemble("none", [["take_first", None]])] + self.fewshot_rnd: Optional[random.Random] = ( + None # purposely induce errors in case of improper usage + ) + + def download( + self, + data_dir: Optional[str] = None, + cache_dir: Optional[str] = None, + download_mode=None, + ) -> None: + """Downloads and returns the task dataset. + Override this method to download the dataset from a custom API. + + :param data_dir: str + Stores the path to a local folder containing the `Task`'s data files. + Use this to specify the path to manually downloaded data (usually when + the dataset is not publicly accessible). + :param cache_dir: str + The directory to read/write the `Task` dataset. This follows the + HuggingFace `datasets` API with the default cache directory located at: + `~/.cache/huggingface/datasets` + NOTE: You can change the cache location globally for a given process + by setting the shell environment variable, `HF_DATASETS_CACHE`, + to another directory: + `export HF_DATASETS_CACHE="/path/to/another/directory"` + :param download_mode: datasets.DownloadMode + How to treat pre-existing `Task` downloads and data. + - `datasets.DownloadMode.REUSE_DATASET_IF_EXISTS` + Reuse download and reuse dataset. + - `datasets.DownloadMode.REUSE_CACHE_IF_EXISTS` + Reuse download with fresh dataset. + - `datasets.DownloadMode.FORCE_REDOWNLOAD` + Fresh download and fresh dataset. + """ + self.dataset = datasets.load_dataset( + path=self.DATASET_PATH, + name=self.DATASET_NAME, + data_dir=data_dir, + cache_dir=cache_dir, + download_mode=download_mode, + ) + + @property + def config(self) -> TaskConfig: + """Returns the TaskConfig associated with this class.""" + return self._config + + @abc.abstractmethod + def has_training_docs(self): + """Whether the task has a training set""" + pass + + @abc.abstractmethod + def has_validation_docs(self): + """Whether the task has a validation set""" + pass + + @abc.abstractmethod + def has_test_docs(self): + """Whether the task has a test set""" + pass + + def training_docs(self) -> Iterable: + """ + :return: Iterable[obj] + A iterable of any object, that doc_to_text can handle + """ + return [] + + def validation_docs(self) -> Iterable: + """ + :return: Iterable[obj] + A iterable of any object, that doc_to_text can handle + """ + return [] + + def test_docs(self) -> Iterable: + """ + :return: Iterable[obj] + A iterable of any object, that doc_to_text can handle + """ + return [] + + def fewshot_docs(self) -> Iterable: + """ + :return: Iterable[obj] + A iterable of any object, that doc_to_text can handle + """ + if self.has_training_docs(): + return self.training_docs() + elif self.has_validation_docs(): + return self.validation_docs() + else: + if self.config.get("num_fewshot", 0) > 0: + eval_logger.warning( + f"[Task: {self.config.task}] has_training_docs and has_validation_docs are False" + ", using test_docs as fewshot_docs but this is not recommended." + ) + return self.test_docs() + + def _process_doc(self, doc: dict) -> dict: + """ + Override this to process (detokenize, strip, replace, etc.) individual + documents. This can be used in a map over documents of a data split. + E.g. `map(self._process_doc, self.dataset["validation"])` + + :return: dict + The processed version of the specified `doc`. + """ + return doc + + @property + def instances(self) -> List[Instance]: + """After calling `task.build_all_requests()`, tasks + maintain a list of the dataset instances which will be evaluated. + """ + return self._instances + + def fewshot_examples(self, k, rnd): + if self._training_docs is None: + self._training_docs = list(self.training_docs()) + + return rnd.sample(self._training_docs, k) + + def doc_to_decontamination_query(self, doc): + raise NotImplementedError( + "Override doc_to_decontamination_query with document specific decontamination query." + ) + + @abc.abstractmethod + def doc_to_text(self, doc): + pass + + @abc.abstractmethod + def doc_to_target(self, doc): + pass + + # not an abstractmethod because not every language-only task has to implement this + def doc_to_image(self, doc): + raise NotImplementedError + + def doc_to_audio(self, doc): + raise NotImplementedError + + def doc_to_prefix(self, doc): + return "" + + def build_all_requests( + self, + *, + limit: Union[int, None] = None, + rank: int = 0, + world_size: int = 1, + cache_requests: bool = False, + rewrite_requests_cache: bool = False, + system_instruction: Optional[str] = None, + apply_chat_template: bool = False, + fewshot_as_multiturn: bool = False, + chat_template: Optional[Callable] = None, + tokenizer_name: str = "", + ) -> None: + """Build a set of Instances for a task, and store them in task.instances""" + + # used with caching + og_limit = limit + + cache_key = f"requests-{self._config.task}-{self.config.num_fewshot}shot-rank{rank}-world_size{world_size}" + cache_key += "-chat_template" if apply_chat_template else "" + cache_key += "-fewshot_as_multiturn" if fewshot_as_multiturn else "" + cache_key += ( + f"-system_prompt_hash{utils.hash_string(system_instruction)}" + if system_instruction is not None + else "" + ) + cache_key += f"-tokenizer{tokenizer_name}" + + cached_instances = load_from_cache(file_name=cache_key, cache=cache_requests) + + if cache_requests and cached_instances and not rewrite_requests_cache: + cached_instances = cached_instances[:limit] + + flattened_instances = [ + instance + for instance_group in cached_instances + for instance in instance_group + ] + + self._instances = flattened_instances + return + + eval_logger.info(f"Building contexts for {self.config.task} on rank {rank}...") + + instances = [] + + # process all documents when caching is specified for simplicity + if ( + cache_requests + and (not cached_instances or rewrite_requests_cache) + and limit is not None + ): + limit = None + + doc_id_docs = list( + self.doc_iterator(rank=rank, limit=limit, world_size=world_size) + ) + + num_docs = len(doc_id_docs) + + for doc_id, doc in tqdm( + doc_id_docs, + total=num_docs, + ): + # sample fewshot context #TODO: need to offset doc_id by rank now! + fewshot_ctx = self.fewshot_context( + doc, + 0 if self.config.num_fewshot is None else self.config.num_fewshot, + system_instruction, + apply_chat_template, + fewshot_as_multiturn, + chat_template, + gen_prefix=self.doc_to_prefix(doc), + ) + + # TODO: we should override self.config.repeats if doing greedy gen so users don't waste time+compute + inst = self.construct_requests( + doc=doc, + ctx=fewshot_ctx, + metadata=(self.config["task"], doc_id, self.config.repeats), + apply_chat_template=apply_chat_template, + chat_template=chat_template, + ) + + if not isinstance(inst, list): + inst = [inst] + + instances.append(inst) + + # now flatten, this is to allow slicing to work with pickles + + sliced_instances = instances[:og_limit] + + flattened_instances = [ + instance + for instance_group in sliced_instances + for instance in instance_group + ] + + self._instances = flattened_instances + + if len(self._instances) == 0: + raise ValueError("task.build_requests() did not find any docs!") + + if cache_requests and (not cached_instances or rewrite_requests_cache): + save_to_cache(file_name=cache_key, obj=instances) + + @abc.abstractmethod + def construct_requests(self, doc, ctx, **kwargs): + """Uses RequestFactory to construct Requests and returns an iterable of + Requests which will be sent to the LM. + + :param doc: + The document as returned from training_docs, validation_docs, or test_docs. + :param ctx: str + The context string, generated by fewshot_context. This includes the natural + language description, as well as the few shot examples, and the question + part of the document for `doc`. + :param doc_idx: int + The index of a document within `self.test_docs()` or `self.validation_docs()`, + whichever is the main split used. + :param repeats: int + TODO: update this docstring + The number of times each instance in a dataset is inferred on. Defaults to 1, + can be increased for techniques like majority voting. + """ + pass + + @abc.abstractmethod + def process_results(self, doc, results): + """Take a single document and the LM results and evaluates, returning a + dict where keys are the names of submetrics and values are the values of + the metric for that one document + + :param doc: + The document as returned from training_docs, validation_docs, or test_docs. + :param results: + The results of the requests created in construct_requests. + """ + pass + + @abc.abstractmethod + def aggregation(self): + """ + :returns: {str: [metric_score] -> float} + A dictionary where keys are the names of submetrics and values are + functions that aggregate a list of metric scores + """ + pass + + @abc.abstractmethod + def higher_is_better(self): + """ + :returns: {str: bool} + A dictionary where keys are the names of submetrics and values are + whether a higher value of the submetric is better + """ + pass + + def get_config(self, key: str) -> Any: + return getattr(self._config, key, None) + + @classmethod + def count_bytes(cls, doc): + """Used for byte-level perplexity metrics in rolling loglikelihood""" + return len(doc.encode("utf-8")) + + @classmethod + def count_words(cls, doc): + """Downstream loglikelihood_rolling perplexity tasks with custom word boundaries should override this!""" + return len(re.split(r"\s+", doc)) + + @utils.positional_deprecated + def fewshot_context(self, doc, num_fewshot, rnd=None, description=None, **kwargs): + """Returns a fewshot context string that is made up of a prepended description + (if provided), the `num_fewshot` number of examples, and an appended prompt example. + + :param doc: str + The document as returned from training_docs, validation_docs, or test_docs. + :param num_fewshot: int + The number of fewshot examples to provide in the returned context string. + :param rnd: random.Random + The pseudo-random number generator used to randomly sample examples. + WARNING: This is currently a required arg although it's optionalized with a default `None`. + :param description: str + The task's description that will be prepended to the fewshot examples. + :returns: str + The fewshot context. + """ + if rnd is None: + if self.fewshot_rnd is not None: + rnd = self.fewshot_rnd + else: + raise ValueError( + "A `random.Random` generator argument must be provided to `rnd`" + ) + + description = description if description else "" + + if num_fewshot == 0: + labeled_examples = "" + else: + # for sets with no training docs, draw from other set *but ensure no overlap with current doc* + if self.has_training_docs(): + fewshotex = self.fewshot_examples(k=num_fewshot, rnd=rnd) + else: + if self._fewshot_docs is None: + self._fewshot_docs = list( + self.validation_docs() + if self.has_validation_docs() + else self.test_docs() + ) + + fewshotex = rnd.sample(self._fewshot_docs, num_fewshot + 1) + + # get rid of the doc that's the one we're evaluating, if it's in the fewshot + fewshotex = [x for x in fewshotex if x != doc][:num_fewshot] + + labeled_examples = ( + "\n\n".join( + [ + self.doc_to_text(doc) + self.doc_to_target(doc) + for doc in fewshotex + ] + ) + + "\n\n" + ) + + example = self.doc_to_text(doc) + return description + labeled_examples + example + + def apply_filters(self) -> Optional[List[Instance]]: + """Iterates over FilterEnsembles and applies them to instances""" + if hasattr(self, "_filters"): + for f in self._filters: + f.apply(self._instances) + else: + eval_logger.warning("No filter defined, passing through instances") + return self._instances + + def dump_config(self) -> dict: + """Returns the config as a dictionary.""" + # TODO: this should only return the overrides applied to a non-YAML task's configuration. + # (num_fewshot) + return self.config.to_dict() + + def set_config(self, key: str, value: Any, update: bool = False) -> None: + """Set or update the configuration for a given key.""" + if key is None: + raise ValueError("Key must be provided.") + + if update: + current_value = getattr(self._config, key, {}) + if not isinstance(current_value, dict): + raise TypeError( + f"Expected a dict for key '{key}', got {type(current_value).__name__} instead." + ) + current_value.update(value) + else: + setattr(self._config, key, value) + + def override_metric(self, metric_name: str) -> None: + """ + Override the default metrics used for evaluation with custom metrics. + + Parameters: + - metric_name (str): The name of the custom metric to override. Should be registered in api.metrics. + """ + ( + self._metric_fn_list, + self._aggregation_list, + self._metric_fn_kwargs, + self._higher_is_better, + ) = ({}, {}, {}, {}) + self._metric_fn_list[metric_name] = get_metric(metric_name) + self._aggregation_list[metric_name] = get_metric_aggregation(metric_name) + self._higher_is_better[metric_name] = is_higher_better(metric_name) + self._metric_fn_kwargs[metric_name] = {} + if not isinstance(self, ConfigurableTask): + self.process_results = lambda x, y: {metric_name: get_metric(metric_name)} + self.aggregation = lambda: { + metric_name: get_metric_aggregation(metric_name) + } + setattr(self._config, "metric_list", [{"metric": metric_name}]) + setattr(self._config, "process_results", None) + + def set_fewshot_seed(self, seed: Optional[int] = None) -> None: + self.fewshot_rnd = random.Random(seed) + if hasattr(self, "sampler"): + self.sampler.rnd = self.fewshot_rnd + + @property + def eval_docs(self) -> Union[datasets.Dataset, List[dict]]: + if self.has_test_docs(): + return self.test_docs() + elif self.has_validation_docs(): + return self.validation_docs() + else: + raise ValueError( + f"Task dataset (path={self.DATASET_PATH}, name={self.DATASET_NAME}) must have valid or test docs!" + ) + + def doc_iterator( + self, *, rank: int = 0, limit: Union[int, None] = None, world_size: int = 1 + ) -> Iterator[Tuple[int, Any]]: + limit = int(limit) if limit else None + doc_iterator = utils.create_iterator( + enumerate(self.eval_docs), + rank=int(rank), + limit=limit, + world_size=int(world_size), + ) + return doc_iterator + + +class ConfigurableTask(Task): + VERSION = "Yaml" + OUTPUT_TYPE = None + CONFIG = None + + def __init__( + self, + data_dir=None, + cache_dir=None, + download_mode=None, + config: Optional[dict] = None, + ) -> None: # TODO no super() call here + # Get pre-configured attributes + self._config = self.CONFIG + + # Use new configurations if there was no preconfiguration + if self.config is None: + self._config = TaskConfig(**config) + # Overwrite configs + else: + if config is not None: + self._config.__dict__.update(config) + + if self.config is None: + raise ValueError( + "Must pass a config to ConfigurableTask, either in cls.CONFIG or `config` kwarg" + ) + + if isinstance(self.config.metadata, dict): + if "version" in self.config.metadata: + self.VERSION = self.config.metadata["version"] + + if self.config.output_type is not None: + if self.config.output_type not in ALL_OUTPUT_TYPES: + raise ValueError( + f"Got invalid output_type '{self.config.output_type}', must be in '{','.join(ALL_OUTPUT_TYPES)}'" + ) + self.OUTPUT_TYPE = self.config.output_type + + if self.config.doc_to_image is not None: + # mark the task as requiring multimodality. + self.MULTIMODAL = True + + if self.config.doc_to_audio: + # mark the task as requiring multimodality. + self.MULTIMODAL = True + + if self.config.unsafe_code is not False: + self.UNSAFE_CODE = True + + if self.config.dataset_path is not None: + self.DATASET_PATH = self.config.dataset_path + + if self.config.dataset_name is not None: + self.DATASET_NAME = self.config.dataset_name + + self._metric_fn_list = {} + self._metric_fn_kwargs = {} + self._aggregation_list = {} + self._higher_is_better = {} + + if self.config.metric_list is None: + # TODO: handle this in TaskConfig.__post_init__ ? + _metric_list = DEFAULT_METRIC_REGISTRY[self.config.output_type] + + for metric_name in _metric_list: + self._metric_fn_list[metric_name] = get_metric(metric_name) + self._metric_fn_kwargs[metric_name] = {} + self._aggregation_list[metric_name] = get_metric_aggregation( + metric_name + ) + self._higher_is_better[metric_name] = is_higher_better(metric_name) + else: + for metric_config in self.config.metric_list: + if "metric" not in metric_config: + raise ValueError( + "'metric' key not provided for an entry in 'metric_list', must be specified!" + ) + metric_name = metric_config["metric"] + kwargs = { + key: metric_config[key] + for key in metric_config + if key + not in ["metric", "aggregation", "higher_is_better", "hf_evaluate"] + } + hf_evaluate_metric = ( + "hf_evaluate" in metric_config + and metric_config["hf_evaluate"] is True + ) + + if self.config.process_results is not None: + self._metric_fn_list[metric_name] = None + self._metric_fn_kwargs[metric_name] = {} + elif callable(metric_name): + metric_fn = metric_name.__call__ + metric_name = metric_name.__name__ + self._metric_fn_list[metric_name] = metric_fn + self._metric_fn_kwargs[metric_name] = kwargs + else: + self._metric_fn_list[metric_name] = get_metric( + metric_name, hf_evaluate_metric + ) + self._metric_fn_kwargs[metric_name] = kwargs + + if "aggregation" in metric_config: + agg_name = metric_config["aggregation"] + if isinstance(agg_name, str): + self._aggregation_list[metric_name] = get_aggregation(agg_name) + elif callable(agg_name): # noqa: E721 + self._aggregation_list[metric_name] = metric_config[ + "aggregation" + ] + else: + INV_AGG_REGISTRY = {v: k for k, v in AGGREGATION_REGISTRY.items()} + metric_agg = get_metric_aggregation(metric_name) + eval_logger.warning( + f"[Task: {self.config.task}] metric {metric_name} is defined, but aggregation is not. " + f"using default " + f"aggregation={INV_AGG_REGISTRY[metric_agg]}" + ) + self._aggregation_list[metric_name] = metric_agg + + if "higher_is_better" in metric_config: + self._higher_is_better[metric_name] = metric_config[ + "higher_is_better" + ] + else: + eval_logger.warning( + f"[Task: {self.config.task}] metric {metric_name} is defined, but higher_is_better is not. " + f"using default " + f"higher_is_better={is_higher_better(metric_name)}" + ) + self._higher_is_better[metric_name] = is_higher_better(metric_name) + + self.download(self.config.dataset_kwargs) + self._training_docs = None + self._fewshot_docs = None + + if self.config.filter_list is not None: + self._filters = [] + for filter_config in self.config.filter_list: + filter_name = filter_config["name"] + filter_functions = filter_config["filter"] + components = [] + for function in filter_functions: + kwargs = { + key: function[key] for key in function if key != "function" + } + components.append([function["function"], kwargs]) + filter_pipeline = build_filter_ensemble(filter_name, components) + self._filters.append(filter_pipeline) + else: + # TODO: handle repeats in a more general way rather than just discarding + eval_logger.debug( + "No custom filters defined. Using default 'take_first' filter for handling repeats." + ) + self._filters = [build_filter_ensemble("none", [["take_first", None]])] + + if self.config.use_prompt is not None: + eval_logger.info(f"loading prompt {self.config.use_prompt}") + self.prompt = get_prompt( + self.config.use_prompt, self.DATASET_PATH, self.DATASET_NAME + ) + else: + self.prompt = None + + if self.fewshot_docs() is not None: + self.fewshot_rnd = ( + random.Random() + ) # setting with no seed, to be overridden at a later time + config_sampler: Union[str, Callable] = ( + self.config.fewshot_config.get("sampler", "default") + if self.config.fewshot_config + else "default" + ) + if isinstance(config_sampler, str): + self.sampler = samplers.get_sampler(config_sampler)( + list(self.fewshot_docs()), self, rnd=self.fewshot_rnd + ) + elif callable(config_sampler) and issubclass( + config_sampler, samplers.ContextSampler + ): + self.sampler = config_sampler( + docs=list(self.fewshot_docs()), task=self, rnd=self.fewshot_rnd + ) + else: + raise TypeError( + f"fewshot_config.sampler should be a string or callable of ContextSampler type, " + f"not {type(config_sampler)}" + ) + + self.task_docs = self.eval_docs + + # Test One Doc + self.features = list(self.task_docs.features.keys()) + self.multiple_input = 0 + self.multiple_target = 0 + test_doc = self.task_docs[0] + test_text = self.doc_to_text(test_doc) + test_target = self.doc_to_target(test_doc) + + if self.config.doc_to_choice is not None: + test_choice = self.doc_to_choice(test_doc) + if not isinstance(test_choice, list): + eval_logger.error("doc_to_choice must return list") + else: + num_choice = len(test_choice) + + if isinstance(test_text, int): + self.multiple_input = num_choice + else: + test_choice = None + + if isinstance(test_target, list): + self.multiple_target = len(test_target) + else: + if (isinstance(test_target, int)) and (test_choice is not None): + test_target = test_choice[test_target] + else: + test_target = str(test_target) + + if test_choice is not None: + check_choices = test_choice + else: + check_choices = [test_target] + if self.config.doc_to_choice is not None: + for choice in check_choices: + choice_has_whitespace = True if choice[0].isspace() else False + delimiter_has_whitespace = ( + True + if self.config.target_delimiter.rstrip() + != self.config.target_delimiter + else False + ) + + if delimiter_has_whitespace and choice_has_whitespace: + eval_logger.debug( + f'Both target_delimiter "{self.config.target_delimiter}" and target choice: "{choice}" have whitespace' + ) + elif (not delimiter_has_whitespace) and (not choice_has_whitespace): + eval_logger.debug( + f'Both target_delimiter "{self.config.target_delimiter}" and target choice: "{choice}" do not have whitespace, ignore if the language you are evaluating on does not require/use whitespace' + ) + + def download( + self, dataset_kwargs: Optional[Dict[str, Any]] = None, **kwargs + ) -> None: + if isinstance(self.config.custom_dataset, Callable): + eval_logger.warning( + f"{self.config.task}: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager." + + "\nFor example --metadata='{\"max_seq_lengths\":[4096, 8192]}'. For details see task Readme." + ) + self.dataset = self.config.custom_dataset( + **(self.config.metadata or {}), **(self.config.dataset_kwargs or {}) + ) + else: + self.dataset = datasets.load_dataset( + path=self.DATASET_PATH, + name=self.DATASET_NAME, + **dataset_kwargs if dataset_kwargs is not None else {}, + ) + + def has_training_docs(self) -> bool: + if self.config.training_split is not None: + return True + else: + return False + + def has_validation_docs(self) -> bool: + if self.config.validation_split is not None: + return True + else: + return False + + def has_test_docs(self) -> bool: + if self.config.test_split is not None: + return True + else: + return False + + def training_docs(self) -> datasets.Dataset: + if self.has_training_docs(): + if self.config.process_docs is not None: + return self.config.process_docs( + self.dataset[self.config.training_split] + ) + return self.dataset[self.config.training_split] + + def validation_docs(self) -> datasets.Dataset: + if self.has_validation_docs(): + if self.config.process_docs is not None: + return self.config.process_docs( + self.dataset[self.config.validation_split] + ) + return self.dataset[self.config.validation_split] + + def test_docs(self) -> datasets.Dataset: + if self.has_test_docs(): + if self.config.process_docs is not None: + return self.config.process_docs(self.dataset[self.config.test_split]) + return self.dataset[self.config.test_split] + + def fewshot_docs(self): + if self.config.fewshot_split is not None: + if self.config.process_docs is not None: + return self.config.process_docs(self.dataset[self.config.fewshot_split]) + return self.dataset[self.config.fewshot_split] + elif ( + self.config.fewshot_config is not None + and self.config.fewshot_config.get("samples", None) is not None + ): + if isinstance(self.config.fewshot_config["samples"], list): + return self.config.fewshot_config["samples"] + elif callable(self.config.fewshot_config["samples"]): + return self.config.fewshot_config["samples"]() + else: + raise Exception( + "`fewshot_config['samples']` was incorrectly defined in the configuration. It should be either a list of samples as a dict, or function returning this list." + ) + else: + if (self.config.num_fewshot is not None) and (self.config.num_fewshot > 0): + eval_logger.warning( + f"[Task: {self.config.task}] " + "num_fewshot > 0 but fewshot_split is None. " + "using preconfigured rule." + ) + return super().fewshot_docs() + + @staticmethod + def append_target_question( + labeled_examples: List[Dict[str, str]], + question: str, + fewshot_as_multiturn: bool = False, + gen_prefix: Optional[str] = None, + ) -> None: + """Adds a target question to the labeled examples list. + If fewshot_as_multiturn is True, or labeled_examples is empty, or the last entry is a system turn, appends the question as a new user entry. + Otherwise, it is appended to the last user entry, ensuring that the conversation alternates between the user and the assistant. + """ + if not fewshot_as_multiturn: + # if no messages or last message is system, append as new user entry + if len(labeled_examples) == 0 or labeled_examples[-1]["role"] == "system": + labeled_examples.append({"role": "user", "content": question}) + # if last message is user, append to it to avoid two user messages in a row + else: + labeled_examples[-1]["content"] += question + else: + # if fewshot_as_multiturn is True, append as next user entry (last is always assistant) + labeled_examples.append({"role": "user", "content": question}) + if gen_prefix: + labeled_examples.append({"role": "assistant", "content": gen_prefix}) + + @utils.positional_deprecated + def fewshot_context( + self, + doc: dict, + num_fewshot: int, + system_instruction: Optional[str] = None, + apply_chat_template: bool = False, + fewshot_as_multiturn: bool = False, + chat_template: Optional[Callable] = None, + gen_prefix: Optional[str] = None, + ) -> Union[str, List[str]]: + """Returns a fewshot context string that is made up of a prepended description + (if provided), the `num_fewshot` number of examples, and an appended prompt example. + + :param doc: str + The document as returned from training_docs, validation_docs, or test_docs. + :param num_fewshot: int + The number of fewshot examples to provide in the returned context string. + :param system_instruction: str + System instruction to be applied to the prompt. + :param apply_chat_template: bool + Whether to apply the chat template to the fewshot context. + :param fewshot_as_multiturn: bool + Whether to provide the fewshot examples as a multiturn conversation or a single user turn. + :param chat_template: + callable (from lm.apply_chat_template) that takes in a list[Dict] chat transcript and renders it into a string. + :param gen_prefix: + String to append after the <|assistant|> token. + :returns: str + The fewshot context. + """ + if apply_chat_template: + labeled_examples = [] + else: + labeled_examples = "" + + # get task description + if description := self.config.description: + description = utils.apply_template(self.config.description, doc) + + # create system prompt based on the provided system instruction and description + if system_instruction is not None and description: + system_prompt = ( + f"{system_instruction}{self.sampler.fewshot_delimiter}{description}" + ) + elif system_instruction is not None: + system_prompt = system_instruction + elif description: + system_prompt = description + else: + system_prompt = "" + + # add system prompt if specified + if system_prompt: + if apply_chat_template: + labeled_examples.append({"role": "system", "content": system_prompt}) + else: + labeled_examples = system_prompt + # if few-shot - append examples after the system prompt + if num_fewshot > 0: + if apply_chat_template: + labeled_examples.extend( + self.sampler.get_chat_context( + doc, + num_fewshot, + fewshot_as_multiturn, + gen_prefix=gen_prefix, + ) + ) + else: + labeled_examples += self.sampler.get_context( + doc, num_fewshot, gen_prefix=gen_prefix + ) + + example = self.doc_to_text(doc) + if apply_chat_template: + if self.multiple_input: + # TODO: append prefill? + if not labeled_examples: + return "" + return chat_template(labeled_examples) + if isinstance(example, str): + self.append_target_question( + labeled_examples, + example, + fewshot_as_multiturn, + gen_prefix=gen_prefix, + ) + # for loglikelihood create a list of questions with appended choices + elif isinstance(example, list): + labeled_examples_list = [] + # copy chat history for each example and append the answer + for ex in example: + chat = deepcopy(labeled_examples) + self.append_target_question( + chat, + ex, + fewshot_as_multiturn, + gen_prefix=gen_prefix, + ) + # TODO: append prefill? + labeled_examples_list.append( + chat_template( + chat, + add_generation_prompt=False if gen_prefix else True, + ) + ) + return labeled_examples_list + # if example is an integer, append the choice or convert to string + elif isinstance(example, int): + if self.config.doc_to_choice is not None: + choices = self.doc_to_choice(doc) + self.append_target_question( + labeled_examples, + choices[example], + fewshot_as_multiturn, + gen_prefix=gen_prefix, + ) + else: + self.append_target_question( + labeled_examples, + str(example), + fewshot_as_multiturn, + gen_prefix=gen_prefix, + ) + # return lm.apply_chat_template(labeled_examples) + return chat_template( + labeled_examples, + add_generation_prompt=False if gen_prefix else True, + ) + else: + prefix = ( + self.config.target_delimiter + gen_prefix + if gen_prefix is not None + else "" + ) + if self.multiple_input: + return labeled_examples + if isinstance(example, str): + return labeled_examples + example + prefix + elif isinstance(example, list): + return [labeled_examples + ex + prefix for ex in example] + elif isinstance(example, int): + if self.config.doc_to_choice is not None: + choices = self.doc_to_choice(doc) + return labeled_examples + choices[example] + prefix + else: + return labeled_examples + str(example) + prefix + + def apply_filters(self) -> Optional[List[Instance]]: + """Iterates over FilterEnsembles and applies them to instances""" + if hasattr(self, "_filters"): + for f in self._filters: + f.apply(self._instances) + else: + eval_logger.warning("No filter defined, passing through instances") + return self._instances + + def should_decontaminate(self): + return self.config.should_decontaminate + + def doc_to_decontamination_query(self, doc: dict): + if self.config.should_decontaminate: + if self.config.doc_to_decontamination_query is None: + return self.doc_to_text(doc) + else: + doc_to_decontamination_query = self.config.doc_to_decontamination_query + if doc_to_decontamination_query in self.features: + return doc[doc_to_decontamination_query] + elif callable(doc_to_decontamination_query): + return doc_to_decontamination_query(doc) + else: + return ast.literal_eval( + utils.apply_template( + self.config.doc_to_decontamination_query, doc + ) + ) + + def _process_doc(self, doc: dict) -> dict: + """ + Override this to process (detokenize, strip, replace, etc.) individual + documents. This can be used in a map over documents of a data split. + E.g. `map(self._process_doc, self.dataset["validation"])` + + :return: dict + The processed version of the specified `doc`. + """ + return doc + + def doc_to_text(self, doc, doc_to_text=None): + if self.prompt is not None: + doc_to_text = self.prompt + elif doc_to_text is not None: + doc_to_text = doc_to_text + else: + doc_to_text = self.config.doc_to_text + + if isinstance(doc_to_text, int): + return doc_to_text + elif isinstance(doc_to_text, str): + if doc_to_text in self.features: + # if self.config.doc_to_choice is not None: + # return self.doc_to_choice(doc)[doc[doc_to_text]] + # else: + return doc[doc_to_text] + else: + text_string = utils.apply_template(doc_to_text, doc) + if text_string.isdigit() and self._config.doc_to_choice is not None: + return ast.literal_eval(text_string) + else: + return text_string + elif callable(doc_to_text): + return doc_to_text(doc) + # Used when applying a Promptsource template + elif hasattr(doc_to_text, "apply"): + applied_prompt = doc_to_text.apply(doc) + if len(applied_prompt) == 2: + return applied_prompt[0] + else: + eval_logger.warning("Applied prompt returns empty string") + return self.config.fewshot_delimiter + else: + print(type(doc_to_text)) + raise TypeError + + def doc_to_target(self, doc: Mapping, doc_to_target=None) -> Union[int, str, list]: + if self.prompt is not None: + doc_to_target = self.prompt + elif doc_to_target is not None: + doc_to_target = doc_to_target + else: + doc_to_target = self.config.doc_to_target + + if isinstance(doc_to_target, int): + return doc_to_target + elif isinstance(doc_to_target, str): + if doc_to_target in self.features: + # if self.config.doc_to_choice is not None: + # return self.doc_to_choice(doc)[doc[doc_to_target]] + # else: + return doc[doc_to_target] + else: + target_string = utils.apply_template(doc_to_target, doc) + if target_string.isdigit() and self._config.doc_to_choice is not None: + return ast.literal_eval(target_string) + elif ( + len(target_string) >= 2 + and (target_string[0] == "[") + and (target_string[-1] == "]") + ): + try: + return ast.literal_eval(target_string) + except (SyntaxError, ValueError): + return target_string + else: + return target_string + elif isinstance(doc_to_target, list): + return doc_to_target + elif callable(doc_to_target): + return doc_to_target(doc) + # Used when applying a Promptsource template + elif hasattr(doc_to_target, "apply"): + applied_prompt = doc_to_target.apply(doc) + if len(applied_prompt) == 2: + return applied_prompt[1] + else: + eval_logger.warning("Applied prompt returns empty string") + return self.config.fewshot_delimiter + else: + raise TypeError + + def doc_to_choice(self, doc: Any, doc_to_choice=None) -> List[str]: + if self.prompt is not None: + doc_to_choice = self.prompt + elif doc_to_choice is not None: + doc_to_choice = doc_to_choice + elif self.config.doc_to_choice is None: + eval_logger.error("doc_to_choice was called but not set in config") + else: + doc_to_choice = self.config.doc_to_choice + + if isinstance(doc_to_choice, str): + if doc_to_choice in self.features: + return doc[doc_to_choice] + else: + return ast.literal_eval(utils.apply_template(doc_to_choice, doc)) + elif isinstance(doc_to_choice, list): + return doc_to_choice + elif isinstance(doc_to_choice, dict): + return list(doc_to_choice.values()) + elif callable(doc_to_choice): + return doc_to_choice(doc) + elif hasattr(doc_to_choice, "get_answer_choices_list"): + return doc_to_choice.get_answer_choices_list(doc) + else: + raise TypeError + + def doc_to_image(self, doc: Any, doc_to_image=None) -> Union[int, str, list]: + if doc_to_image is not None: + doc_to_image = doc_to_image + elif self.config.doc_to_image is not None: + doc_to_image = self.config.doc_to_image + else: + return None + + if isinstance(doc_to_image, list): + image_feature = [ + self.doc_to_image(doc, feature) for feature in doc_to_image + ] + return [feature for feature in image_feature if feature is not None] + elif isinstance(doc_to_image, str): + if doc_to_image in self.features: + return doc[doc_to_image] + else: + return ast.literal_eval(utils.apply_template(doc_to_image, doc)) + elif callable(doc_to_image): + return doc_to_image(doc) + else: + return None + + def doc_to_audio(self, doc: Any, doc_to_audio=None) -> Union[int, str, list]: + if doc_to_audio is not None: + doc_to_audio = doc_to_audio + elif self.config.doc_to_audio is not None: + doc_to_audio = self.config.doc_to_audio + else: + return None + + if isinstance(doc_to_audio, list): + audio_feature = [ + self.doc_to_audio(doc, feature) for feature in doc_to_audio + ] + return [feature for feature in audio_feature if feature is not None] + elif isinstance(doc_to_audio, str): + if doc_to_audio in self.features: + return doc[doc_to_audio] + else: + return ast.literal_eval(utils.apply_template(doc_to_audio, doc)) + elif callable(doc_to_audio): + return doc_to_audio(doc) + else: + return None + + def doc_to_prefix(self, doc): + if (gen_prefix := self.config.gen_prefix) is not None: + if gen_prefix in self.features: + return doc[gen_prefix] + else: + return utils.apply_template(gen_prefix, doc) + return None + + def construct_requests( + self, doc: dict, ctx: str, **kwargs + ) -> Union[List[Instance], Instance]: + apply_chat_template = kwargs.pop("apply_chat_template", False) + chat_template: Callable | None = kwargs.pop("chat_template", None) + + aux_arguments = None + + if self.OUTPUT_TYPE == "loglikelihood": + arguments = (ctx, self.doc_to_target(doc)) + elif self.OUTPUT_TYPE == "loglikelihood_rolling": + arguments = (self.doc_to_target(doc),) + elif self.OUTPUT_TYPE == "multiple_choice": + choices = self.doc_to_choice(doc) + target_delimiter = self.config.target_delimiter + if apply_chat_template: + target_delimiter = "" + if self.multiple_input: + # If there are multiple inputs, choices are placed in the ctx + # apply chat_template to choices if apply_chat_template + cont = self.doc_to_target(doc) + + arguments = [ + ( + ctx + + ( + chat_template([{"role": "user", "content": choice}]) + if apply_chat_template + else choice + ), + f"{target_delimiter}{cont}", + ) + for choice in choices + ] + else: + # Otherwise they are placed in the continuation + arguments = [(ctx, f"{target_delimiter}{cont}") for cont in choices] + + # TODO: we should raise a warning telling users this will at most ~2x runtime. + if "acc_mutual_info" in self._metric_fn_list.keys(): + # if we are calculating multiple choice accuracy + # using mutual information instead of raw loglikelihood as metric, need unconditional lls. + + # here mutual info refers to calculating + # log(P(choice|ctx) / P(choice)) = log(P(choice|ctx)) - log(P(choice)) + # in other words normalizing by subtracting the unconditional logprob of each choice. + aux_arguments = [("", f"{choice}") for choice in choices] + + arguments.extend(aux_arguments) + + elif self.OUTPUT_TYPE == "generate_until": + arguments = (ctx, deepcopy(self.config.generation_kwargs)) + + multimodal_arg = {} + if ( + self.config.doc_to_image + ): # TODO: ensure that non-multimodal tasks aren't getting visual args + multimodal_arg = { + **multimodal_arg, + **{"visual": self.doc_to_image(doc)}, + } + + if ( + self.config.doc_to_audio + ): # TODO: ensure that non-multimodal tasks aren't getting audio args + multimodal_arg = { + **multimodal_arg, + **{"audio": self.doc_to_audio(doc)}, + } + + if bool(multimodal_arg): + if isinstance(arguments, list): + arguments = [arg + (multimodal_arg,) for arg in arguments] + else: + arguments = arguments + (multimodal_arg,) + + if self.OUTPUT_TYPE == "multiple_choice": + request_list = [ + Instance( + request_type="loglikelihood", + doc=doc, + arguments=arg, + idx=i, + **kwargs, + ) + for i, arg in enumerate(arguments) + ] + + return request_list + + return Instance( + request_type=self.OUTPUT_TYPE, + doc=doc, + arguments=arguments, + idx=0, + **kwargs, + ) + + def process_results(self, doc, results): + if callable(self.config.process_results): + return self.config.process_results(doc, results) + + result_dict = {} + use_metric = list(self._metric_fn_list.keys()) + if self.OUTPUT_TYPE == "loglikelihood": + results = results[0] + ll, is_greedy = results + return { + **({"perplexity": ll} if "perplexity" in use_metric else {}), + **({"acc": int(is_greedy)} if "acc" in use_metric else {}), + } + elif self.OUTPUT_TYPE == "loglikelihood_rolling": + (loglikelihood,) = results + _words = self.count_words(self.doc_to_target(doc)) + _bytes = self.count_bytes(self.doc_to_target(doc)) + return { + **( + {"word_perplexity": (loglikelihood, _words)} + if "word_perplexity" in use_metric + else {} + ), + **( + {"byte_perplexity": (loglikelihood, _bytes)} + if "byte_perplexity" in use_metric + else {} + ), + **( + {"bits_per_byte": (loglikelihood, _bytes)} + if "bits_per_byte" in use_metric + else {} + ), + } + elif self.OUTPUT_TYPE == "multiple_choice": + lls, is_greedy = zip(*results) + + # retrieve choices in List[str] form, to compute choice lengths, etc. + choices = self.doc_to_choice(doc) + completion_len = np.array([float(len(i)) for i in choices]) + + if ( + 2 * len(choices) == len(lls) + and "acc_mutual_info" in self._metric_fn_list.keys() + ): + # then we are doing mutual info. + # this stores the "dryrun" / unconditional answer loglikelihoods + lls_unconditional = lls[1::2] + if len(lls_unconditional) != len(choices): + raise ValueError + # and this stores our "regular" conditional loglikelihoods + lls = lls[::2] + + pred = np.argmax(lls) + pred_norm = np.argmax(lls / completion_len) + + if self.multiple_input: + gold = self.doc_to_text(doc) + else: + gold = self.doc_to_target(doc) + + gold_index_error = False + if isinstance(gold, list): + gold = [i if i < len(choices) else -100 for i in gold] + if -100 in gold: + gold_index_error = True + else: + if isinstance(gold, int): + gold = gold if gold < len(choices) else -100 + elif isinstance(gold, str): + gold = choices.index(gold) if gold in choices else -100 + + if gold == -100: + gold_index_error = True + + if gold_index_error: + eval_logger.warning( + f"Label index was not in within range of available choices," + f"Sample:\n\n{doc}\n\n" + ) + + if self.multiple_target: + acc = 1.0 if pred in gold else 0.0 + acc_norm = 1.0 if pred_norm in gold else 0.0 + exact_match = int(any([is_greedy[i] if i != -100 else 0 for i in gold])) + else: + acc = 1.0 if pred == gold else 0.0 + acc_norm = 1.0 if pred_norm == gold else 0.0 + # TODO: this gets score of 0 on arc_challenge for pythia-70m. need to test that this works properly + exact_match = int(is_greedy[gold]) if gold != -100 else 0 + + prob_norm = utils.softmax(lls) + + # TODO use keyword arguments to the metric? + # gold, pred, norm stuff, the original lls, + result_dict = { + **({"acc": acc} if "acc" in use_metric else {}), + **({"f1": (gold, pred)} if "f1" in use_metric else {}), + **({"mcc": (gold, pred)} if "mcc" in use_metric else {}), + **({"acc_norm": acc_norm} if "acc_norm" in use_metric else {}), + **({"exact_match": exact_match} if "exact_match" in use_metric else {}), + **( + {"brier_score": (gold, prob_norm)} + if "brier_score" in use_metric + else {} + ), + } + + if "acc_mutual_info" in use_metric: + lls_mutual_info = [ + ll_c - ll_u for ll_c, ll_u in zip(lls, lls_unconditional) + ] + acc_mutual_info = 1.0 if np.argmax(lls_mutual_info) == gold else 0.0 + result_dict["acc_mutual_info"] = acc_mutual_info + + elif self.OUTPUT_TYPE == "generate_until": + gold = self.doc_to_target(doc) + result = results[0] + if self.config.doc_to_choice is not None: + # If you set doc_to_choice, + # it assumes that doc_to_target returns a number. + choices = self.doc_to_choice(doc) + gold = choices[gold] + # we expect multiple_targets to be a list. + elif self.multiple_target: + gold = list(gold) + # TODO: handle this better + elif type(gold) is not type(result) and not ( + "bypass" in self._metric_fn_list.keys() or isinstance(result, list) + ): + # cast gold to the same type as result + gold = type(result)(gold) + + for metric in self._metric_fn_list.keys(): + if self.multiple_target: + # in the case where we have multiple targets, + # return true if any are true + # TODO: this may break for multipLe_target, non zero-or-1 metrics + scores = [] + if not isinstance(gold, list): + # sometimes, a multiple_target dataset has exceptions where one doc has only one string answer + # print(gold) + gold = [gold] + if metric == "exact_match": + result = [result for _ in range(len(gold))] + scores = self._metric_fn_list[metric]( + references=gold, + predictions=result, + **self._metric_fn_kwargs[metric], + )[metric] + result_score = 1.0 if scores > 0.0 else 0.0 + else: + for gold_option in gold: + try: + result_score = self._metric_fn_list[metric]( + references=[gold_option], + predictions=[result], + **self._metric_fn_kwargs[metric], + ) + except ( + TypeError + ): # TODO: this is hacky and I don't want to do it + result_score = self._metric_fn_list[metric]( + [gold_option, result] + ) + if isinstance(result_score, dict): + # TODO: this handles the case where HF evaluate returns a dict. + result_score = result_score[metric] + scores.append(result_score) + if any(scores): + result_score = 1.0 + else: + result_score = 0.0 + else: + try: + result_score = self._metric_fn_list[metric]( + references=[gold], + predictions=[result], + **self._metric_fn_kwargs[metric], + ) + except TypeError: # needed for now in order to use a different interface between our own metrics and HF Evaluate metrics + result_score = self._metric_fn_list[metric]([gold, result]) + if isinstance(result_score, dict): + # TODO: this handles the case where HF evaluate returns a dict. + # This allows for multiple metrics to be returned from the same function + for k, v in result_score.items(): + result_dict[k] = v + else: + result_dict[metric] = result_score + else: + raise ValueError( + f"Passed invalid output_type '{self.OUTPUT_TYPE}' ! Please use one of ", + "'loglikelihood', 'loglikelihood_rolling', 'generate_until' or 'multiple_choice'", + ) + + return result_dict + + def aggregation(self) -> dict: + return self._aggregation_list + + def higher_is_better(self) -> dict: + return self._higher_is_better + + def get_config(self, key: str) -> Any: + return getattr(self._config, key, None) + + @property + def task_name(self) -> Any: + return getattr(self.config, "task", None) + + def __repr__(self): + return ( + f"ConfigurableTask(task_name={getattr(self.config, 'task', None)}," + f"output_type={self.OUTPUT_TYPE}," + f"num_fewshot={getattr(self.config, 'num_fewshot', None)}," + f"num_samples={len(self.eval_docs)})" + ) + + +class MultipleChoiceTask(Task): + OUTPUT_TYPE = "loglikelihood" + + def doc_to_target(self, doc: dict) -> str: + return " " + doc["choices"][doc["gold"]] + + def construct_requests(self, doc: dict, ctx: str, **kwargs) -> List[Instance]: + # TODO: add mutual info here? + return [ + Instance( + request_type="loglikelihood", + doc=doc, + arguments=(ctx, " {}".format(choice)), + idx=i, + **kwargs, + ) + for i, choice in enumerate(doc["choices"]) + ] + + def process_results(self, doc: dict, results: Iterable[Tuple[float, bool]]) -> dict: + results = [ + res[0] for res in results + ] # only retain loglikelihoods, discard is_greedy TODO: do we need is_greedy anywhere? + gold = doc["gold"] + + acc = 1.0 if np.argmax(results) == gold else 0.0 + completion_len = np.array([float(len(i)) for i in doc["choices"]]) + acc_norm = 1.0 if np.argmax(results / completion_len) == gold else 0.0 + + return { + "acc": acc, + "acc_norm": acc_norm, + } + + def higher_is_better(self) -> dict: + return { + "acc": True, + "acc_norm": True, + } + + def aggregation(self) -> dict: + return { + "acc": mean, + "acc_norm": mean, + } + + +class PerplexityTask(Task): + OUTPUT_TYPE = "loglikelihood_rolling" + + def has_training_docs(self) -> bool: + return False + + def fewshot_examples(self, k: int, rnd) -> List: + if k != 0: + raise ValueError( + "The number of fewshot examples must be 0 for perplexity tasks." + ) + return [] + + def fewshot_context(self, doc: dict, num_fewshot: int) -> Literal[""]: + if num_fewshot != 0: + raise ValueError( + "The number of fewshot examples must be 0 for perplexity tasks." + ) + + return "" + + def higher_is_better(self) -> dict: + return { + "word_perplexity": False, + "byte_perplexity": False, + "bits_per_byte": False, + } + + def doc_to_decontamination_query(self, doc): + return doc + + def doc_to_text(self, doc) -> str: + return "" + + def doc_to_target(self, doc): + return doc + + def construct_requests(self, doc: dict, ctx: Optional[str], **kwargs): + if bool(ctx): + raise ValueError + + return Instance( + request_type=self.OUTPUT_TYPE, + doc=doc, + arguments=(self.doc_to_target(doc),), + idx=0, + **kwargs, + ) + + def process_results(self, doc: dict, results: Tuple[float]) -> dict: + (loglikelihood,) = results + words = self.count_words(self.doc_to_target(doc)) + bytes_ = self.count_bytes(self.doc_to_target(doc)) + return { + "word_perplexity": (loglikelihood, words), + "byte_perplexity": (loglikelihood, bytes_), + "bits_per_byte": (loglikelihood, bytes_), + } + + def aggregation(self) -> dict: + return { + "word_perplexity": weighted_perplexity, + "byte_perplexity": weighted_perplexity, + "bits_per_byte": bits_per_byte, + } + + @classmethod + def count_bytes(cls, doc) -> int: + return len(doc.encode("utf-8")) + + @classmethod + def count_words(cls, doc) -> int: + """Downstream tasks with custom word boundaries should override this!""" + return len(re.split(r"\s+", doc)) diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/caching/__init__.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/caching/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/caching/cache.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/caching/cache.py new file mode 100644 index 0000000000000000000000000000000000000000..f8d293b0ff8b1ebac186f5ac078cdb49227562db --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/caching/cache.py @@ -0,0 +1,59 @@ +import hashlib +import logging +import os + +import dill + + +eval_logger = logging.getLogger(__name__) + + +MODULE_DIR = os.path.dirname(os.path.realpath(__file__)) + +OVERRIDE_PATH = os.getenv("LM_HARNESS_CACHE_PATH") + + +PATH = OVERRIDE_PATH if OVERRIDE_PATH else f"{MODULE_DIR}/.cache" + +# This should be sufficient for uniqueness +HASH_INPUT = "EleutherAI-lm-evaluation-harness" + +HASH_PREFIX = hashlib.sha256(HASH_INPUT.encode("utf-8")).hexdigest() + +FILE_SUFFIX = f".{HASH_PREFIX}.pickle" + + +def load_from_cache(file_name: str, cache: bool = False): + if not cache: + return + try: + path = f"{PATH}/{file_name}{FILE_SUFFIX}" + + with open(path, "rb") as file: + cached_task_dict = dill.loads(file.read()) + return cached_task_dict + + except Exception: + eval_logger.debug(f"{file_name} is not cached, generating...") + pass + + +def save_to_cache(file_name, obj): + if not os.path.exists(PATH): + os.mkdir(PATH) + + file_path = f"{PATH}/{file_name}{FILE_SUFFIX}" + + eval_logger.debug(f"Saving {file_path} to cache...") + with open(file_path, "wb") as file: + file.write(dill.dumps(obj)) + + +# NOTE the "key" param is to allow for flexibility +def delete_cache(key: str = ""): + files = os.listdir(PATH) + + for file in files: + if file.startswith(key) and file.endswith(FILE_SUFFIX): + file_path = f"{PATH}/{file}" + os.unlink(file_path) diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/decontamination/__init__.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/decontamination/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/decontamination/archiver.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/decontamination/archiver.py new file mode 100644 index 0000000000000000000000000000000000000000..c132232116c2ae5f5ab1dc3a2a0afc0dbd4ef1bd --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/decontamination/archiver.py @@ -0,0 +1,174 @@ +import datetime +import io +import json +import mmap +import os +from pathlib import Path +from typing import Any + +import jsonlines +import tqdm +import zstandard + + +def json_serial(obj: Any) -> str: + """JSON serializer for objects not serializable by default json code""" + + if isinstance(obj, (datetime.datetime,)): + return obj.isoformat() + raise TypeError("Type %s not serializable" % type(obj)) + + +# Modified version of lm_dataformat Archive for single file. +class Archive: + def __init__(self, file_path: str, compression_level: int = 3) -> None: + self.file_path = file_path + dir_name = os.path.dirname(file_path) + if dir_name: + os.makedirs(dir_name, exist_ok=True) + self.fh = open(self.file_path, "wb") + self.cctx = zstandard.ZstdCompressor(level=compression_level) + self.compressor = self.cctx.stream_writer(self.fh) + + def add_data(self, data, meta=None) -> None: + if meta is None: + meta = {} + self.compressor.write( + json.dumps({"text": data, "meta": meta}, default=json_serial).encode( + "UTF-8" + ) + + b"\n" + ) + + def commit(self) -> None: + self.compressor.flush(zstandard.FLUSH_FRAME) + self.fh.flush() + self.fh.close() + + +# Modified version of lm_dataformat Reader with self.fh set, allowing peeking for tqdm. +class Reader: + def __init__(self) -> None: + pass + + def read( + self, + file, + get_meta: bool = False, + autojoin_paragraphs: bool = True, + para_joiner: str = "\n\n", + ): + with open(file, "rb") as fh: + self.fh = fh + cctx = zstandard.ZstdDecompressor() + reader = io.BufferedReader(cctx.stream_reader(fh)) + rdr = jsonlines.Reader(reader) + for ob in rdr: + # naive jsonl where each object is just the string itself, with no meta. For legacy compatibility. + if isinstance(ob, str): + assert not get_meta + yield ob + continue + + text = ob["text"] + + if autojoin_paragraphs and isinstance(text, list): + text = para_joiner.join(text) + + if get_meta: + yield text, (ob["meta"] if "meta" in ob else {}) + else: + yield text + + +class TextArchive: + def __init__(self, file_path, mode: str = "rb+") -> None: + self.file_path = file_path + dir_name = os.path.dirname(file_path) + if dir_name: + os.makedirs(dir_name, exist_ok=True) + + if not os.path.exists(file_path): + Path(file_path).touch() + + self.fh = open(self.file_path, mode) + + def add_data(self, data) -> None: + self.fh.write(data.encode("UTF-8") + b"\n") + + def commit(self) -> None: + self.fh.flush() + self.fh.close() + + +class TextReader: + def __init__(self, file_path) -> None: + self.file_path = file_path + + # Optimized mmap read with infrequent tqdm updates to maintain speed + # Tested up to 250MB/s. + def read_tqdm(self, update_frequency: int = 10000): + current_file_position = 0 + line_counter = 0 + with ( + open(self.file_path, "r", encoding="utf-8") as fh, + tqdm.tqdm( + total=os.path.getsize(self.file_path), + dynamic_ncols=True, + unit="byte", + unit_scale=1, + ) as progress, + ): + with mmap.mmap(fh.fileno(), length=0, access=mmap.ACCESS_READ) as mmap_obj: + for line in iter(mmap_obj.readline, b""): + line = line.decode("utf-8") + line_counter += 1 + if line_counter == update_frequency: + new_file_pos = mmap_obj.tell() + bytes_read = new_file_pos - current_file_position + current_file_position = new_file_pos + progress.update(bytes_read) + line_counter = 0 + yield line[:-1] + + def read_and_tell(self): + current_file_position = 0 + with open(self.file_path, "r", encoding="utf8") as fh: + with mmap.mmap(fh.fileno(), length=0, access=mmap.ACCESS_READ) as mmap_obj: + for line in iter(mmap_obj.readline, b""): + line = line.decode("utf-8") + new_file_pos = mmap_obj.tell() + raw_bytes_read = new_file_pos - current_file_position + current_file_position = new_file_pos + yield line[:-1], raw_bytes_read + + def read(self): + with open(self.file_path, "r", encoding="utf8") as fh: + with mmap.mmap(fh.fileno(), length=0, access=mmap.ACCESS_READ) as mmap_obj: + for line in iter(mmap_obj.readline, b""): + line = line.decode("utf-8") + yield line[:-1] + + def read_slow(self): + with open(self.file_path, "r", encoding="utf8") as fh: + while True: + line = fh.readline() + if line == -1 or line == "": + break + else: + yield line[:-1] + + +# Optimized for speed. Decompresses the archive in shell before +# using the mmap'd TextReader. +class ZStdTextReader: + def __init__(self, file) -> None: + self.file = file + + def read_tqdm(self): + decompressed_file = self.file[:-4] + print("Decompressing file, please wait...") + os.system(f"zstd -d {self.file}") # linux decompress is faster + reader = TextReader(decompressed_file) + yield from reader.read_tqdm() + os.remove(decompressed_file) diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/decontamination/decontaminate.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/decontamination/decontaminate.py new file mode 100644 index 0000000000000000000000000000000000000000..2d1250d39bf7cd0272e412452d970ec7c52992c5 --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/decontamination/decontaminate.py @@ -0,0 +1,166 @@ +import collections +import glob +import json +import os +import pickle +import random +import time + +from .archiver import ZStdTextReader +from .janitor import Janitor, word_ngrams + + +# Was used for testing the evaluator decoupled from the full logic below +def get_train_overlap_stub(docs: dict, ngrams_path: str, ngrams_n_size: str): + simulated_overlap = 0.1 + contaminated = int(len(docs) * simulated_overlap) + return random.sample(range(len(docs)), contaminated) + + +# Returns a dictionary containing all overlapping documents in each +# task. In the standard use case, an overlap occurs when any of the 13-grams +# found in the task document exist in the training set documents. +# +# To generate 13-grams for the pile see scripts/clean_training_data. The final output of these +# scripts are an info.json file containing the n_gram_size (13) and a bunch of "ngrams_{x}.bkt.txt.sorted.zst" +# files. These should exist in the "ngrams_path" provided to this function. + + +# Algorithm: +# 1. Build lookups for each dataset {ngram: list(document_ids)} +# 2. Merge into an overall lookup {ngram: [(task_name, task_set, doc_ids),]} +# 3. Full scan the 13-grams from the training set against the merged lookup, +# saving matches in the "duplicates" dictionary {(task_name, task_set): set(doc_ids)} +# 4. Strip the task_set from the dictionary keys and return +# +# We cache the task+set lookups as well as the overlaps. +def get_train_overlap(docs_by_task_set: dict, ngrams_path: str, limit: int) -> dict: + # return get_train_overlap_stub(docs, ngrams_path, ngrams_n_size) + + info_dict_path = os.path.join(ngrams_path, "info.json") + info_dict = json.load(open(info_dict_path, "r", encoding="utf-8")) + ngrams_n_size = info_dict["ngram_size"] + + janitor = Janitor() + + # Build lookup for each dataset first in case we use different task combinations later + print("Building Lookups...") + start = time.perf_counter() + + def get_overlaps_dump_path(task_name, task_set, ngrams_n_size, limit) -> str: + return f"data/{task_name}/{task_set}_{ngrams_n_size}grams_limit{limit}.overlaps" + + lookups = {} + duplicates = {} # (task_name, task_set): set(doc_ids)} + sets_to_decontaminate = len(docs_by_task_set.keys()) + + for (task_name, task_set), docs in docs_by_task_set.items(): + if not os.path.exists(f"data/{task_name}"): + os.mkdir(f"data/{task_name}") + + # Check if we've decontaminated this combination before + overlaps_dump_path = get_overlaps_dump_path( + task_name, task_set, ngrams_n_size, limit + ) + if os.path.exists(overlaps_dump_path): + duplicates[(task_name, task_set)] = pickle.load( + open(overlaps_dump_path, "rb") + ) + sets_to_decontaminate -= 1 + continue + else: + duplicates[(task_name, task_set)] = set() + + # Build/load the task lookup {ngram: set(documents)}. + task_set_lookup_path = ( + f"data/{task_name}/{task_set}_{ngrams_n_size}grams_limit{limit}.lookup" + ) + if os.path.exists(task_set_lookup_path): + print(f"{task_set_lookup_path} available, loading...") + lookups[(task_name, task_set)] = pickle.load( + open(task_set_lookup_path, "rb") + ) + else: + print(f"{task_set_lookup_path} not available, building...") + lookup = collections.defaultdict(set) + + for doc_id, document in enumerate(docs): + ngrams = word_ngrams(janitor.normalize_string(document), ngrams_n_size) + for ngram in ngrams: + lookup[ngram].add(doc_id) + + pickle.dump(lookup, open(task_set_lookup_path, "wb")) + lookups[(task_name, task_set)] = lookup + + elapsed = time.perf_counter() - start + print(f"Building lookups took {elapsed:0.5f} seconds.") + + matched_ngrams = [] + + if sets_to_decontaminate > 0: + print("Merging lookups...") + start = time.perf_counter() + merged_lookup = collections.defaultdict(list) + for (task_name, task_set), lookup in lookups.items(): + for ngram, doc_ids in lookup.items(): + merged_lookup[ngram].append((task_name, task_set, doc_ids)) + + elapsed = time.perf_counter() - start + print(f"Merging lookups took {elapsed:0.5f} seconds.") + + print(f"{ngrams_n_size} grams files found in {ngrams_path}:") + files = glob.glob(os.path.join(ngrams_path, "*.sorted.zst")) + print(files) + + for file in files: + start = time.perf_counter() + print(f"Scanning {file}") + reader = ZStdTextReader(file) + total_ngrams = 0 + unique_ngrams = 0 + matching_unique = 0 + non_matching_unique = 0 + + current_ngram = "" + for line in reader.read_tqdm(): # Scan training set ngrams file + total_ngrams += 1 + [ngram, document_id] = line.rsplit(" ", 1) + if ( + ngram != current_ngram + ): # Only need to match the ngram once in training set + unique_ngrams += 1 + current_ngram = ngram + if ngram in merged_lookup: + matched_ngrams.append(ngram) # For logging + matching_unique += 1 + for task_name, task_set, doc_ids in merged_lookup[ngram]: + task_doc_set = duplicates[(task_name, task_set)] + for doc_id in doc_ids: # Record contamination across all relevant task/set combos + task_doc_set.add(doc_id) + del merged_lookup[ngram] # No point matching again + else: + non_matching_unique += 1 + + print(f"Total Ngrams: {total_ngrams}") + print(f"Unique Ngrams: {unique_ngrams}") + print(f"Unique Matching: {matching_unique}") + print(f"Unique Non Matching: {non_matching_unique}") + print("Matched ngrams:") + for ngram in matched_ngrams: + print(ngram) + + elapsed = time.perf_counter() - start + print(f"Read took {elapsed:0.5f} seconds.") + print(f"Speed: {(os.path.getsize(file) / 1000000.0) / elapsed}MB/second") + + print(duplicates) + + # Dump overlaps separately + for (task_name, task_set), doc_ids in duplicates.items(): + overlaps_dump_path = get_overlaps_dump_path( + task_name, task_set, ngrams_n_size, limit + ) + pickle.dump(doc_ids, open(overlaps_dump_path, "wb")) + + # Strip task set and return + return {task_name: doc_ids for (task_name, task_set), doc_ids in duplicates.items()} diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/decontamination/janitor.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/decontamination/janitor.py new file mode 100644 index 0000000000000000000000000000000000000000..cedf8a5717aa8156674836ba236fdcabf36e0487 --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/decontamination/janitor.py @@ -0,0 +1,328 @@ +import pickle +import re +import string +import traceback +from typing import Iterator, List, Sequence, Tuple, TypeVar + + +# This is a cpp module. Compile janitor_util.cpp with: +# c++ -O3 -Wall -shared -std=c++11 -fPIC $(python3 -m pybind11 --includes) janitor_util.cpp -o janitor_util$(python3-config --extension-suffix) -undefined dynamic_lookup +try: + import janitor_util + + JANITOR_CPP = True +except Exception: + print("WARNING: C++ module could not be loaded. Janitor running in python mode") + traceback.print_exc() + JANITOR_CPP = False + +T = TypeVar("T") + + +# Implementation from nltk source +# https://www.nltk.org/_modules/nltk/util.html +def form_ngrams(sequence: Iterator[T], n: int) -> Iterator[Tuple[T, ...]]: + history = [] + while n > 1: + # PEP 479, prevent RuntimeError from being raised when StopIteration bubbles out of generator + try: + next_item = next(sequence) + except StopIteration: + # no more data, terminate the generator + return + history.append(next_item) + n -= 1 + for item in sequence: + history.append(item) + yield tuple(history) + del history[0] + + +def word_ngrams(s: str, n: int) -> Iterator[str]: + """Splits a string into ngram words""" + tokens = s.split() # not a generator :( + ngram_seqs = form_ngrams(iter(tokens), n) + return (" ".join(ngram) for ngram in ngram_seqs) + + +# Does character sequences only - combined faster function to play around with later +# def word_ngrams_indices_combined(sequence, n): +# current_word = "" +# history = [] +# gap = False; +# start = 0 +# end = 0 +# for character in sequence: +# if character == " ": +# if not gap: +# gap = True +# history.append(current_word) +# end += len(current_word) - 1 +# current_word = "" +# if len(history) == n: +# yield (tuple(history), start, end) +# del history[0] +# start = end + 1 +# end = start +# else: +# gap = False +# current_word += character + + +# https://stackoverflow.com/questions/13734451/string-split-with-indices-in-python +def split_indices(s: str) -> Iterator[Tuple[str, Tuple[int, int]]]: + """Splits a string on whitespaces and records the indices of each in the original string. + @:return generator((word, (start_idx, end_idx)), ...) + """ + return ((m.group(0), (m.start(), m.end() - 1)) for m in re.finditer(r"\S+", s)) + + +def word_ngrams_indices(s: str, n: int) -> Iterator[Tuple[str, Tuple[int, int]]]: + """Splits a string into pairs of (ngram words, their start/end indices)""" + tokens_with_indices = split_indices(s) + + # Generator of ngrams of (word, idx_pairs) + # ( + # [(word, (start,end)), (word, (start, end))...], + # [(word, (start, end)), ...], + # ... + # ) + ngram_seqs_with_indices = form_ngrams(tokens_with_indices, n) + + # Generator of pairs of word and index ngrams + # ( + # ([word, word, ...], [(start,end), (start,end), ...]), + # ... + # ) + ngram_indices_pairs = ( + zip(*ngram_with_indices) for ngram_with_indices in ngram_seqs_with_indices + ) + + # Generator of ( (word_ngram, (start, end)), (word_ngram, start, end)), ...) + return ( + (" ".join(ngram_seq), (indices[0][0], indices[-1][1])) + for ngram_seq, indices in ngram_indices_pairs + ) + + +class Janitor: + # FIXME delete_chars: Should anything else go here? Special chars? + def __init__( + self, + ngram_n: int = 13, + window_to_remove: int = 200, + too_dirty_cutoff: int = 10, + minimum_slice_length: int = 200, + delete_chars: str = string.punctuation, + ) -> None: + self.ngram_n = ngram_n + self.window_to_remove = window_to_remove + self.too_dirty_cutoff = too_dirty_cutoff + self.minimum_slice_length = minimum_slice_length + self.delete_chars = delete_chars + + self.dirt_ngrams = set() + + # If in python, we'll translate uppercase to lowercase and delete naughty characters. + # This is fast by python standards + # https://stackoverflow.com/questions/638893/what-is-the-most-efficient-way-in-python-to-convert-a-string-to-all-lowercase-st + self.translation_table = str.maketrans( + string.ascii_lowercase + string.ascii_uppercase, # These characters + string.ascii_lowercase * 2, # Become these characters + self.delete_chars, # These are deleted + ) + + ############## + # I/O for saving contamination ngrams + ############## + + def save_contamination_ngrams(self, filename: str) -> None: + with open(filename, "wb") as fp: + pickle.dump(filename, fp) + + def load_contamination_ngrams(self, filename: str) -> None: + with open(filename, "rb") as fp: + self.dirt_ngrams = pickle.load(fp) + + ############## + # Call these :) + ############## + + def register_contaminant(self, dirt_string: str) -> None: + """Register a string as contamination to be removed, e.g. a test set + This breaks the dirt_string into ngrams to store for future cleaning""" + if JANITOR_CPP: + return self.register_contaminant_cpp(dirt_string) + else: + print("WARNING: Janitor running in python mode") + return self.register_contaminant_python(dirt_string) + + def clean(self, dirty_string: str) -> List[str]: + """Clean a string (e.g. a training set) by removing all ngrams previously + registered as contaminants. Returns a list of clean chunks, or empty if + the string was too dirty""" + if JANITOR_CPP: + return self.clean_cpp(dirty_string) + else: + print("WARNING: Janitor running in python mode") + return self.clean_python(dirty_string) + + def _split_chunks( + self, dirty_string: str, dirty_parts: Sequence[Tuple] + ) -> List[str]: + clean_chunks = [] + splice_idx = 0 + end = -1 + for i, (ngram, start, end) in enumerate(dirty_parts): + if i >= self.too_dirty_cutoff: + return [] + start = max(0, start - self.window_to_remove) + end = min(len(dirty_string), end + self.window_to_remove) + + if start - splice_idx > self.minimum_slice_length: + clean_chunks.append(dirty_string[splice_idx:start]) + splice_idx = end + + if end < len(dirty_string) - self.minimum_slice_length: + clean_chunks.append(dirty_string[end + 1 :]) + + return clean_chunks + + ############## + # Fast C++ + ############## + + def register_contaminant_cpp(self, dirt_string) -> None: + self.dirt_ngrams.update( + janitor_util.clean_ngram(dirt_string, self.delete_chars, self.ngram_n) + ) + + def clean_cpp(self, dirty_string: str) -> List[str]: + contamination_indices = janitor_util.clean_ngram_with_indices( + dirty_string, self.delete_chars, self.ngram_n + ) + return self._split_chunks(dirty_string, contamination_indices) + + ############## + # Slow python + ############## + + def normalize_string(self, s: str) -> str: + return s.translate(self.translation_table) + + def register_contaminant_python(self, dirt_string: str) -> None: + self.dirt_ngrams.update( + word_ngrams(self.normalize_string(dirt_string), self.ngram_n) + ) + + def clean_python(self, dirty_string: str) -> List[str]: + contamination_indices = ( + (None, *idx_pair) + for dirty_ngram, idx_pair in word_ngrams_indices(dirty_string, self.ngram_n) + if self.normalize_string(dirty_ngram) in self.dirt_ngrams + ) + return self._split_chunks(dirty_string, contamination_indices) + + +################################################################## +# Tests +################################################################# + +# def print_cpp(): +# source = """ ,, I'm a very !dirty,, ,, dirty boy. Clean me daddy. \n\nhe he he hehe heh. lastword """ * 2 + +# for i in range(1, 10, 2): +# pprint(janitor_util.clean_ngram(source, string.punctuation, i)) +# for ngram, start, end in \ +# janitor_util.clean_ngram_with_indices(source, string.punctuation, i): +# print(ngram, "\t", start, end, source[start:end].replace("\n", "\\n")) + + +# def test_cpp(): +# source = """ ,, I'm a very !dirty,, ,, dirty boy. Clean me daddy. \n\nhe he he hehe heh. lastword """ * 2 +# contaminant = "dirty boy. Clean he he" + +# jan_python = Janitor() +# jan_cpp = Janitor() + +# jan_python.register_contaminant_python(contaminant) +# jan_cpp.register_contaminant(contaminant) + +# assert jan_python.dirt_ngrams == jan_cpp.dirt_ngrams, (jan_python.dirt_ngrams, jan_cpp.dirt_ngrams) + +# assert jan_python.clean_python(source) == jan_cpp.clean(source), \ +# (jan_python.clean_python(source), jan_cpp.clean(source)) + +# print("Passed test, python==cpp") + + +# def benchmark(): +# # Download and put in data folder: enwik8 (100 MB) from https://cs.fit.edu/~mmahoney/compression/textdata.html +# setup = \ +# """ +# with open("data/enwik8", "r") as f: +# data = f.read() +# jan = Janitor(too_dirty_cutoff=1000) +# jan.register_contaminant(''' +# theories is that there is a connection between "geekdom" and autism. +# This is hinted, for instance, by a ''Wired Magazine'' article in 2001 entitled " +# The [[Geek]] Syndrome", which is a point argued by many in the autism rights +# movement{{ref|Wired}}. This article, many professionals assert, is just one example of +# the media's application of mental disease labels to what is actually variant normal behavior +# &mdash;they argue that shyness, lack of athletic ability or social skills, and intellectual +# interests, even when they seem unusual to others, are not in themselves signs of autism or +# Asperger's syndrome. Others assert that it is actually the medical profession which is applying +# mental disease labels to children who in the past would have simply been accepted as a little +# different or even labeled 'gifted'. See [[clinomorphism]] for further discussion of this issue. +# Due to the recent publicity surrounding autism and autis +# ultan Al Nahyan]] granted [[Petroleum]] concessions, and oil was first found in 1958. At first, +# oil money had a marginal impact. A few lowrise concete buildings were erected, and the first +# paved road was completed in 1961, but Sheikh Shakbut, uncertain whether the new oil royalties +# would last, took a cautious approach, preferring to save the revenue rather than investing it in +# development. His brother, [[Zayed bin Sultan Al Nahayan]], saw that oil wealth had the potential +# to transform Abu Dhabi. The ruling Al Nahayan family decided that Sheikh Zayed should replace his +# brother as Ruler and carry out his vision of developing the country. On [[August 6]], [[1966]], +# with the assistance of the British, Sheikh Zayed became the new ruler. See generally, Al-Fahim, M, +# ''From Rags to Riches: A Story of Abu Dhabi'', Chapter Six (London Centre of Arab Studies, 1995), +# ISBN 1 900404 00 1. With the announcement by Britain in 1968 that it would withdraw from the +# Gulf area by 1971, Sheikh Zayed became the main driving force behind the formation of the +# [[United Arab Emirates]]. After the Emirates gained independence in 1971, +# ''') +# """ + +# n = 1 +# print(f"Timing {n} run on 100 MB") +# print("Register contaminant") +# # print("\tPython", timeit.timeit("jan.register_contaminant_python(data)", setup=setup, globals=globals(), number=n)) +# print("\tCpp", timeit.timeit("jan.register_contaminant(data)", setup=setup, globals=globals(), number=n)) + +# print("Clean") +# # print("\tPython", timeit.timeit("jan.clean_python(data)", setup=setup, globals=globals(), number=n)) +# print("\tCpp", timeit.timeit("jan.clean(data)", setup=setup, globals=globals(), number=n)) + + +# def test_janitor_general(): +# source = """ ,, I'm a very !dirty,, ,, dirty boy. Clean me daddy. \n\nhe he he hehe heh. lastword """ * 2 +# contaminant = "dirty boy. Clean he he" + +# jan = Janitor(ngram_n=3) +# jan.register_contaminant(contaminant) +# cleaned = " ".join(jan.clean(source)) +# for contam in jan.dirt_ngrams: +# assert contam not in cleaned, contam + +# filename = "data/saved_contam" +# jan.save_contamination_ngrams(filename) + +# jan = Janitor(ngram_n=3) +# jan.load_contamination_ngrams(filename) +# cleaned = " ".join(jan.clean(source)) +# for contam in jan.dirt_ngrams: +# assert contam not in cleaned, contam + + +# if __name__ == "__main__": +# test() +# # print_cpp() +# # test_cpp() +# # benchmark() diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/evaluator.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..0d26a64c6f999690e8810419ae930a3cd74a34f5 --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/evaluator.py @@ -0,0 +1,736 @@ +import itertools +import json +import logging +import random +import time +from collections import defaultdict +from typing import TYPE_CHECKING, List, Optional, Union + +import numpy as np +import torch + +import lm_eval.api.metrics +import lm_eval.api.registry +import lm_eval.api.task +import lm_eval.models +from lm_eval.caching.cache import delete_cache +from lm_eval.evaluator_utils import ( + consolidate_group_results, + consolidate_results, + get_sample_size, + get_subtask_list, + get_task_list, + prepare_print_tasks, + print_writeout, + run_task_tests, +) +from lm_eval.loggers import EvaluationTracker +from lm_eval.loggers.utils import add_env_info, add_tokenizer_info, get_git_commit_hash +from lm_eval.tasks import ( + TaskManager, + get_task_dict, +) +from lm_eval.utils import ( + handle_non_serializable, + hash_string, + positional_deprecated, + setup_logging, + simple_parse_args_string, +) + + +if TYPE_CHECKING: + from lm_eval.api.model import LM + from lm_eval.api.task import Task + +eval_logger = logging.getLogger(__name__) + + +@positional_deprecated +def simple_evaluate( + model, + model_args: Optional[Union[str, dict]] = None, + tasks: Optional[List[Union[str, dict, object]]] = None, + num_fewshot: Optional[int] = None, + batch_size: Optional[Union[int, str]] = None, + max_batch_size: Optional[int] = None, + device: Optional[str] = None, + use_cache: Optional[str] = None, + cache_requests: bool = False, + rewrite_requests_cache: bool = False, + delete_requests_cache: bool = False, + limit: Optional[Union[int, float]] = None, + bootstrap_iters: int = 100000, + check_integrity: bool = False, + write_out: bool = False, + log_samples: bool = True, + evaluation_tracker: Optional[EvaluationTracker] = None, + system_instruction: Optional[str] = None, + apply_chat_template: Union[bool, str] = False, + fewshot_as_multiturn: bool = False, + gen_kwargs: Union[str, dict, None] = None, + task_manager: Optional[TaskManager] = None, + verbosity=None, + predict_only: bool = False, + random_seed: int = 0, + numpy_random_seed: int = 1234, + torch_random_seed: int = 1234, + fewshot_random_seed: int = 1234, + confirm_run_unsafe_code: bool = False, + metadata: Optional[dict] = None, +): + """Instantiate and evaluate a model on a list of tasks. + + :param model: Union[str, LM] + Name of model or LM object, see lm_eval.models.get_model + :param model_args: Optional[str, dict] + String or dict arguments for each model class, see LM.create_from_arg_string and LM.create_from_arg_object. + Ignored if `model` argument is a LM object. + :param tasks: list[Union[str, dict, Task]] + List of task names or Task objects. Task objects will be taken to have name task.EVAL_HARNESS_NAME if defined and type(task).__name__ otherwise. + :param num_fewshot: int + Number of examples in few-shot context + :param batch_size: int or str, optional + Batch size for model + :param max_batch_size: int, optional + Maximal batch size to try with automatic batch size detection + :param device: str, optional + PyTorch device (e.g. "cpu" or "cuda:0") for running models + :param use_cache: str, optional + A path to a sqlite db file for caching model responses. `None` if not caching. + :param cache_requests: bool, optional + Speed up evaluation by caching the building of dataset requests. `None` if not caching. + :param rewrite_requests_cache: bool, optional + Rewrites all the request cache if set to `True`. `None` if not desired. + :param delete_requests_cache: bool, optional + Deletes all the request cache if set to `True`. `None` if not desired. + :param limit: int or float, optional + Limit the number of examples per task (only use this for testing), If <1, limit is a percentage of the total number of examples. + :param bootstrap_iters: + Number of iterations for bootstrap statistics, used when calculating stderrs. set to 0 for no stderr calculations to be performed. + :param check_integrity: bool + Whether to run the relevant part of the test suite for the tasks + :param write_out: bool + If True, write out an example document and model input for checking task integrity + :param log_samples: bool + If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis + :param system_instruction: str + System instruction to be applied to the prompt + :param apply_chat_template: Union[bool, str] + Specifies whether to apply a chat template to the prompt. + - If set to True, the default chat template is applied. + - If set to a string, applies the specified chat template by name. + Defaults to False (no chat template applied). + :param fewshot_as_multiturn: bool + Whether to provide the fewshot examples as a multiturn conversation or a single user turn. + :param gen_kwargs: dict or comma-separated string + Arguments for model generation + Ignored for all tasks with loglikelihood output_type + :param verbosity: str + Verbosity level for logging + :param predict_only: bool + If true only model outputs will be generated and returned. Metrics will not be evaluated + :param random_seed: int + Random seed for python's random module. If set to None, the seed will not be set. + :param numpy_random_seed: int + Random seed for numpy. If set to None, the seed will not be set. + :param torch_random_seed: int + Random seed for torch. If set to None, the seed will not be set. + :param fewshot_random_seed: int + Random seed for fewshot sampler random generator. If set to None, the seed of generator will be set to None. + :param metadata: dict + Additional metadata to be added to the task manager. Will get passed to the download function of the task. + + return + Dictionary of results + """ + if verbosity is not None: + setup_logging(verbosity=verbosity) + start_date = time.time() + + if isinstance(model_args, str) and ( + "instruct" in model_args and not apply_chat_template + ): + eval_logger.warning( + "Instruct model detected, but chat template not applied. Recommend setting `apply_chat_template` (optionally `fewshot_as_multiturn`)." + ) + + if delete_requests_cache: + eval_logger.info("Deleting requests cache...") + delete_cache() + + seed_message = [] + if random_seed is not None: + # See https://github.com/EleutherAI/lm-evaluation-harness/pull/1412 + seed_message.append(f"Setting random seed to {random_seed}") + random.seed(random_seed) + + if numpy_random_seed is not None: + seed_message.append(f"Setting numpy seed to {numpy_random_seed}") + np.random.seed(numpy_random_seed) + + if torch_random_seed is not None: + seed_message.append(f"Setting torch manual seed to {torch_random_seed}") + torch.manual_seed(torch_random_seed) + + if fewshot_random_seed is not None: + seed_message.append(f"Setting fewshot manual seed to {fewshot_random_seed}") + + if seed_message: + eval_logger.info(" | ".join(seed_message)) + + if tasks is None: + tasks = [] + if len(tasks) == 0: + raise ValueError( + "No tasks specified, or no tasks found. Please verify the task names." + ) + + if gen_kwargs is not None: + if isinstance(gen_kwargs, str): + gen_kwargs = simple_parse_args_string(gen_kwargs) + eval_logger.warning( + f"generation_kwargs: {gen_kwargs} specified through cli, these settings will update set parameters in yaml tasks. " + "Ensure 'do_sample=True' for non-greedy decoding!" + ) + if not gen_kwargs: + gen_kwargs = None + + if isinstance(model, str): + if model_args is None: + eval_logger.warning("model_args not specified. Using defaults.") + model_args = "" + + if isinstance(model_args, dict): + eval_logger.info( + f"Initializing {model} model, with arguments: {model_args}" + ) + lm = lm_eval.api.registry.get_model(model).create_from_arg_obj( + model_args, + { + "batch_size": batch_size, + "max_batch_size": max_batch_size, + "device": device, + }, + ) + + else: + eval_logger.info( + f"Initializing {model} model, with arguments: {simple_parse_args_string(model_args)}" + ) + lm = lm_eval.api.registry.get_model(model).create_from_arg_string( + model_args, + { + "batch_size": batch_size, + "max_batch_size": max_batch_size, + "device": device, + }, + ) + else: + if not isinstance(model, lm_eval.api.model.LM): + raise TypeError( + f"The value of `model` passed to simple_evaluate() was of type {type(model)}, but is required to be a subclass of lm_eval.api.model.LM . This may be because you are passing an initialized Hugging Face PreTrainedModel without having wrapped it in `lm_eval.models.huggingface.HFLM(pretrained=my_model)` first." + ) + eval_logger.info("Using pre-initialized model") + lm = model + + if use_cache is not None: + eval_logger.info(f"Using cache at {use_cache + '_rank' + str(lm.rank) + '.db'}") + lm = lm_eval.api.model.CachingLM( + lm, + use_cache + # each rank receives a different cache db. + # necessary to avoid multiple writes to cache at once + + "_rank" + + str(lm.rank) + + ".db", + ) + + if task_manager is None: + metadata = ( + simple_parse_args_string(model_args) + if isinstance(model_args, str) + else model_args + if isinstance(model_args, dict) + else {} + ) | (metadata or {}) + task_manager = TaskManager(metadata=metadata) + + task_dict = get_task_dict( + tasks, + task_manager, + ) + + # helper function to recursively apply config overrides to leaf subtasks, skipping their constituent groups. + # (setting of num_fewshot ; bypassing metric calculation ; setting fewshot seed) + def _adjust_config(task_dict): + adjusted_task_dict = {} + for task_name, task_obj in task_dict.items(): + if isinstance(task_obj, dict): + adjusted_task_dict = { + **adjusted_task_dict, + **{task_name: _adjust_config(task_obj)}, + } + + else: + if task_obj.get_config("output_type") == "generate_until": + if gen_kwargs is not None: + task_obj.set_config( + key="generation_kwargs", value=gen_kwargs, update=True + ) + eval_logger.info( + f"{task_obj.config.task}: Using gen_kwargs: {task_obj.config.generation_kwargs}" + ) + + if predict_only: + eval_logger.info( + f"Processing {task_name} in output-only mode. Metrics will not be calculated!" + ) + # we have to change the class properties post-hoc. This is pretty hacky. + task_obj.override_metric(metric_name="bypass") + + # override tasks' fewshot values to the provided num_fewshot arg value + # except if tasks have it set to 0 manually in their configs--then we should never overwrite that + if num_fewshot is not None: + if (default_num_fewshot := task_obj.get_config("num_fewshot")) == 0: + eval_logger.info( + f"num_fewshot has been set to 0 for {task_name} in its config. Manual configuration will be ignored." + ) + else: + eval_logger.warning( + f"Overwriting default num_fewshot of {task_name} from {default_num_fewshot} to {num_fewshot}" + ) + task_obj.set_config(key="num_fewshot", value=num_fewshot) + else: + # if num_fewshot not provided, and the task does not define a default one, default to 0 + if ( + default_num_fewshot := task_obj.get_config("num_fewshot") + ) is None: + task_obj.set_config(key="num_fewshot", value=0) + # fewshot_random_seed set for tasks, even with a default num_fewshot (e.g. in the YAML file) + task_obj.set_fewshot_seed(seed=fewshot_random_seed) + + adjusted_task_dict[task_name] = task_obj + + return adjusted_task_dict + + task_dict = _adjust_config(task_dict) + + if check_integrity: + run_task_tests(task_list=tasks) + + if evaluation_tracker is not None: + evaluation_tracker.general_config_tracker.log_experiment_args( + model_source=model, + model_args=model_args, + system_instruction=system_instruction, + chat_template=lm.chat_template(apply_chat_template) + if apply_chat_template + else None, + fewshot_as_multiturn=fewshot_as_multiturn, + ) + + results = evaluate( + lm=lm, + task_dict=task_dict, + limit=limit, + cache_requests=cache_requests, + rewrite_requests_cache=rewrite_requests_cache, + bootstrap_iters=bootstrap_iters, + write_out=write_out, + log_samples=True if predict_only else log_samples, + system_instruction=system_instruction, + apply_chat_template=apply_chat_template, + fewshot_as_multiturn=fewshot_as_multiturn, + verbosity=verbosity, + confirm_run_unsafe_code=confirm_run_unsafe_code, + ) + if verbosity is not None: + setup_logging(verbosity=verbosity) + + if lm.rank == 0: + if isinstance(model, str): + model_name = model + elif hasattr(model, "config") and hasattr(model.config, "_name_or_path"): + model_name = model.config._name_or_path + else: + model_name = type(model).__name__ + + # add info about the model and few shot config + results["config"] = { + "model": model_name, + "model_args": model_args, + } + # add more detailed model info if available + if isinstance(lm, lm_eval.models.huggingface.HFLM): + results["config"].update(lm.get_model_info()) + # add info about execution + results["config"].update( + { + "batch_size": batch_size, + "batch_sizes": ( + list(lm.batch_sizes.values()) if hasattr(lm, "batch_sizes") else [] + ), + "device": device, + "use_cache": use_cache, + "limit": limit, + "bootstrap_iters": bootstrap_iters, + "gen_kwargs": gen_kwargs, + "random_seed": random_seed, + "numpy_seed": numpy_random_seed, + "torch_seed": torch_random_seed, + "fewshot_seed": fewshot_random_seed, + } + ) + results["git_hash"] = get_git_commit_hash() + results["date"] = start_date + add_env_info(results) # additional environment info to results + add_tokenizer_info(results, lm) # additional info about tokenizer + return results + else: + return None + + +@positional_deprecated +def evaluate( + lm: "LM", + task_dict, + limit: Optional[int] = None, + cache_requests: bool = False, + rewrite_requests_cache: bool = False, + bootstrap_iters: Optional[int] = 100000, + write_out: bool = False, + log_samples: bool = True, + system_instruction: Optional[str] = None, + apply_chat_template: Union[bool, str] = False, + fewshot_as_multiturn: bool = False, + verbosity: str = "INFO", + confirm_run_unsafe_code: bool = False, +): + """Instantiate and evaluate a model on a list of tasks. + + :param lm: obj + Language Model + :param task_dict: dict[str, Task] + Dictionary of tasks. Tasks will be taken to have name type(task).config.task . + :param limit: int, optional + Limit the number of examples per task (only use this for testing) + :param cache_requests: bool, optional + Speed up evaluation by caching the building of dataset requests. + :param rewrite_requests_cache: bool, optional + Rewrites all the request cache if set to `True`. + :param bootstrap_iters: + Number of iterations for bootstrap statistics, used when calculating stderr. Set to 0 for skipping all stderr calculations. + :param write_out: bool + If True, write out an example document and model input for checking task integrity + :param log_samples: bool + If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis + :param system_instruction: str + System instruction to be applied to the prompt + :param apply_chat_template: Union[bool, str] + Specifies whether to apply a chat template to the prompt. + - If set to True, the default chat template is applied. + - If set to a string, applies the specified chat template by name. + Defaults to False (no chat template applied). + :param fewshot_as_multiturn: bool + Whether to provide the fewshot examples as a multiturn conversation or a single user turn. + :param verbosity: str + Verbosity level for logging + :param confirm_run_unsafe_code: bool + Whether to confirm running tasks marked as unsafe. + :return + Dictionary of results + """ + + if apply_chat_template: + eval_logger.warning( + "Chat template formatting change affects loglikelihood and multiple-choice tasks. See docs/chat-template-readme.md for details." + ) + + # tracks all Instances/requests a model must generate output on. + requests = defaultdict(list) + # stores the amount to pad out reqs per req. type so that + # number of fwd passes per distributed rank is equal + padding_requests = defaultdict(int) + + # get lists of group hierarchy and each type of request + eval_tasks = get_task_list(task_dict) + if not log_samples: + if not all( + "bypass" not in getattr(task_output.task, "_metric_fn_list", {}).keys() + for task_output in eval_tasks + ): + raise ValueError("log_samples must be True for 'bypass' metric-only tasks") + + # validation checks: + # 1.are we running multimodal task <-> non-multimodal model class, or vice-versa. + # 2.are we running code that is marked as unsafe. + incompatible_tasks = [] + for task_output in eval_tasks: + task: Task = task_output.task + + if getattr(lm, "MULTIMODAL", False) != getattr(task, "MULTIMODAL", False): + incompatible_tasks.append(task_output.task_name) + elif getattr(task, "UNSAFE_CODE", False) and not confirm_run_unsafe_code: + raise ValueError( + f"Attempted to run task: {task_output.task_name} which is marked as unsafe. Set confirm_run_unsafe_code=True to run this task." + ) + if len(incompatible_tasks) > 0: + if not getattr(lm, "MULTIMODAL", False): + raise ValueError( + f"Attempted to run tasks: {incompatible_tasks} which require multimodal input, but the selected model type does not currently implement this. Multimodal support is currently restricted to the ['hf-multimodal', 'vllm-vlm'] model type." + ) + else: + raise ValueError( + f"Attempted to run tasks: {incompatible_tasks} which are text-only, but used a model type which only currently supports multimodal tasks." + ) + # end validation check + + # Cache the limit arg. + limit_arg = limit + limits = [] + for task_output in eval_tasks: + task: Task = task_output.task + + limit = get_sample_size(task, limit_arg) + limits.append(limit) + task.build_all_requests( + limit=limit, + rank=lm.rank, + world_size=lm.world_size, + cache_requests=cache_requests, + rewrite_requests_cache=rewrite_requests_cache, + system_instruction=system_instruction, + apply_chat_template=bool(apply_chat_template), + fewshot_as_multiturn=fewshot_as_multiturn, + chat_template=getattr(lm, "apply_chat_template") + if apply_chat_template + else None, + tokenizer_name=getattr(lm, "tokenizer_name", "") + if apply_chat_template + else "", + ) + eval_logger.debug( + f"Task: {task_output.task_name}; number of requests on this rank: {len(task.instances)}" + ) + if write_out: + print_writeout(task) + # aggregate Instances by LM method requested to get output. + for instance in task.instances: + reqtype = instance.request_type + requests[reqtype].append(instance) + + if lm.world_size > 1: + instances_rnk = torch.tensor(len(task._instances), device=lm.device) + gathered_item = ( + lm.accelerator.gather(instances_rnk).cpu().detach().numpy().tolist() + ) + # "multiple_choice" task types dispatch (several) "loglikelihood" request types + reqtype = ( + "loglikelihood" + if task.OUTPUT_TYPE == "multiple_choice" + else task.OUTPUT_TYPE + ) + # compute number of pseudo-batches to pad with (FSDP/DDP require even batches among ranks) + numpad = max(gathered_item) - gathered_item[lm.rank] + # todo: may not account for padding in cases like SquadV2 which has multiple req types + padding_requests[reqtype] += numpad + + ### Run LM on inputs, get all outputs ### + # execute each type of request + for reqtype, reqs in requests.items(): + eval_logger.info(f"Running {reqtype} requests") + # create `K` copies of each request `req` based off `K = req.repeats` + cloned_reqs = [] + for req in reqs: + cloned_reqs.extend([req] * req.repeats) + + if (lm.world_size > 1) and (padding_requests[reqtype] > 0): + for _ in range(padding_requests[reqtype]): + cloned_reqs.extend([req] * req.repeats) + + # run requests through model + resps = getattr(lm, reqtype)(cloned_reqs) + + # put responses from model into a list of length K for each request. + for x, req in zip(resps, cloned_reqs): + req.resps.append(x) + + if lm.world_size > 1: + lm.accelerator.wait_for_everyone() + + RANK = lm.rank + WORLD_SIZE = lm.world_size + ### Postprocess outputs ### + # TODO: del model here, maybe (idea: allow user to specify device of e.g. reward model separately) + for task_output, limit in zip(eval_tasks, limits): + task = task_output.task + task.apply_filters() + + ### Collect values of metrics on all datapoints ### + # # unpack results and sort back in order and return control to Task + # TODO: make it possible to use a different metric per filter + # Pre-process task.instances to group by doc_id + instances_by_doc_id = defaultdict(list) + for instance in task.instances: + instances_by_doc_id[instance.doc_id].append(instance) + # Sort instances within each group + for instances in instances_by_doc_id.values(): + instances.sort(key=lambda x: x.idx) + # iterate over different filters used + for filter_key in task.instances[0].filtered_resps.keys(): + doc_iterator = task.doc_iterator( + rank=RANK, limit=limit, world_size=WORLD_SIZE + ) + for doc_id, doc in doc_iterator: + requests = instances_by_doc_id[doc_id] + metrics = task.process_results( + doc, [req.filtered_resps[filter_key] for req in requests] + ) + if log_samples: + target = task.doc_to_target(doc) + example = { + "doc_id": doc_id, + "doc": doc, + "target": target, + "arguments": [req.args for req in requests], + "resps": [req.resps for req in requests], + "filtered_resps": [ + req.filtered_resps[filter_key] for req in requests + ], + "filter": filter_key, + "metrics": list(metrics.keys()), + "doc_hash": hash_string( + json.dumps( + requests[0].doc, + indent=2, + default=handle_non_serializable, + ensure_ascii=False, + ) + ), + "prompt_hash": hash_string(requests[0].arguments[0]), + "target_hash": hash_string(str(target)), + } + example.update(metrics) + task_output.logged_samples.append(example) + for metric, value in metrics.items(): + task_output.sample_metrics[(metric, filter_key)].append(value) + + if WORLD_SIZE > 1: + # if multigpu, then gather data across all ranks to rank 0 + # first gather logged samples across all ranks + for task_output in eval_tasks: + if log_samples: + # for task_name, task_samples in list(samples.items()): + full_samples = [None] * WORLD_SIZE if RANK == 0 else None + torch.distributed.gather_object( + obj=task_output.logged_samples, + object_gather_list=full_samples, + dst=0, + ) + + if RANK == 0: + task_output.logged_samples = list( + itertools.chain.from_iterable(full_samples) + ) + + # then collect metrics across all ranks + for metrics in task_output.sample_metrics: + metric_list = [None] * WORLD_SIZE if RANK == 0 else None + torch.distributed.gather_object( + obj=task_output.sample_metrics[metrics], + object_gather_list=metric_list, + dst=0, + ) + if RANK == 0: + task_output.sample_metrics[metrics] = list( + itertools.chain.from_iterable(metric_list) + ) + + if RANK == 0: + ### Aggregate results over all datapoints ### + # aggregate results ; run bootstrap CIs + for task_output in eval_tasks: + task_output.calculate_aggregate_metric(bootstrap_iters=bootstrap_iters) + ( + results, + samples, + configs, + versions, + num_fewshot, + higher_is_better, + ) = consolidate_results(eval_tasks) + + ### Calculate group metrics ### + if bool(results): + results, versions, show_group_table, *_ = consolidate_group_results( + results, versions, task_dict + ) + + results_agg, group_agg = prepare_print_tasks(task_dict, results) + subtask_list = get_subtask_list(task_dict) + + # collect all higher_is_better values for metrics + # in the group's subtasks. + # TODO: clean this up ; unify with the below metric_list loop? + _higher_is_better = {} + for group, task_list in subtask_list.items(): + if ( + len(task_list) != 0 + ): # subtask list will list "task_name": [] for solo tasks + for task in task_list: + for m, h in higher_is_better[task].items(): + if m not in _higher_is_better.keys(): + _higher_is_better[m] = h + + if ( + m in _higher_is_better + and _higher_is_better[m] is not None + and _higher_is_better[m] != h + ): + eval_logger.warning( + f"Higher_is_better values for metric {m} in group {group} are not consistent. Defaulting to None." + ) + _higher_is_better[m] = None + higher_is_better[group] = _higher_is_better + + results_dict = { + "results": dict(results_agg.items()), + **( + {"groups": dict(group_agg.items())} + if (bool(group_agg) & show_group_table) + else {} + ), + "group_subtasks": dict(reversed(subtask_list.items())), + "configs": dict(sorted(configs.items())), + "versions": dict(sorted(versions.items())), + "n-shot": dict(sorted(num_fewshot.items())), + "higher_is_better": dict(sorted(higher_is_better.items())), + "n-samples": { + task_output.task_name: { + "original": len(task_output.task.eval_docs), + "effective": min( + limit if limit else len(task_output.task.eval_docs), + len(task_output.task.eval_docs), + ), + } + for task_output, limit in zip(eval_tasks, limits) + }, + } + if log_samples: + results_dict["samples"] = dict(samples) + + return results_dict + + else: + return None + + +def request_caching_arg_to_dict(cache_requests: str) -> dict: + request_caching_args = { + "cache_requests": cache_requests in {"true", "refresh"}, + "rewrite_requests_cache": cache_requests == "refresh", + "delete_requests_cache": cache_requests == "delete", + } + + return request_caching_args diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/evaluator_utils.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/evaluator_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0bd87b6c7a3923d7e2cdf71894ca87b10137ac9c --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/evaluator_utils.py @@ -0,0 +1,554 @@ +import collections +import logging +import math +import pathlib +import sys +from typing import List, Optional, Tuple, Union + +from lm_eval.api.group import ConfigurableGroup +from lm_eval.api.metrics import ( + aggregate_subtask_metrics, + mean, + pooled_sample_stderr, + stderr_for_metric, +) +from lm_eval.api.task import Task +from lm_eval.utils import positional_deprecated + + +eval_logger = logging.getLogger(__name__) + + +class TaskOutput: + """ + Wrapper class for Task outputs.It contains various attributes and methods to manage and calculate metrics for the task. + + Attributes: + task (object): The task object. + task_name (str): The name of the task. + task_config (dict): The configuration of the task. + version (str): The version of the task. + group_name (str): The name of the task group. + n_shot (int): The number of shots for the task. + task_alias (str): The alias of the task. + group_alias (str): The alias of the task group. + is_group (bool): Indicates if the task is a group. + logged_samples (list): The list of logged samples. + sample_len (int): The length of the samples. + sample_metrics (defaultdict): The dictionary of samples' metrics. + agg_metrics (defaultdict): The dictionary of aggregate metrics. + + Methods: + from_taskdict(cls, task_name: str, task): + Creates a TaskOutput instance from a task dictionary. + + calculate_aggregate_metric(bootstrap_iters=100000) -> None: + Calculates the aggregate metrics for the task. + """ + + def __init__( + self, + task=None, + task_name=None, + task_config=None, + version=None, + group_name=None, + n_shot=None, + task_alias=None, + group_alias=None, + is_group=None, + ): + self.task = task + self.task_config = task_config + self.task_name = task_name + self.group_name = group_name + self.version = version + self.n_shot = n_shot + self.task_alias = task_alias + self.group_alias = group_alias + self.is_group = is_group + self.logged_samples = [] + self.sample_len = None + self.sample_metrics = collections.defaultdict(list) + self.agg_metrics = collections.defaultdict(list) + + @classmethod + def from_taskdict(cls, task_name: str, task): + if isinstance(task, tuple): + group_name, task = task + else: + group_name = None + if not task: + # these gets filtered out in get_task_list + # once they are added to group hierarchy + is_group = True + return cls( + task=task, task_name=task_name, is_group=is_group, group_name=group_name + ) + version = task.VERSION + task_config = dict(task.dump_config()) + if (n_shot := task_config.get("num_fewshot")) == 0: + n_shot = task_config.get("metadata", {}).get("num_fewshot", 0) + task_alias = task_config.get("alias") + group_alias = task_config.get("group_alias") + return cls( + task=task, + task_name=task_name, + task_config=task_config, + group_name=group_name, + version=version, + n_shot=n_shot, + task_alias=task_alias, + group_alias=group_alias, + ) + + def calculate_aggregate_metric(self, bootstrap_iters=100000) -> None: + for (metric, filter_key), items in self.sample_metrics.items(): + try: + agg_fn = self.task.aggregation()[metric] + except KeyError: + # This is when process results output an arbitrary metric + # TODO: Handle this better and allow other aggregate functions other than mean. + agg_fn = mean + metric_key = f"{metric},{filter_key}" + self.agg_metrics[metric_key] = agg_fn(items) + self.sample_len = len(items) # TODO: same sample size for each metric? + if isinstance(bootstrap_iters, int): + stderr_fn = stderr_for_metric( + metric=agg_fn, + bootstrap_iters=min(bootstrap_iters, 100) + if metric in ["bleu", "chrf", "ter"] + else bootstrap_iters, + ) + self.agg_metrics[f"{metric}_stderr,{filter_key}"] = ( + stderr_fn(items) if (stderr_fn and len(items) > 1) else "N/A" + ) + else: + raise ValueError( + f"Received bootstrap_iters '{bootstrap_iters}' but expected an integer. Set to 0 to turn off stderr calculations." + ) + + def __repr__(self): + return ( + f"TaskOutput(task_name={self.task_name}, " + f"group_name={self.group_name}, " + f"version={self.version}, " + f"n_shot={self.n_shot}, " + f"task_alias={self.task_alias}, " + f"group_alias={self.group_alias})" + ) + + +def get_task_list(task_dict: dict) -> List[TaskOutput]: + outputs = [] + for task_name, task_obj in task_dict.items(): + if isinstance(task_obj, dict): + _outputs = get_task_list(task_obj) + outputs.extend(_outputs) + else: + task_output = TaskOutput.from_taskdict(task_name, task_obj) + outputs.append(task_output) + + return outputs + + +def get_subtask_list(task_dict, task_root=None, depth=0): + subtask_list = {} + for group_obj, task_obj in task_dict.items(): + if isinstance(group_obj, ConfigurableGroup): + # group_name = group_obj.group_name + group_name = group_obj.group_name + else: + group_name = group_obj + if isinstance(task_obj, dict): + _subtask_list = get_subtask_list( + task_obj, task_root=group_name, depth=depth + 1 + ) + if task_root: + subtask_list.setdefault((task_root, depth), []).extend( + [ + _task + for (_task, _depth) in _subtask_list.keys() + if (_depth - 1) == depth + ] + ) + + subtask_list = {**subtask_list, **_subtask_list} + else: + if isinstance(task_obj, ConfigurableGroup): + # group_or_task_name = task_obj.group_name + group_or_task_name = task_obj.group_name + elif isinstance(task_obj, Task): + # group_or_task_name = task_obj.task_name + group_or_task_name = task_obj.task_name + + if task_root is None: + subtask_list.setdefault((group_or_task_name, depth), []) + else: + subtask_list.setdefault((task_root, depth), []).append( + group_or_task_name + ) + + if depth == 0: + _subtask_list = {} + for group_key, task_list in subtask_list.items(): + group_name, depth = group_key + _subtask_list[group_name] = task_list + subtask_list = _subtask_list + + return subtask_list + + +def print_writeout(task) -> None: + for inst in task.instances: + # print the prompt for the first few documents + if inst.doc_id < 1: + eval_logger.info( + f"Task: {task}; document {inst.doc_id}; context prompt (starting on next line):\ + \n{inst.args[0]}\n(end of prompt on previous line)\ntarget string or answer choice index (starting on next line):\n{task.doc_to_target(inst.doc)}\n(end of target on previous line)" + ) + eval_logger.info(f"Request: {str(inst)}") + + +def get_sample_size(task, limit: Optional[int]) -> Union[int, None]: + if limit is not None: + limit = ( + int(math.ceil(len(task.eval_docs) * limit)) if limit < 1.0 else int(limit) + ) + return limit + + +def prepare_print_tasks( + task_dict: dict, + results: dict, + task_depth=0, + group_depth=0, +) -> Tuple[dict, dict]: + """ + @param task_dict: Dictionary representing the group hierarchy of tasks. Each key is a group name and its + value is a list of task names. + @param results: Dictionary containing the results of each task. Each key is a + group name and its value is a dictionary of task results. + @param task_depth: The indentation level for printing the task + hierarchy. Default is 0. + @param group_depth: The indentation level for printing the group + hierarchy. Default is 0. + @return: A tuple of two dictionaries: results_agg and groups_agg. results_agg contains + aggregated results for each task, and groups_agg contains aggregated results for each group. + + Prepares the task hierarchy and aggregates the results for each task and group recursively for printing. + """ + + def _sort_task_dict(task_dict): + """ + Helper utility. Sorts the task dict at the current level of the hierarchy based on alphabetized task name. + Required so that we end up sorting within each sub-header correctly. + """ + + return dict( + sorted( + task_dict.items(), + key=lambda item: item[0].group_name + if isinstance(item[0], ConfigurableGroup) + else item[0], + ) + ) + + task_agg = collections.defaultdict(dict) + group_agg = collections.defaultdict(dict) + task_dict = _sort_task_dict(task_dict) + for task_or_group_name, task_or_group_obj in task_dict.items(): + tab_string = " " * task_depth + "- " if task_depth > 0 else "" + if isinstance(task_or_group_name, ConfigurableGroup): + # string_name = task_or_group_name.group_name + name = task_or_group_name.group_name + from_configurable_group = True + task_or_group_obj = _sort_task_dict(task_or_group_obj) + elif isinstance(task_or_group_name, str): + name = task_or_group_name + if isinstance(task_or_group_obj, Task): + # string_name = task_or_group_obj.task_name + name = task_or_group_obj.task_name + from_configurable_group = False + + task_agg[name] = results[name].copy() + if from_configurable_group: + if task_or_group_name.group_alias is not None: + alias = task_or_group_name.group_alias + else: + alias = task_or_group_name.group + else: + if "alias" in task_agg[name]: + alias = task_agg[name]["alias"] + else: + alias = name + + task_agg[name]["alias"] = tab_string + alias + if "samples" in task_agg[name]: + task_agg[name].pop("samples") + + if from_configurable_group and (" " not in results[name]): + group_tab_string = " " * group_depth + "- " if group_depth > 0 else "" + group_agg[name] = results[name].copy() + group_agg[name]["alias"] = group_tab_string + alias + if "samples" in group_agg[name]: + group_agg[name].pop("samples") + + if isinstance(task_or_group_obj, dict): + task_depth += 1 + group_depth += 1 + _task_agg, _group_agg = prepare_print_tasks( + task_or_group_obj, results, task_depth, group_depth + ) + task_agg = { + **task_agg, + **_task_agg, + } + group_agg = {**group_agg, **_group_agg} + task_depth -= 1 + group_depth -= 1 + return task_agg, group_agg + + +def consolidate_results( + eval_tasks: List[TaskOutput], +) -> Tuple[dict, dict, dict, dict, dict, dict]: + """ + @param eval_tasks: list(TaskOutput). + @return: A tuple containing the consolidated results, samples, configs, versions, and num_fewshot. + + Consolidates the results of multiple evaluation tasks into a single structure. + + The method iterates over each evaluation instance and extracts relevant information to create the consolidated + results structure. The consolidated results structure has the following properties: + + - results: A defaultdict with task names as keys and dictionaries as values. Each dictionary contains + metric/filter pairs as keys and corresponding metric values as values. The "alias" key is used to store task + aliases specified in the task configuration. + - samples: A defaultdict with task names as keys and lists of log samples as values. + - configs: A defaultdict with task names as keys and task configurations as values. + - versions: A defaultdict with task names as keys and task versions as values. + - num_fewshot: A defaultdict with task names as keys and number of few-shot samples as values. + - higher_is_better: A defaultdict with task names as keys and indicators of whether higher values are better + for each metric as values. + + The method then returns the consolidated results, samples, configs, versions, and num_fewshot as a tuple. + """ + # stores the final result for each task, for each metric/filter pair. + results = collections.defaultdict(dict) + # logs info about each document evaluated. + samples = collections.defaultdict(list) + # store num-fewshot value per task + num_fewshot = collections.defaultdict(int) + # Tracks the YAML configs of all chosen task + configs = collections.defaultdict(dict) + # Tracks each task's version. + versions = collections.defaultdict(dict) + # Track `higher_is_better` for each metric + higher_is_better = collections.defaultdict(dict) + + for task_output in eval_tasks: + if "task_alias" in (task_config := task_output.task_config): + results[task_output.task_name]["alias"] = task_config["task_alias"] + else: + results[task_output.task_name]["alias"] = task_output.task_name + if group_alias := task_output.group_alias: + if group_alias not in results and (group_name := task_output.group_name): + results[group_name]["alias"] = group_alias + num_fewshot[task_output.task_name] = task_output.n_shot + configs[task_output.task_name] = task_output.task_config + versions[task_output.task_name] = task_output.version + samples[task_output.task_name] = task_output.logged_samples + higher_is_better[task_output.task_name] = task_output.task.higher_is_better() + for (metric, filter_key), items in task_output.sample_metrics.items(): + metric_key = f"{metric},{filter_key}" + results[task_output.task_name][metric_key] = task_output.agg_metrics[ + metric_key + ] + results[task_output.task_name]["samples"] = task_output.sample_len + results[task_output.task_name][f"{metric}_stderr,{filter_key}"] = ( + task_output.agg_metrics[f"{metric}_stderr,{filter_key}"] + ) + return results, samples, configs, versions, num_fewshot, higher_is_better + + +def consolidate_group_results( + results, + versions, + task_dict, + task_root=None, + show_group_table=False, + task_aggregation_list=None, +) -> Tuple[dict, dict, bool, Union[None,]]: + """ + (Recursively) calculates groups' aggregated metrics and updates the results and versions dictionaries with this info. + + @return: a tuple [results, versions, show_group_table, task_aggregation_list] with formats described below: + + - results: A defaultdict with task names (and, after this function is called, group names of + groups that perform aggregation) as keys, and dictionaries with "alias" and metric,filter_name pairs as keys. + - versions: A defaultdict with task names (and, after this function is called, group names of + groups that perform aggregation) as keys, and float values representing the task or group's version if a version is specified. (defaulting to None). + - show_group_table: a boolean which is true if there exists a group that requires printing of its aggregated scores in a group table. + - task_aggregation_list: a defaultdict listing the subtasks to average over to produce a given group's end metric. + + The method then returns the updated results, versions, show_group_table, and task_aggregation_list as a tuple. + In the top-level invocation of this function, task_aggregation_list is ignored. + """ + if task_root is None: + task_root = {} + + if task_aggregation_list is None: + task_aggregation_list = {} + + for group_or_task, group_or_task_info in task_dict.items(): + # Convert to string + if isinstance(group_or_task, ConfigurableGroup): + group_config = group_or_task.config + group_or_task = group_or_task.group_name + else: + group_config = None + + if isinstance(group_or_task_info, Task): + if task_root: + task_aggregation_list.setdefault(task_root, []).append( + group_or_task_info.task_name + ) + else: + ( + results, + versions, + show_group_table, + _task_aggregation_list, + ) = consolidate_group_results( + results, + versions, + group_or_task_info, + group_or_task, + show_group_table, + task_aggregation_list, + ) + if task_root: + task_aggregation_list.setdefault(task_root, []).extend( + task_aggregation_list.get(group_or_task, []) + ) + + if (group_config is None) or ( + group_config["aggregate_metric_list"] is None + ): + results[group_or_task][" "] = " " + continue + + if "aggregate_metric_list" in group_config: + agg_metric_list = group_config["aggregate_metric_list"] + + show_group_table = show_group_table | bool( + group_config["aggregate_metric_list"] + ) + + task_list = _task_aggregation_list[group_or_task] + + metric_list = list( + { + key + for task in task_list + for key in results[task].keys() + if "_stderr" not in key and key not in ["task", "alias", "samples"] + } + ) + for metric in metric_list: + stderr = "_stderr,".join(metric.split(",")) + + # gather metrics, sizes, and stderrs from subtasks + metrics = [ + results[task][metric] + for task in task_list + if metric in results[task] + ] # TODO: copy? + stderrs = [ + results[task][stderr] + for task in task_list + if stderr in results[task] + ] + sizes = [ + results[task]["samples"] + for task in task_list + if metric in results[task] + ] + + for metric_config in agg_metric_list: + for filter_name in metric_config["filter_list"]: + if metric != ",".join([metric_config["metric"], filter_name]): + continue + + # compute group's pooled metric and stderr + if metric_config["aggregation"] == "mean": + aggregate_fn = aggregate_subtask_metrics + elif callable(metric_config["aggregation"]): + aggregate_fn = metric_config["aggregation"] + else: + raise ValueError( + f"Currently, only 'mean' is supported for automatically aggregating scores across groups' subtasks. Got '{metric_config['aggregation']}' for group '{group_or_task}'" + ) + + results[group_or_task][metric] = aggregate_fn( + metrics, + sizes, + metric_config["weight_by_size"], + ) + # TODO: calculate groups' metrics using arbitrary agg fns + if "N/A" in stderrs: + results[group_or_task][stderr] = "N/A" + else: + # NOTE: this assumes we are using the mean to aggregate. There are warnings about this elsewhere + results[group_or_task][stderr] = pooled_sample_stderr( + stderrs, sizes + ) + + results[group_or_task]["samples"] = sum(sizes) + group_metadata = group_config.get("metadata", None) + if group_metadata is not None: + versions[group_or_task] = group_metadata.get("version", None) + # print(results) + return results, versions, show_group_table, task_aggregation_list + + +@positional_deprecated +def find_test_root(start_path: pathlib.Path) -> pathlib.Path: + """ + Search upward in the directory tree to a maximum of three layers + to find and return the package root (containing the 'tests' folder) + """ + cur_path = start_path.resolve() + max_layers = 3 + for _ in range(max_layers): + if (cur_path / "tests" / "test_version_stable.py").exists(): + return cur_path + else: + cur_path = cur_path.parent.resolve() + raise FileNotFoundError( + f"Unable to find package root within {max_layers} upwards" + f"of {start_path}" + ) + + +@positional_deprecated +def run_task_tests(task_list: List[str]): + """ + Find the package root and run the tests for the given tasks + """ + import pytest + + package_root = find_test_root(start_path=pathlib.Path(__file__)) + task_string = " or ".join(task_list) + args = [ + f"{package_root}/tests/test_version_stable.py", + f"--rootdir={package_root}", + "-k", + f"{task_string}", + ] + sys.path.append(str(package_root)) + pytest_return_val = pytest.main(args) + if pytest_return_val: + raise ValueError( + f"Not all tests for the specified tasks ({task_list}) ran successfully! Error code: {pytest_return_val}" + ) diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/filters/__init__.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/filters/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..be5c9d43624ea901cc578c65689be5bd263209a5 --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/filters/__init__.py @@ -0,0 +1,25 @@ +from functools import partial +from typing import List + +from lm_eval.api.filter import FilterEnsemble +from lm_eval.api.registry import get_filter + +from . import custom, extraction, selection, transformation + + +def build_filter_ensemble( + filter_name: str, components: List[List[str]] +) -> FilterEnsemble: + """ + Create a filtering pipeline. + """ + filters = [] + for function, kwargs in components: + if kwargs is None: + kwargs = {} + # create a filter given its name in the registry + f = partial(get_filter(function), **kwargs) + # add the filter as a pipeline step + filters.append(f) + + return FilterEnsemble(name=filter_name, filters=filters) diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/filters/custom.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/filters/custom.py new file mode 100644 index 0000000000000000000000000000000000000000..ab22c51eda74670aaea6699fc68992994c41932d --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/filters/custom.py @@ -0,0 +1,17 @@ +from lm_eval.api.filter import Filter +from lm_eval.api.registry import register_filter + + +@register_filter("custom") +class CustomFilter(Filter): + """ + Custom filter that applies a custom, user-defined function to the model responses. + """ + + def __init__(self, **kwargs) -> None: + self.filter_fn = kwargs.pop("filter_fn") + + super().__init__(**kwargs) + + def apply(self, resps, docs): + return self.filter_fn(resps, docs) diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/filters/decontamination.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/filters/decontamination.py new file mode 100644 index 0000000000000000000000000000000000000000..4eda4e022445355f191926790b2edf8f0cfa4bbd --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/filters/decontamination.py @@ -0,0 +1,25 @@ +from lm_eval.api.filter import Filter +from lm_eval.api.registry import register_filter + + +@register_filter("decontaminate") +class DecontaminationFilter(Filter): + """ + A filter which evaluates + """ + + name = "track_decontamination" + + def __init__(self, path) -> None: + """ + + TODO: make sure only ever run one time on the train set (should this be cached as a class var? keyed by value for "path"). + should further cache result on a given (task_name, doc_id) + """ + self._decontam_results = None + + def apply(self, resps, docs) -> None: + """ + Return {"no_contamination", "only_contamination"} keys for the 2 different subsets + """ + pass diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/filters/extraction.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/filters/extraction.py new file mode 100644 index 0000000000000000000000000000000000000000..9c8d796b6099d89fb6b6e5b2e17444cfa66f1b06 --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/filters/extraction.py @@ -0,0 +1,188 @@ +import re +import sys +import unicodedata + +from lm_eval.api.filter import Filter +from lm_eval.api.registry import register_filter + + +@register_filter("regex") +class RegexFilter(Filter): + """A filter that extracts values from text using regex pattern matching. + + This filter applies a regex pattern to each model response and extracts matched values. + If no match is found, returns a fallback value. Useful for extracting structured data + (like numbers) from unstructured model outputs. + """ + + def __init__( + self, + regex_pattern: str = r"#### (\-?[0-9\.\,]+)", + group_select: int = 0, + fallback: str = "[invalid]", + ) -> None: + """ + pass a string `regex` to run `re.compile(r"regex")` on. + `fallback` defines the output returned if no matches for the regex are located. + """ + self.regex_pattern = regex_pattern + self.regex = re.compile(regex_pattern) + self.group_select = group_select + self.fallback = fallback + + def apply(self, resps: list[list[str]], docs: list[dict]) -> list[list[str]]: + # here, we assume we have a list, in which each element is + # a list of model responses for some particular input/target pair. + # so we process each of these (same input/target response sets) + # independently (and keep them a list.) + def filter_set(inst): + filtered = [] + for resp in inst: + match = self.regex.findall(resp) + if match: + match = match[self.group_select] + if isinstance(match, tuple): + match = [m for m in match if m] + if match: + match = match[0] + else: + match = self.fallback + match = match.strip() + else: + match = self.fallback + filtered.append(match) + return filtered + + filtered_resps = list(map(lambda x: filter_set(x), resps)) + + return filtered_resps + + +@register_filter("remove_whitespace") +class WhitespaceFilter(Filter): + """Filters out leading whitespace from responses.""" + + def apply(self, resps: list[list[str]], docs: list[dict]) -> list[list[str]]: + def filter_set(inst): + filtered_resp = [] + for resp in inst: + resp = resp.lstrip() + filtered_resp.append(resp) + return filtered_resp + + filtered_resps = [filter_set(resp) for resp in resps] + + return filtered_resps + + +@register_filter("multi_choice_regex") +class MultiChoiceRegexFilter(RegexFilter): + """ + A filter used to extract a model's answer on multiple choice questions with + letter answers. assumes each document has a "choices" field + containing the list of answer choices and that the answer label symbols + are of the form (A), (B), (C), ... or A, B, C. + """ + + def __init__( + self, + regex_pattern: str = r"#### (\-?[0-9\.\,]+)", + group_select=0, + fallback: str = "[invalid]", + ignore_case=False, + ignore_punctuation=False, + regexes_to_ignore=None, + ) -> None: + """ + regex_pattern: The basic regex pattern to use. If fails to match, we will use the customized match procedure + - step 1 : We parse the choices between ([A-Z])s then try to find these choices in the response. + - step 2 : We parse the choice with regex :[\s]*([A-?]), where ? varies by number of choices. + group_select: Selects the (group_select)th match from the findall result. + ignore_case: Ignores the case during step 1 matching + ignore_punctuation: Remove the punctuation during step 1 matching + regexes_to_ignore: Remove these regexes during step 1 matching + """ + super().__init__(regex_pattern, group_select, fallback) + self.ignore_case = ignore_case + self.ignore_punctuation = ignore_punctuation + self.regexes_to_ignore = regexes_to_ignore + + def apply(self, resps: list[list[str]], docs: list[dict]) -> list[list[str]]: + # here, we assume we have a list, in which each element is + # a list of model responses for some particular input/target pair. + # so we process each of these (same input/target response sets) + # independently (and keep them a list.) + + def find_match(regex, resp, convert_dict={}): + match = regex.findall(resp) + if match: + match = match[self.group_select] + if isinstance(match, tuple): + match = [m for m in match if m][0] + match = match.strip() + if match and match in convert_dict: + match = convert_dict[match] + return match + + punct_tbl = dict.fromkeys( + i + for i in range(sys.maxunicode) + if unicodedata.category(chr(i)).startswith("P") + ) + + def filter_ignores(st): + if self.regexes_to_ignore is not None: + for s in self.regexes_to_ignore: + st = re.sub(s, "", st) + + if self.ignore_case: + st = st.lower() + + if self.ignore_punctuation: + # https://stackoverflow.com/a/266162 + st = st.translate(punct_tbl) + return st + + filtered_resps = [] + + for r, doc in zip(resps, docs): + fallback_regexes = [] + choice_to_alpha = {} + next_alpha = "A" + + without_paren_fallback_regexes = [] + without_paren_to_target = {} + + choices = doc["choices"] + for c in choices: + m = filter_ignores(c.strip()) + fallback_regexes.append(f"{re.escape(m)}") + choice_to_alpha[m] = f"({next_alpha})" + + without_paren_fallback_regexes.append(next_alpha) + without_paren_to_target[next_alpha] = f"({next_alpha})" + + next_alpha = chr(ord(next_alpha) + 1) + fallback_regex = re.compile("|".join(fallback_regexes)) + without_paren_fallback_regex = "|".join(without_paren_fallback_regexes) + without_paren_fallback_regex = re.compile( + rf":[\s]*({without_paren_fallback_regex})" + ) + + filtered = [] + for resp in r: + match = find_match(self.regex, resp) + if not match: + match = find_match( + fallback_regex, filter_ignores(resp), choice_to_alpha + ) + if not match: + match = find_match( + without_paren_fallback_regex, resp, without_paren_to_target + ) + if not match: + match = self.fallback + filtered.append(match) + filtered_resps.append(filtered) + + return filtered_resps diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/filters/selection.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/filters/selection.py new file mode 100644 index 0000000000000000000000000000000000000000..8c670ed74d00655441cc45181fba1265f0db5290 --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/filters/selection.py @@ -0,0 +1,61 @@ +from collections import Counter + +from lm_eval.api.filter import Filter +from lm_eval.api.registry import register_filter + + +# TODO: implement "arg_max" filter. either it should take in an arbitrary "scoring"/reward function +# that takes an input and returns a scalar and then should select the max reward, +# or should implement different filters for different ways of handling a reward model's inference. + + +@register_filter("take_first") +class TakeFirstFilter(Filter): + def __init__(self) -> None: + """ + Can define custom behavior here, if an individual instantiation of a Filter class should have state. + """ + + def apply(self, resps, docs): + """ + Assuming each entry of `resps` is a list of model responses, we discard all but the first response. + """ + return map(lambda r: r[0], resps) + + +@register_filter("take_first_k") +class TakeKFilter(Filter): + def __init__(self, **kwargs) -> None: + self.k = kwargs.pop("k") + + super().__init__(**kwargs) + + def apply(self, resps, docs): + # need resp to be subscriptable to check below + resps = list(resps) + # check we have at least k responses per doc, else we can't take the first k + assert len(resps[0]) >= self.k, ( + f"Need at least {self.k} responses per doc to take first {self.k}, but got {len(resps[0])} only! Please increase TaskConfig.repeats ." + ) + return map(lambda r: r[: self.k], resps) + + +@register_filter("majority_vote") +class MajorityVoteFilter(Filter): + def __init__(self) -> None: + """ + Can define custom behavior here, if an individual instantiation of a Filter class should have state. + """ + + def apply(self, resps, docs): + """ + Each entry of `resps` is a list of model responses. + We select the response that occurs most frequently in each entry of `resps`. + """ + + def select_majority(resp): + counts = Counter(resp) + vote = counts.most_common(1)[0][0] + return vote + + return map(lambda r: [select_majority(r)], resps) diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/filters/transformation.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/filters/transformation.py new file mode 100644 index 0000000000000000000000000000000000000000..1a3592b6dd4811dcef39ff090dfa42e926613b5c --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/filters/transformation.py @@ -0,0 +1,56 @@ +from lm_eval.api.filter import Filter +from lm_eval.api.registry import register_filter + + +@register_filter("lowercase") +class LowercaseFilter(Filter): + def __init__(self) -> None: + pass + + def apply(self, resps, docs): + def filter_set(inst): + return [resp.lower() for resp in inst] + + return [filter_set(resp) for resp in resps] + + +@register_filter("uppercase") +class UppercaseFilter(Filter): + def __init__(self) -> None: + pass + + def apply(self, resps, docs): + def filter_set(inst): + return [resp.upper() for resp in inst] + + return [filter_set(resp) for resp in resps] + + +@register_filter("map") +class MapFilter(Filter): + def __init__(self, mapping_dict: dict = None, default_value=None) -> None: + """ + Initializes the MapFilter with a given mapping dictionary and default value. + + Args: + - mapping_dict (dict): A dictionary containing the key-value mappings. + Default is an empty dictionary. + - default_value (Any): The value to be returned when a key is not found in the mapping_dict. + Default is None. + + Example: + mapper = MapFilter({'A': 1, 'B': 2}, default_value=0) + """ + if mapping_dict is None: + mapping_dict = {} + assert isinstance(mapping_dict, dict), ( + "Provided mapping_dict is not a dictionary" + ) + self.mapping_dict = mapping_dict + self.default_value = default_value + + def apply(self, resps, docs): + def filter_set(inst): + return [self.mapping_dict.get(resp, self.default_value) for resp in inst] + + return [filter_set(resp) for resp in resps] diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/loggers/__init__.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/loggers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..02b7a6834c6486fde35ef02d715e90be3fba223a --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/loggers/__init__.py @@ -0,0 +1,2 @@ +from .evaluation_tracker import EvaluationTracker +from .wandb_logger import WandbLogger diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/loggers/evaluation_tracker.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/loggers/evaluation_tracker.py new file mode 100644 index 0000000000000000000000000000000000000000..ef56965d2ad19162811767f50cb4372abe0096b3 --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/loggers/evaluation_tracker.py @@ -0,0 +1,524 @@ +import json +import logging +import os +import re +import time +from collections import defaultdict +from dataclasses import asdict, dataclass +from datetime import datetime +from pathlib import Path + +from datasets import load_dataset +from datasets.utils.metadata import MetadataConfigs +from huggingface_hub import ( + DatasetCard, + DatasetCardData, + HfApi, + hf_hub_url, +) +from huggingface_hub.utils import build_hf_headers, get_session, hf_raise_for_status + +from lm_eval.utils import ( + get_file_datetime, + get_file_task_name, + get_results_filenames, + get_sample_results_filenames, + handle_non_serializable, + hash_string, + sanitize_list, + sanitize_model_name, + sanitize_task_name, +) + + +eval_logger = logging.getLogger(__name__) + + +@dataclass(init=False) +class GeneralConfigTracker: + """ + Tracker for the evaluation parameters. + + Attributes: + model_source (str): Source of the model (e.g. Hugging Face, GGUF, etc.) + model_name (str): Name of the model. + model_name_sanitized (str): Sanitized model name for directory creation. + start_time (float): Start time of the experiment. Logged at class init. + end_time (float): Start time of the experiment. Logged when calling [`GeneralConfigTracker.log_end_time`] + total_evaluation_time_seconds (str): Inferred total evaluation time in seconds (from the start and end times). + """ + + model_source: str = None + model_name: str = None + model_name_sanitized: str = None + system_instruction: str = None + system_instruction_sha: str = None + fewshot_as_multiturn: bool = None + chat_template: str = None + chat_template_sha: str = None + start_time: float = None + end_time: float = None + total_evaluation_time_seconds: str = None + + def __init__(self) -> None: + """Starts the evaluation timer.""" + self.start_time = time.perf_counter() + + @staticmethod + def _get_model_name(model_args: str) -> str: + """Extracts the model name from the model arguments.""" + + def extract_model_name(model_args: str, key: str) -> str: + """Extracts the model name from the model arguments using a key.""" + args_after_key = model_args.split(key)[1] + return args_after_key.split(",")[0] + + # order does matter, e.g. peft and delta are provided together with pretrained + prefixes = ["peft=", "delta=", "pretrained=", "model=", "path=", "engine="] + for prefix in prefixes: + if prefix in model_args: + return extract_model_name(model_args, prefix) + return "" + + def log_experiment_args( + self, + model_source: str, + model_args: str, + system_instruction: str, + chat_template: str, + fewshot_as_multiturn: bool, + ) -> None: + """Logs model parameters and job ID.""" + self.model_source = model_source + self.model_name = GeneralConfigTracker._get_model_name(model_args) + self.model_name_sanitized = sanitize_model_name(self.model_name) + self.system_instruction = system_instruction + self.system_instruction_sha = ( + hash_string(system_instruction) if system_instruction else None + ) + self.chat_template = chat_template + self.chat_template_sha = hash_string(chat_template) if chat_template else None + self.fewshot_as_multiturn = fewshot_as_multiturn + + def log_end_time(self) -> None: + """Logs the end time of the evaluation and calculates the total evaluation time.""" + self.end_time = time.perf_counter() + self.total_evaluation_time_seconds = str(self.end_time - self.start_time) + + +class EvaluationTracker: + """ + Keeps track and saves relevant information of the evaluation process. + Compiles the data from trackers and writes it to files, which can be published to the Hugging Face hub if requested. + """ + + def __init__( + self, + output_path: str = None, + hub_results_org: str = "", + hub_repo_name: str = "", + details_repo_name: str = "", + results_repo_name: str = "", + push_results_to_hub: bool = False, + push_samples_to_hub: bool = False, + public_repo: bool = False, + token: str = "", + leaderboard_url: str = "", + point_of_contact: str = "", + gated: bool = False, + ) -> None: + """ + Creates all the necessary loggers for evaluation tracking. + + Args: + output_path (str): Path to save the results. If not provided, the results won't be saved. + hub_results_org (str): The Hugging Face organization to push the results to. If not provided, the results will be pushed to the owner of the Hugging Face token. + hub_repo_name (str): The name of the Hugging Face repository to push the results to. If not provided, the results will be pushed to `lm-eval-results`. + details_repo_name (str): The name of the Hugging Face repository to push the details to. If not provided, the results will be pushed to `lm-eval-results`. + result_repo_name (str): The name of the Hugging Face repository to push the results to. If not provided, the results will not be pushed and will be found in the details_hub_repo. + push_results_to_hub (bool): Whether to push the results to the Hugging Face hub. + push_samples_to_hub (bool): Whether to push the samples to the Hugging Face hub. + public_repo (bool): Whether to push the results to a public or private repository. + token (str): Token to use when pushing to the Hugging Face hub. This token should have write access to `hub_results_org`. + leaderboard_url (str): URL to the leaderboard on the Hugging Face hub on the dataset card. + point_of_contact (str): Contact information on the Hugging Face hub dataset card. + gated (bool): Whether to gate the repository. + """ + self.general_config_tracker = GeneralConfigTracker() + + self.output_path = output_path + self.push_results_to_hub = push_results_to_hub + self.push_samples_to_hub = push_samples_to_hub + self.public_repo = public_repo + self.leaderboard_url = leaderboard_url + self.point_of_contact = point_of_contact + self.api = HfApi(token=token) if token else None + self.gated_repo = gated + + if not self.api and (push_results_to_hub or push_samples_to_hub): + raise ValueError( + "Hugging Face token is not defined, but 'push_results_to_hub' or 'push_samples_to_hub' is set to True. " + "Please provide a valid Hugging Face token by setting the HF_TOKEN environment variable." + ) + + if ( + self.api + and hub_results_org == "" + and (push_results_to_hub or push_samples_to_hub) + ): + hub_results_org = self.api.whoami()["name"] + eval_logger.warning( + f"hub_results_org was not specified. Results will be pushed to '{hub_results_org}'." + ) + + if hub_repo_name == "": + details_repo_name = ( + details_repo_name if details_repo_name != "" else "lm-eval-results" + ) + results_repo_name = ( + results_repo_name if results_repo_name != "" else details_repo_name + ) + else: + details_repo_name = hub_repo_name + results_repo_name = hub_repo_name + eval_logger.warning( + "hub_repo_name was specified. Both details and results will be pushed to the same repository. Using hub_repo_name is no longer recommended, details_repo_name and results_repo_name should be used instead." + ) + + self.details_repo = f"{hub_results_org}/{details_repo_name}" + self.details_repo_private = f"{hub_results_org}/{details_repo_name}-private" + self.results_repo = f"{hub_results_org}/{results_repo_name}" + self.results_repo_private = f"{hub_results_org}/{results_repo_name}-private" + + def save_results_aggregated( + self, + results: dict, + samples: dict, + ) -> None: + """ + Saves the aggregated results and samples to the output path and pushes them to the Hugging Face hub if requested. + + Args: + results (dict): The aggregated results to save. + samples (dict): The samples results to save. + """ + self.general_config_tracker.log_end_time() + + if self.output_path: + try: + eval_logger.info("Saving results aggregated") + + # calculate cumulative hash for each task - only if samples are provided + task_hashes = {} + if samples: + for task_name, task_samples in samples.items(): + sample_hashes = [ + s["doc_hash"] + s["prompt_hash"] + s["target_hash"] + for s in task_samples + ] + task_hashes[task_name] = hash_string("".join(sample_hashes)) + + # update initial results dict + results.update({"task_hashes": task_hashes}) + results.update(asdict(self.general_config_tracker)) + dumped = json.dumps( + results, + indent=2, + default=handle_non_serializable, + ensure_ascii=False, + ) + + path = Path(self.output_path if self.output_path else Path.cwd()) + path = path.joinpath(self.general_config_tracker.model_name_sanitized) + path.mkdir(parents=True, exist_ok=True) + + self.date_id = datetime.now().isoformat().replace(":", "-") + file_results_aggregated = path.joinpath(f"results_{self.date_id}.json") + file_results_aggregated.open("w", encoding="utf-8").write(dumped) + + if self.api and self.push_results_to_hub: + repo_id = ( + self.results_repo + if self.public_repo + else self.results_repo_private + ) + self.api.create_repo( + repo_id=repo_id, + repo_type="dataset", + private=not self.public_repo, + exist_ok=True, + ) + self.api.upload_file( + repo_id=repo_id, + path_or_fileobj=str( + path.joinpath(f"results_{self.date_id}.json") + ), + path_in_repo=os.path.join( + self.general_config_tracker.model_name, + f"results_{self.date_id}.json", + ), + repo_type="dataset", + commit_message=f"Adding aggregated results for {self.general_config_tracker.model_name}", + ) + eval_logger.info( + "Successfully pushed aggregated results to the Hugging Face Hub. " + f"You can find them at: {repo_id}" + ) + + except Exception as e: + eval_logger.warning("Could not save results aggregated") + eval_logger.info(repr(e)) + else: + eval_logger.info( + "Output path not provided, skipping saving results aggregated" + ) + + def save_results_samples( + self, + task_name: str, + samples: dict, + ) -> None: + """ + Saves the samples results to the output path and pushes them to the Hugging Face hub if requested. + + Args: + task_name (str): The task name to save the samples for. + samples (dict): The samples results to save. + """ + if self.output_path: + try: + eval_logger.info(f"Saving per-sample results for: {task_name}") + + path = Path(self.output_path if self.output_path else Path.cwd()) + path = path.joinpath(self.general_config_tracker.model_name_sanitized) + path.mkdir(parents=True, exist_ok=True) + + file_results_samples = path.joinpath( + f"samples_{task_name}_{self.date_id}.jsonl" + ) + + for sample in samples: + # we first need to sanitize arguments and resps + # otherwise we won't be able to load the dataset + # using the datasets library + arguments = {} + for i, arg in enumerate(sample["arguments"]): + arguments[f"gen_args_{i}"] = {} + for j, tmp in enumerate(arg): + arguments[f"gen_args_{i}"][f"arg_{j}"] = tmp + + sample["resps"] = sanitize_list(sample["resps"]) + sample["filtered_resps"] = sanitize_list(sample["filtered_resps"]) + sample["arguments"] = arguments + sample["target"] = str(sample["target"]) + + sample_dump = ( + json.dumps( + sample, + default=handle_non_serializable, + ensure_ascii=False, + ) + + "\n" + ) + + with open(file_results_samples, "a", encoding="utf-8") as f: + f.write(sample_dump) + + if self.api and self.push_samples_to_hub: + repo_id = ( + self.details_repo + if self.public_repo + else self.details_repo_private + ) + self.api.create_repo( + repo_id=repo_id, + repo_type="dataset", + private=not self.public_repo, + exist_ok=True, + ) + try: + if self.gated_repo: + headers = build_hf_headers() + r = get_session().put( + url=f"https://huggingface.co/api/datasets/{repo_id}/settings", + headers=headers, + json={"gated": "auto"}, + ) + hf_raise_for_status(r) + except Exception as e: + eval_logger.warning("Could not gate the repository") + eval_logger.info(repr(e)) + self.api.upload_folder( + repo_id=repo_id, + folder_path=str(path), + path_in_repo=self.general_config_tracker.model_name_sanitized, + repo_type="dataset", + commit_message=f"Adding samples results for {task_name} to {self.general_config_tracker.model_name}", + ) + eval_logger.info( + f"Successfully pushed sample results for task: {task_name} to the Hugging Face Hub. " + f"You can find them at: {repo_id}" + ) + + except Exception as e: + eval_logger.warning("Could not save sample results") + eval_logger.info(repr(e)) + else: + eval_logger.info("Output path not provided, skipping saving sample results") + + def recreate_metadata_card(self) -> None: + """ + Creates a metadata card for the evaluation results dataset and pushes it to the Hugging Face hub. + """ + + eval_logger.info("Recreating metadata card") + repo_id = self.details_repo if self.public_repo else self.details_repo_private + + files_in_repo = self.api.list_repo_files(repo_id=repo_id, repo_type="dataset") + results_files = get_results_filenames(files_in_repo) + sample_files = get_sample_results_filenames(files_in_repo) + + # Build a dictionary to store the latest evaluation datetime for: + # - Each tested model and its aggregated results + # - Each task and sample results, if existing + # i.e. { + # "org__model_name__gsm8k": "2021-09-01T12:00:00", + # "org__model_name__ifeval": "2021-09-01T12:00:00", + # "org__model_name__results": "2021-09-01T12:00:00" + # } + latest_task_results_datetime = defaultdict(lambda: datetime.min.isoformat()) + + for file_path in sample_files: + file_path = Path(file_path) + filename = file_path.name + model_name = file_path.parent + task_name = get_file_task_name(filename) + results_datetime = get_file_datetime(filename) + task_name_sanitized = sanitize_task_name(task_name) + # Results and sample results for the same model and task will have the same datetime + samples_key = f"{model_name}__{task_name_sanitized}" + results_key = f"{model_name}__results" + latest_datetime = max( + latest_task_results_datetime[samples_key], + results_datetime, + ) + latest_task_results_datetime[samples_key] = latest_datetime + latest_task_results_datetime[results_key] = max( + latest_task_results_datetime[results_key], + latest_datetime, + ) + + # Create metadata card + card_metadata = MetadataConfigs() + + # Add the latest aggregated results to the metadata card for easy access + for file_path in results_files: + file_path = Path(file_path) + results_filename = file_path.name + model_name = file_path.parent + eval_date = get_file_datetime(results_filename) + eval_date_sanitized = re.sub(r"[^\w\.]", "_", eval_date) + results_filename = Path("**") / Path(results_filename).name + config_name = f"{model_name}__results" + sanitized_last_eval_date_results = re.sub( + r"[^\w\.]", "_", latest_task_results_datetime[config_name] + ) + + if eval_date_sanitized == sanitized_last_eval_date_results: + # Ensure that all results files are listed in the metadata card + current_results = card_metadata.get(config_name, {"data_files": []}) + current_results["data_files"].append( + {"split": eval_date_sanitized, "path": [str(results_filename)]} + ) + card_metadata[config_name] = current_results + # If the results file is the newest, update the "latest" field in the metadata card + card_metadata[config_name]["data_files"].append( + {"split": "latest", "path": [str(results_filename)]} + ) + + # Add the tasks details configs + for file_path in sample_files: + file_path = Path(file_path) + filename = file_path.name + model_name = file_path.parent + task_name = get_file_task_name(filename) + eval_date = get_file_datetime(filename) + task_name_sanitized = sanitize_task_name(task_name) + eval_date_sanitized = re.sub(r"[^\w\.]", "_", eval_date) + results_filename = Path("**") / Path(filename).name + config_name = f"{model_name}__{task_name_sanitized}" + sanitized_last_eval_date_results = re.sub( + r"[^\w\.]", "_", latest_task_results_datetime[config_name] + ) + if eval_date_sanitized == sanitized_last_eval_date_results: + # Ensure that all sample results files are listed in the metadata card + current_details_for_task = card_metadata.get( + config_name, {"data_files": []} + ) + current_details_for_task["data_files"].append( + {"split": eval_date_sanitized, "path": [str(results_filename)]} + ) + card_metadata[config_name] = current_details_for_task + # If the samples results file is the newest, update the "latest" field in the metadata card + card_metadata[config_name]["data_files"].append( + {"split": "latest", "path": [str(results_filename)]} + ) + + # Get latest results and extract info to update metadata card examples + latest_datetime = max(latest_task_results_datetime.values()) + latest_model_name = max( + latest_task_results_datetime, key=lambda k: latest_task_results_datetime[k] + ) + last_results_file = [ + f for f in results_files if latest_datetime.replace(":", "-") in f + ][0] + last_results_file_path = hf_hub_url( + repo_id=repo_id, filename=last_results_file, repo_type="dataset" + ) + latest_results_file = load_dataset( + "json", data_files=last_results_file_path, split="train" + ) + results_dict = latest_results_file["results"][0] + new_dictionary = {"all": results_dict} + new_dictionary.update(results_dict) + results_string = json.dumps(new_dictionary, indent=4) + + dataset_summary = ( + "Dataset automatically created during the evaluation run of model " + ) + if self.general_config_tracker.model_source == "hf": + dataset_summary += f"[{self.general_config_tracker.model_name}](https://huggingface.co/{self.general_config_tracker.model_name})\n" + else: + dataset_summary += f"{self.general_config_tracker.model_name}\n" + dataset_summary += ( + f"The dataset is composed of {len(card_metadata) - 1} configuration(s), each one corresponding to one of the evaluated task.\n\n" + f"The dataset has been created from {len(results_files)} run(s). Each run can be found as a specific split in each " + 'configuration, the split being named using the timestamp of the run.The "train" split is always pointing to the latest results.\n\n' + 'An additional configuration "results" store all the aggregated results of the run.\n\n' + "To load the details from a run, you can for instance do the following:\n" + ) + if self.general_config_tracker.model_source == "hf": + dataset_summary += ( + "```python\nfrom datasets import load_dataset\n" + f'data = load_dataset(\n\t"{repo_id}",\n\tname="{latest_model_name}",\n\tsplit="latest"\n)\n```\n\n' + ) + dataset_summary += ( + "## Latest results\n\n" + f"These are the [latest results from run {latest_datetime}]({last_results_file_path.replace('/resolve/', '/blob/')}) " + "(note that there might be results for other tasks in the repos if successive evals didn't cover the same tasks. " + 'You find each in the results and the "latest" split for each eval):\n\n' + f"```python\n{results_string}\n```" + ) + card_data = DatasetCardData( + dataset_summary=dataset_summary, + repo_url=f"https://huggingface.co/{self.general_config_tracker.model_name}", + pretty_name=f"Evaluation run of {self.general_config_tracker.model_name}", + leaderboard_url=self.leaderboard_url, + point_of_contact=self.point_of_contact, + ) + card_metadata.to_dataset_card_data(card_data) + card = DatasetCard.from_template( + card_data, + pretty_name=card_data.pretty_name, + ) + card.push_to_hub(repo_id, repo_type="dataset") diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/loggers/utils.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/loggers/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c2640953603635d2d96dd85bde9ef14b0175cac2 --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/loggers/utils.py @@ -0,0 +1,149 @@ +import logging +import os +import re +import subprocess +from importlib.metadata import version +from pathlib import Path +from typing import Any, Dict, Optional, Tuple, Union + +import numpy as np +from torch.utils.collect_env import get_pretty_env_info +from transformers import __version__ as trans_version + + +logger = logging.getLogger(__name__) + + +def remove_none_pattern(input_string: str) -> Tuple[str, bool]: + """Remove the ',none' substring from the input_string if it exists at the end. + + Args: + input_string (str): The input string from which to remove the ',none' substring. + + Returns: + Tuple[str, bool]: A tuple containing the modified input_string with the ',none' substring removed + and a boolean indicating whether the modification was made (True) or not (False). + """ + # Define the pattern to match ',none' at the end of the string + pattern = re.compile(r",none$") + + # Use sub() to replace ',none' with an empty string + result = re.sub(pattern, "", input_string) + + # check if the input_string changed + removed = result != input_string + + return result, removed + + +def _handle_non_serializable(o: Any) -> Union[int, str, list]: + """Handle non-serializable objects by converting them to serializable types. + + Args: + o (Any): The object to be handled. + + Returns: + Union[int, str, list]: The converted object. If the object is of type np.int64 or np.int32, + it will be converted to int. If the object is of type set, it will be converted + to a list. Otherwise, it will be converted to str. + """ + if isinstance(o, np.int64) or isinstance(o, np.int32): + return int(o) + elif isinstance(o, set): + return list(o) + else: + return str(o) + + +def get_commit_from_path(repo_path: Union[Path, str]) -> Optional[str]: + try: + git_folder = Path(repo_path, ".git") + if git_folder.is_file(): + git_folder = Path( + git_folder.parent, + git_folder.read_text(encoding="utf-8").split("\n")[0].split(" ")[-1], + ) + if Path(git_folder, "HEAD").exists(): + head_name = ( + Path(git_folder, "HEAD") + .read_text(encoding="utf-8") + .split("\n")[0] + .split(" ")[-1] + ) + head_ref = Path(git_folder, head_name) + git_hash = head_ref.read_text(encoding="utf-8").replace("\n", "") + else: + git_hash = None + except Exception as err: + logger.debug( + f"Failed to retrieve a Git commit hash from path: {str(repo_path)}. Error: {err}" + ) + return None + return git_hash + + +def get_git_commit_hash(): + """ + Gets the git commit hash of your current repo (if it exists). + Source: https://github.com/EleutherAI/gpt-neox/blob/b608043be541602170bfcfb8ec9bf85e8a0799e0/megatron/neox_arguments/neox_args.py#L42 + """ + try: + git_hash = subprocess.check_output(["git", "describe", "--always"]).strip() + git_hash = git_hash.decode() + except (subprocess.CalledProcessError, FileNotFoundError): + # FileNotFoundError occurs when git not installed on system + git_hash = get_commit_from_path(os.getcwd()) # git hash of repo if exists + return git_hash + + +def add_env_info(storage: Dict[str, Any]): + try: + pretty_env_info = get_pretty_env_info() + except Exception as err: + pretty_env_info = str(err) + try: + lm_eval_version = version("lm_eval") + except Exception as err: + lm_eval_version = str(err) + transformers_version = trans_version + upper_dir_commit = get_commit_from_path( + Path(os.getcwd(), "..") + ) # git hash of upper repo if exists + added_info = { + "pretty_env_info": pretty_env_info, + "transformers_version": transformers_version, + "lm_eval_version": lm_eval_version, + "upper_git_hash": upper_dir_commit, # in case this repo is submodule + } + storage.update(added_info) + + +def add_tokenizer_info(storage: Dict[str, Any], lm): + if getattr(lm, "tokenizer", False): + try: + tokenizer_info = { + "tokenizer_pad_token": [ + lm.tokenizer.pad_token, + str(lm.tokenizer.pad_token_id), + ], + "tokenizer_eos_token": [ + lm.tokenizer.eos_token, + str(lm.tokenizer.eos_token_id), + ], + "tokenizer_bos_token": [ + lm.tokenizer.bos_token, + str(lm.tokenizer.bos_token_id), + ], + "eot_token_id": getattr(lm, "eot_token_id", None), + "max_length": getattr(lm, "max_length", None), + } + storage.update(tokenizer_info) + except Exception as err: + logger.debug( + f"Logging detailed tokenizer info failed with {err}, skipping..." + ) + # seems gguf and textsynth do not have tokenizer + else: + logger.debug( + "LM does not have a 'tokenizer' attribute, not logging tokenizer metadata to results." + ) diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/loggers/wandb_logger.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/loggers/wandb_logger.py new file mode 100644 index 0000000000000000000000000000000000000000..8795f5d8343143ac3badcde98e536c003a1a3fb8 --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/loggers/wandb_logger.py @@ -0,0 +1,358 @@ +import copy +import json +import logging +from typing import Any, Dict, List, Literal, Tuple + +import numpy as np +import pandas as pd +from packaging.version import Version + +from lm_eval.loggers.utils import _handle_non_serializable, remove_none_pattern + + +logger = logging.getLogger(__name__) + + +def get_wandb_printer() -> Literal["Printer"]: + """Returns a wandb printer instance for pretty stdout.""" + from wandb.sdk.lib.printer import new_printer + + printer = new_printer() + return printer + + +class WandbLogger: + def __init__(self, init_args=None, config_args=None) -> None: + """Attaches to wandb logger if already initialized. Otherwise, passes init_args to wandb.init() and config_args to wandb.config.update() + + Args: + init_args Optional[Dict]: Arguments for init configuration. + config_args Optional[Dict]: Arguments for config + + Parse and log the results returned from evaluator.simple_evaluate() with: + wandb_logger.post_init(results) + wandb_logger.log_eval_result() + wandb_logger.log_eval_samples(results["samples"]) + """ + try: + import wandb + + assert Version(wandb.__version__) >= Version("0.13.6") + if Version(wandb.__version__) < Version("0.13.6"): + wandb.require("report-editing:v0") + except Exception as e: + logger.warning( + "To use the wandb reporting functionality please install wandb>=0.13.6.\n" + "To install the latest version of wandb run `pip install wandb --upgrade`\n" + f"{e}" + ) + + self.wandb_args: Dict[str, Any] = init_args or {} + self.wandb_config_args: Dict[str, Any] = config_args or {} + + # pop the step key from the args to save for all logging calls + self.step = self.wandb_args.pop("step", None) + + # initialize a W&B run + if wandb.run is None: + self.run = wandb.init(**self.wandb_args) + if self.wandb_config_args: + self.run.config.update(self.wandb_config_args) + else: + self.run = wandb.run + + self.printer = get_wandb_printer() + + def post_init(self, results: Dict[str, Any]) -> None: + self.results: Dict[str, Any] = copy.deepcopy(results) + self.task_names: List[str] = list(results.get("results", {}).keys()) + self.group_names: List[str] = list(results.get("groups", {}).keys()) + + def _get_config(self) -> Dict[str, Any]: + """Get configuration parameters.""" + self.task_configs = self.results.get("configs", {}) + cli_configs = self.results.get("config", {}) + configs = { + "task_configs": self.task_configs, + "cli_configs": cli_configs, + } + + return configs + + def _sanitize_results_dict(self) -> Tuple[Dict[str, str], Dict[str, Any]]: + """Sanitize the results dictionary.""" + _results = copy.deepcopy(self.results.get("results", dict())) + + # Remove None from the metric string name + tmp_results = copy.deepcopy(_results) + for task_name in self.task_names: + task_result = tmp_results.get(task_name, dict()) + for metric_name, metric_value in task_result.items(): + _metric_name, removed = remove_none_pattern(metric_name) + if removed: + _results[task_name][_metric_name] = metric_value + _results[task_name].pop(metric_name) + + # remove string valued keys from the results dict + wandb_summary = {} + for task in self.task_names: + task_result = _results.get(task, dict()) + for metric_name, metric_value in task_result.items(): + if isinstance(metric_value, str): + wandb_summary[f"{task}/{metric_name}"] = metric_value + + for summary_metric, summary_value in wandb_summary.items(): + _task, _summary_metric = summary_metric.split("/") + _results[_task].pop(_summary_metric) + + tmp_results = copy.deepcopy(_results) + for task_name, task_results in tmp_results.items(): + for metric_name, metric_value in task_results.items(): + _results[f"{task_name}/{metric_name}"] = metric_value + _results[task_name].pop(metric_name) + for task in self.task_names: + _results.pop(task) + + return wandb_summary, _results + + def _log_results_as_table(self) -> None: + """Generate and log evaluation results as a table to W&B.""" + columns = [ + "Version", + "Filter", + "num_fewshot", + "Metric", + "Value", + "Stderr", + ] + + def make_table(columns: List[str], key: str = "results"): + import wandb + + table = wandb.Table(columns=columns) + results = copy.deepcopy(self.results) + + for k, dic in results.get(key).items(): + if k in self.group_names and not key == "groups": + continue + version = results.get("versions").get(k) + if version == "N/A": + version = None + n = results.get("n-shot").get(k) + + for (mf), v in dic.items(): + m, _, f = mf.partition(",") + if m.endswith("_stderr"): + continue + if m == "alias": + continue + + if m + "_stderr" + "," + f in dic: + se = dic[m + "_stderr" + "," + f] + if se != "N/A": + se = "%.4f" % se + table.add_data(*[k, version, f, n, m, str(v), str(se)]) + else: + table.add_data(*[k, version, f, n, m, str(v), ""]) + + return table + + # log the complete eval result to W&B Table + table = make_table(["Tasks"] + columns, "results") + self.run.log({"evaluation/eval_results": table}, step=self.step) + + if "groups" in self.results.keys(): + table = make_table(["Groups"] + columns, "groups") + self.run.log({"evaluation/group_eval_results": table}, step=self.step) + + def _log_results_as_artifact(self) -> None: + """Log results as JSON artifact to W&B.""" + import wandb + + dumped = json.dumps( + self.results, indent=2, default=_handle_non_serializable, ensure_ascii=False + ) + artifact = wandb.Artifact("results", type="eval_results") + with artifact.new_file("results.json", mode="w", encoding="utf-8") as f: + f.write(dumped) + self.run.log_artifact(artifact) + + def log_eval_result(self) -> None: + """Log evaluation results to W&B.""" + # Log configs to wandb + configs = self._get_config() + self.run.config.update(configs, allow_val_change=self.step is not None) + + wandb_summary, self.wandb_results = self._sanitize_results_dict() + # update wandb.run.summary with items that were removed + self.run.summary.update(wandb_summary) + # Log the evaluation metrics to wandb + self.run.log(self.wandb_results, step=self.step) + # Log the evaluation metrics as W&B Table + self._log_results_as_table() + # Log the results dict as json to W&B Artifacts + self._log_results_as_artifact() + + def _generate_dataset( + self, data: List[Dict[str, Any]], config: Dict[str, Any] + ) -> pd.DataFrame: + """Generate a dataset from evaluation data. + + Args: + data (List[Dict[str, Any]]): The data to generate a dataset for. + config (Dict[str, Any]): The configuration of the task. + + Returns: + pd.DataFrame: A dataframe that is ready to be uploaded to W&B. + """ + ids = [x["doc_id"] for x in data] + labels = [x["target"] for x in data] + instance = [""] * len(ids) + resps = [""] * len(ids) + filtered_resps = [""] * len(ids) + model_outputs = {} + + metrics_list = config["metric_list"] + metrics = {} + for metric in metrics_list: + metric = metric.get("metric") + if metric in ["word_perplexity", "byte_perplexity", "bits_per_byte"]: + metrics[f"{metric}_loglikelihood"] = [x[metric][0] for x in data] + if metric in ["byte_perplexity", "bits_per_byte"]: + metrics[f"{metric}_bytes"] = [x[metric][1] for x in data] + else: + metrics[f"{metric}_words"] = [x[metric][1] for x in data] + else: + metrics[metric] = [x[metric] for x in data] + + if config["output_type"] == "loglikelihood": + instance = [x["arguments"][0][0] for x in data] + labels = [x["arguments"][0][1] for x in data] + resps = [ + f"log probability of continuation is {x['resps'][0][0][0]} " + + "\n\n" + + "continuation will {} generated with greedy sampling".format( + "not be" if not x["resps"][0][0][1] else "be" + ) + for x in data + ] + filtered_resps = [ + f"log probability of continuation is {x['filtered_resps'][0][0]} " + + "\n\n" + + "continuation will {} generated with greedy sampling".format( + "not be" if not x["filtered_resps"][0][1] else "be" + ) + for x in data + ] + elif config["output_type"] == "multiple_choice": + instance = [x["arguments"][0][0] for x in data] + choices = [ + "\n".join([f"{idx}. {y[1]}" for idx, y in enumerate(x["arguments"])]) + for x in data + ] + resps = [np.argmax([n[0][0] for n in x["resps"]]) for x in data] + filtered_resps = [ + np.argmax([n[0] for n in x["filtered_resps"]]) for x in data + ] + elif config["output_type"] == "loglikelihood_rolling": + instance = [x["arguments"][0][0] for x in data] + resps = [x["resps"][0][0] for x in data] + filtered_resps = [x["filtered_resps"][0] for x in data] + elif config["output_type"] == "generate_until": + instance = [x["arguments"][0][0] for x in data] + resps = [x["resps"][0][0] for x in data] + filtered_resps = [x["filtered_resps"][0] for x in data] + + model_outputs["raw_predictions"] = resps + model_outputs["filtered_predictions"] = filtered_resps + + df_data = { + "id": ids, + "data": instance, + } + if config["output_type"] == "multiple_choice": + df_data["choices"] = choices + + tmp_data = { + "input_len": [len(x) for x in instance], + "labels": labels, + "output_type": config["output_type"], + } + df_data.update(tmp_data) + df_data.update(model_outputs) + df_data.update(metrics) + + return pd.DataFrame(df_data) + + def _log_samples_as_artifact( + self, data: List[Dict[str, Any]], task_name: str + ) -> None: + import wandb + + # log the samples as an artifact + dumped = json.dumps( + data, + indent=2, + default=_handle_non_serializable, + ensure_ascii=False, + ) + artifact = wandb.Artifact(f"{task_name}", type="samples_by_task") + with artifact.new_file( + f"{task_name}_eval_samples.json", mode="w", encoding="utf-8" + ) as f: + f.write(dumped) + self.run.log_artifact(artifact) + # artifact.wait() + + def log_eval_samples(self, samples: Dict[str, List[Dict[str, Any]]]) -> None: + """Log evaluation samples to W&B. + + Args: + samples (Dict[str, List[Dict[str, Any]]]): Evaluation samples for each task. + """ + task_names: List[str] = [ + x for x in self.task_names if x not in self.group_names + ] + + ungrouped_tasks = [] + tasks_by_groups = {} + + for task_name in task_names: + group_names = self.task_configs[task_name].get("group", None) + if group_names: + if isinstance(group_names, str): + group_names = [group_names] + + for group_name in group_names: + if not tasks_by_groups.get(group_name): + tasks_by_groups[group_name] = [task_name] + else: + tasks_by_groups[group_name].append(task_name) + else: + ungrouped_tasks.append(task_name) + + for task_name in ungrouped_tasks: + eval_preds = samples[task_name] + + # log the samples as a W&B Table + df = self._generate_dataset(eval_preds, self.task_configs.get(task_name)) + self.run.log({f"{task_name}_eval_results": df}, step=self.step) + + # log the samples as a json file as W&B Artifact + self._log_samples_as_artifact(eval_preds, task_name) + + for group, grouped_tasks in tasks_by_groups.items(): + grouped_df = pd.DataFrame() + for task_name in grouped_tasks: + eval_preds = samples[task_name] + df = self._generate_dataset( + eval_preds, self.task_configs.get(task_name) + ) + df["group"] = group + df["task"] = task_name + grouped_df = pd.concat([grouped_df, df], ignore_index=True) + + # log the samples as a json file as W&B Artifact + self._log_samples_as_artifact(eval_preds, task_name) + + self.run.log({f"{group}_eval_results": grouped_df}, step=self.step) diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/models/__init__.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e46d196a6bce3df235310b278be7e7e11cb950c3 --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/models/__init__.py @@ -0,0 +1,17 @@ +from . import ( + diffllm, + huggingface, +) + + +# TODO: implement __all__ + + +try: + # enable hf hub transfer if available + import hf_transfer # type: ignore # noqa + import huggingface_hub.constants # type: ignore + + huggingface_hub.constants.HF_HUB_ENABLE_HF_TRANSFER = True +except ImportError: + pass diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/models/diffllm.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/models/diffllm.py new file mode 100644 index 0000000000000000000000000000000000000000..9fbc269818264e48062761987cc90ff4e383313f --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/models/diffllm.py @@ -0,0 +1,563 @@ +import logging +import gc +import random +import json +import os +import time +from datetime import timedelta +from typing import List, Optional, Tuple, Type, TypeVar, Union + +import torch +import torch.nn.functional as F +import transformers +from accelerate import ( + Accelerator, + InitProcessGroupKwargs, + find_executable_batch_size, +) +from datasets import Dataset +from packaging import version +from tqdm import tqdm + +from lm_eval import utils +from lm_eval.api.instance import Instance +from lm_eval.api.model import LM +from lm_eval.api.registry import register_model +from lm_eval.models.utils import Collator, get_dtype + +eval_logger = logging.getLogger(__name__) +T = TypeVar("T", bound="LM") + + +def empty_cache_by_memory(threshold_gb=70): + """ + Empty CUDA cache if allocated memory exceeds threshold + Args: + threshold_gb: Memory threshold in GB + """ + if torch.cuda.is_available(): + # Get current memory allocated + allocated = torch.cuda.memory_allocated() / 1024**3 # Convert to GB + + if allocated > threshold_gb: + # Clear cache + gc.collect() + torch.cuda.empty_cache() + print(f"Cache cleared. Memory freed: {allocated:.2f} GB") + +@register_model("diffllm") +class DiffLLM(LM): + def __init__( + self, + pretrained: Union[str, transformers.PreTrainedModel], + batch_size: Optional[Union[int, str]] = 1, + device: Optional[str] = "cuda", + dtype: Optional[Union[str, torch.dtype]] = "auto", + max_prompt_len: Optional[int] = 1024, + max_new_tokens: Optional[int] = 128, + nll_type: Optional[str] = "mc", + log_type: Optional[str] = "ftb", + classifier_free_guidance: Optional[float] = 1.0, + pad_to_max_len: Optional[bool] = False, + sampling_eps: Optional[float] = 1e-3, + diffusion_steps: Optional[int] = 32, + trust_remote_code: Optional[bool] = True, + parallelize: Optional[bool] = False, + autogptq: Optional[Union[bool, str]] = False, + **kwargs, + ) -> None: + super().__init__() + + # prepare for parallelism + assert isinstance(device, str) + assert isinstance(pretrained, str) + assert isinstance(batch_size, (int, str)) + + gpus = torch.cuda.device_count() + accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52)) + accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs]) + + self.accelerator = accelerator + + if "npu" in accelerator.device.type: + gpus = torch.npu.device_count() + + # using one process with no model parallelism + if not (parallelize or accelerator.num_processes > 1): + # use user-passed device + device_list = set( + ["cuda", "cpu"] + + [f"cuda:{i}" for i in range(gpus)] + + ["mps", "mps:0"] + + [f"npu:{i}" for i in range(gpus)] + ) + if device and device in device_list: + self._device = torch.device(device) + eval_logger.info(f"Using device '{device}'") + if device in ("mps", "mps:0") and version.parse( + torch.__version__ + ) < version.parse("2.1"): + raise RuntimeError( + f"mps requires torch >= 2.1. You have {torch.__version__}" + ) + else: + eval_logger.info("Device not specified") + eval_logger.info(f"Cuda Available? {torch.cuda.is_available()}") + self._device = ( + torch.device("cuda") + if torch.cuda.is_available() + else torch.device("cpu") + ) + else: + if device != "cuda": + eval_logger.info( + f"Using `accelerate launch` or `parallelize=True`, device '{device}' will be overridden when placing model." + ) + self._device = self.accelerator.device + + self.batch_size_per_gpu = batch_size + if isinstance(batch_size, str): + self.batch_size_per_gpu = int(batch_size) + self._create_model_and_tokenizer(pretrained, dtype, trust_remote_code) + + if isinstance(pretrained, str): + if gpus >= 1 or str(self.device) == "mps": + if not (parallelize or autogptq or (hasattr(self, "accelerator") and self.accelerator.num_processes > 1)): + try: + self.model.to(self.device) + except ValueError: + eval_logger.debug( + "Failed to place model onto specified device. This may be because the model is quantized via `bitsandbytes` or `device_map` is provided. If the desired GPU is being used, this message is safe to ignore." + ) + if gpus > 1: + if self.accelerator.num_processes > 1: + self._device = torch.device(f"{accelerator.device}") + self._rank = self.accelerator.local_process_index + self._world_size = self.accelerator.num_processes + else: + self._rank = 0 + self._world_size = 1 + else: + self._rank = 0 + self._world_size = 1 + else: + eval_logger.warning( + "Passed an already-initialized model through `pretrained`, assuming single-process call to evaluate() or custom distributed integration" + ) + self._rank = 0 + self._world_size = 1 + + self.max_prompt_len = max_prompt_len + self.max_new_tokens = max_new_tokens + self.diffusion_steps = diffusion_steps + self.temperature = kwargs.get("temperature", 0.7) + self.top_p = kwargs.get("top_p", 0.95) + self.alg = kwargs.get("alg", "entropy") + self.alg_temp = kwargs.get("alg_temp", 0.0) + self.top_k = kwargs.get("top_k", None) + + self.nll_type = nll_type + self.log_type = log_type + self.classifier_free_guidance = classifier_free_guidance + self.pad_to_max_len = pad_to_max_len + self.sampling_eps = sampling_eps + + self.mask_id = 151666 + self.eos_id = 151643 + + raw_use_hts = kwargs.get("use_hts", False) + if isinstance(raw_use_hts, str): + self.use_hts = raw_use_hts.lower() == "true" + else: + self.use_hts = bool(raw_use_hts) + + self.realtime_output = kwargs.get("realtime_output", "eval_results.jsonl") + + if self.use_hts: + from .hts_sampler import HTSSampler + self.hts_sampler = HTSSampler(self.model, self.tokenizer, device=self.device) + eval_logger.info(f"Rank {self.rank}: HTS Sampler initialized for Dream.") + + @property + def batch_size(self): + return self.batch_size_per_gpu + + @property + def device(self): + return self._device + + @property + def rank(self): + return self._rank + + @property + def world_size(self): + return self._world_size + + def _create_model_and_tokenizer(self, pretrained, dtype, trust_remote_code): + self.model = ( + transformers.AutoModel.from_pretrained( + pretrained, + torch_dtype=get_dtype(dtype), + trust_remote_code=trust_remote_code, + ) + .eval() + ).to(self.device) + self.tokenizer = transformers.AutoTokenizer.from_pretrained( + pretrained, trust_remote_code=trust_remote_code + ) + + def tok_decode(self, tokens, skip_special_tokens=True): + return self.tokenizer.decode(tokens, skip_special_tokens=skip_special_tokens) + + def tok_encode(self, text, add_special_tokens=True): + return self.tokenizer( + text, return_tensors="pt", add_special_tokens=add_special_tokens + ).input_ids + + @classmethod + def create_from_arg_string( + cls: Type[T], arg_string: str, additional_config: Optional[dict] = None + ) -> T: + additional_config = {} if additional_config is None else additional_config + args = utils.simple_parse_args_string(arg_string) + args2 = {k: v for k, v in additional_config.items() if v is not None} + return cls(**args, **args2) + + def apply_chat_template( + self, chat_history, add_generation_prompt: bool = True + ) -> str: + chat_templated = self.tokenizer.apply_chat_template( + chat_history, + tokenize=False, + add_generation_prompt=add_generation_prompt, + continue_final_message=not add_generation_prompt, + ) + return chat_templated + + @property + def tokenizer_name(self) -> str: + return self.tokenizer.name_or_path.replace("/", "__") + + def _generate_batch(self, prompts: List[str], gen_kwargs: dict = None) -> Tuple[List[str], List[dict]]: + raw_val = gen_kwargs.get("use_hts", self.use_hts) + use_hts_now = str(raw_val).lower() == "true" if not isinstance(raw_val, bool) else raw_val + + all_stats = [] + if not use_hts_now: + prompt_ids = self.tokenizer(prompts, return_tensors="pt", padding=True, padding_side="left").input_ids + prompt_ids = prompt_ids[:, -self.max_prompt_len:] + attn_mask = prompt_ids.ne(self.tokenizer.pad_token_id).to(self.device) + prompt_ids = prompt_ids.to(device=self.device) + + generation_ids = self.model.diffusion_generate( + prompt_ids, + attention_mask=attn_mask, + max_new_tokens=self.max_new_tokens, + output_history=False, + return_dict_in_generate=True, + steps=self.diffusion_steps, + temperature=self.temperature, + top_p=self.top_p, + top_k=self.top_k, + alg=self.alg, + alg_temp=self.alg_temp, + ) + responses = [ + self.tokenizer.decode(g[len(p) :].tolist()).split(self.tokenizer.eos_token)[0] + for p, g in zip(prompt_ids, generation_ids.sequences) + ] + all_stats = [{} for _ in responses] + return responses, all_stats + else: + if not hasattr(self, "hts_sampler"): + from .hts_sampler import HTSSampler + self.hts_sampler = HTSSampler(self.model, self.tokenizer, device=self.device) + + results = [] + for prompt in prompts: + input_ids = self.tokenizer(prompt, return_tensors="pt").input_ids.to(self.device) + + final_codes, stats = self.hts_sampler.generate_hts( + prompt_text=prompt, + input_ids=input_ids, + initial_N=int(gen_kwargs.get("initial_N", 4)), + final_K=int(gen_kwargs.get("final_K", 1)), + hts_survivor_k=int(gen_kwargs.get("hts_survivor_k", 4)), + reward_mode=gen_kwargs.get("reward_mode", "svf"), + task_type=gen_kwargs.get("task_type", "code"), + steps=self.diffusion_steps, + gen_length=self.max_new_tokens, + temperature=float(gen_kwargs.get("temperature", self.temperature)), + top_p=float(gen_kwargs.get("top_p", self.top_p)), + top_k=gen_kwargs.get("top_k", self.top_k), + until=gen_kwargs.get("until", []), + hts_mode=True, + mask_id=self.mask_id, + eos_id=self.eos_id + ) + + results.append(final_codes[0]) + all_stats.append(stats) + return results, all_stats + + def generate_until(self, requests: List[Instance], disable_tqdm: bool = False): + res = [] + + gen_kwargs_first = requests[0].args[1] + actual_output_path = gen_kwargs_first.get("realtime_output", self.realtime_output) + + raw_val = gen_kwargs_first.get("use_hts", self.use_hts) + self.use_hts = str(raw_val).lower() == "true" if not isinstance(raw_val, bool) else raw_val + + rank_tmp_file = actual_output_path.replace(".jsonl", f"_rank{self.rank}.tmp") + + output_dir = os.path.dirname(rank_tmp_file) + if output_dir and not os.path.exists(output_dir): + os.makedirs(output_dir, exist_ok=True) + + pbar = tqdm( + total=len(requests), + disable=(disable_tqdm or (self.rank != 0)), + desc="Running generate_until", + ) + + for batch_idx in range(0, len(requests), self.batch_size_per_gpu): + batch_requests = requests[batch_idx : batch_idx + self.batch_size_per_gpu] + contexts, task_gen_args = zip(*[req.arguments for req in batch_requests]) + + responses, stats_list = self._generate_batch(contexts, gen_kwargs=task_gen_args[0]) + + for i, r in enumerate(responses): + r = r.replace("```python", "").replace("```", "") + + for s in task_gen_args[0].get('until', []): + r = r.split(s)[0] + + target_val = getattr(batch_requests[i], "target", None) + if target_val is None or target_val == "N/A": + target_val = batch_requests[i].doc.get("answer", batch_requests[i].doc.get("solution", "N/A")) + + save_data = { + "doc": batch_requests[i].doc, + "target": target_val, + "prompt": contexts[i], + "response": r, + } + + if self.use_hts: + save_data.update(stats_list[i]) + + with open(rank_tmp_file, "a", encoding="utf-8") as f: + f.write(json.dumps(save_data, ensure_ascii=False) + "\n") + f.flush() + + responses[i] = r + + if self.rank == 0 and batch_idx == 0: + print(f"Sample Response:\n{responses[0]}\n") + + res.extend(responses) + pbar.update(len(batch_requests)) + + pbar.close() + + self.accelerator.wait_for_everyone() + + if self.rank == 0: + eval_logger.info(f"Merging rank files into {actual_output_path}...") + with open(actual_output_path, "w", encoding="utf-8") as final_f: + for r in range(self.world_size): + temp_f = actual_output_path.replace(".jsonl", f"_rank{r}.tmp") + if os.path.exists(temp_f): + with open(temp_f, "r", encoding="utf-8") as tf: + for line in tf: + final_f.write(line) + os.remove(temp_f) + eval_logger.info("Merge completed.") + + return res + + def _forward_process(self, batch): + b, l = batch.shape + u0 = torch.rand(1, device=batch.device, dtype=torch.float32) + indices = torch.arange(b, device=batch.device).float() + t = (u0 + indices / b) % 1 + p_mask = (1 - self.sampling_eps) * t + self.sampling_eps + p_mask = p_mask[:, None].repeat(1, l) + mask_indices = torch.rand((b, l), device=batch.device) < p_mask + mask_indices[:, 0] = False + mask_indices[:, -1] = False + noisy_batch = torch.where(mask_indices, self.mask_id, batch) + return noisy_batch, p_mask + + @torch.no_grad() + def get_logits(self, batch, prompt_index): + if self.classifier_free_guidance > 1.: + assert len(prompt_index) == batch.shape[1] + prompt_index = prompt_index.unsqueeze(0).repeat(batch.shape[0], 1) + un_batch = batch.clone() + un_batch[prompt_index] = self.mask_id + batch = torch.cat([batch, un_batch]) + + if self.pad_to_max_len: + raise NotImplementedError + else: + input = batch + + with torch.amp.autocast('cuda', dtype=torch.bfloat16): + logits = self.model(input, 'full').logits + + if self.classifier_free_guidance > 1.: + logits, un_logits = torch.chunk(logits, 2, dim=0) + logits = un_logits + self.classifier_free_guidance * (logits - un_logits) + return logits[:, :batch.shape[1]] + + @torch.no_grad() + def _eval_target_nll_mc(self, prefix, target): + if prefix is None: + seq = target[None, :] + else: + seq = torch.concatenate([prefix, target])[None, :] + seq = seq.repeat((self.batch_size, 1)).to(self.device) + + if self.log_type == 'ftb': + prompt_index = torch.arange(seq.shape[1], device=self.device) < len(prefix) + else: + prompt_index = torch.arange(seq.shape[1], device=self.device) >= len(prefix) + + loss_acc = [] + mc_num = self.diffusion_steps + for _ in range(max(mc_num // self.batch_size, 1)): + perturbed_seq = seq.clone() + perturbed_seq_, p_mask = self._forward_process(seq) + if self.log_type == 'ftb': + perturbed_seq[:, -len(target):] = perturbed_seq_[:, -len(target):] + elif self.log_type == 'btf': + perturbed_seq[:, :len(prefix)] = perturbed_seq_[:, :len(prefix)] + elif self.log_type == 'union': + perturbed_seq = perturbed_seq_ + else: + raise NotImplementedError(self.log_type) + + mask_indices = perturbed_seq == self.mask_id + + logits = self.get_logits(perturbed_seq, prompt_index) + + loss = F.cross_entropy(logits[mask_indices], seq[mask_indices], reduction='none') / p_mask[mask_indices] + loss = loss.sum() / self.batch_size + loss_acc.append(loss.item()) + del logits, loss, perturbed_seq, perturbed_seq_, p_mask, mask_indices + empty_cache_by_memory(threshold_gb=70) + + return sum(loss_acc) / len(loss_acc) + + @torch.no_grad() + def _eval_target_nll_ar(self, prefix, target): + prefix, target = prefix.unsqueeze(0), target.unsqueeze(0) # 1*l1, 1*l2 + assert self.log_type in ['ftb', 'btf'] + assert self.nll_type in ['ar_ftb', 'ar_btf'] + + if self.log_type == 'ftb': + prompt_index = torch.arange(prefix.shape[1] + target.shape[1], device=self.device) < prefix.shape[1] + else: + prompt_index = torch.arange(prefix.shape[1] + target.shape[1], device=self.device) >= prefix.shape[1] + + if self.log_type == 'ftb': + perturbed_ = target.repeat(target.shape[1], 1).clone().contiguous() # l2*l2 + else: + perturbed_ = prefix.repeat(prefix.shape[1], 1).clone().contiguous() # l1*l1 + + mask_index = torch.ones((perturbed_.shape[1], perturbed_.shape[1]), dtype=torch.bool) + if self.nll_type == 'ar_ftb': + mask_index = torch.triu(mask_index) + else: + mask_index = torch.tril(mask_index) + perturbed_[mask_index] = self.mask_id + if self.log_type == 'ftb': + perturbed_seq = torch.cat([prefix.repeat(perturbed_.shape[0], 1), perturbed_], dim=-1) + else: + perturbed_seq = torch.cat([perturbed_, target.repeat(perturbed_.shape[0], 1)], dim=-1) + + logits_ = [] + num = len(perturbed_seq) // self.batch_size if len(perturbed_seq) % self.batch_size == 0 else len(perturbed_seq) // self.batch_size + 1 + for i in range(num): + end = (i + 1) * self.batch_size if (i + 1) * self.batch_size < len(perturbed_seq) else len(perturbed_seq) + perturbed_seq_ = perturbed_seq[i * self.batch_size: end] + perturbed_seq_ = perturbed_seq_.to(self.device) + if len(perturbed_seq_.shape) == 1: + perturbed_seq_ = perturbed_seq_.unsqueeze(0) + logits = self.get_logits(perturbed_seq_, prompt_index) + logits_.append(logits.cpu()) + logits = torch.cat(logits_, dim=0) + + temp_index = torch.ones((perturbed_.shape[1], perturbed_.shape[1]), dtype=torch.bool) + if self.nll_type == 'ar_ftb': + temp_index = torch.triu(temp_index, diagonal=1) + else: + temp_index = torch.tril(temp_index, diagonal=-1) + mask_index[temp_index] = False + if self.log_type == 'ftb': + logits_index = torch.cat([torch.zeros((perturbed_.shape[1], prefix.shape[1]), dtype=torch.bool), mask_index], dim=-1) + else: + logits_index = torch.cat([mask_index, torch.zeros((perturbed_.shape[1], target.shape[1]), dtype=torch.bool)], dim=-1) + + if self.log_type == 'ftb': + loss = F.cross_entropy(logits[logits_index], target[0], reduction='sum').cpu().item() + else: + loss = F.cross_entropy(logits[logits_index], prefix[0], reduction='sum').cpu().item() + return loss + + def _encode_pair(self, context, continuation): + n_spaces = len(context) - len(context.rstrip()) + if n_spaces > 0: + continuation = context[-n_spaces:] + continuation + context = context[:-n_spaces] + + whole_enc = self.tokenizer.encode(context + continuation) + [ + self.tokenizer.eos_token_id + ] + context_enc = self.tokenizer.encode(context) + + context_enc_len = len(context_enc) + continuation_enc = whole_enc[context_enc_len:] + + return context_enc, continuation_enc + + def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]: + def _tokenize(e): + prefix, target = self._encode_pair(e["prefix"], e["target"]) + return { + "prefix_text": e["prefix"], + "target_text": e["target"], + "prefix": prefix, + "target": target, + } + + ds = [] + ds = [{"prefix": req.args[0], "target": req.args[1]} for req in requests] + ds = Dataset.from_list(ds) + ds = ds.map(_tokenize) + ds = ds.with_format("torch") + + out = [] + with torch.no_grad(): + for elem in tqdm(ds, desc="Computing likelihood..."): + prefix = elem["prefix"] + target = elem["target"] + + if self.nll_type == 'mc': + ll = -self._eval_target_nll_mc(prefix, target) + if self.log_type == 'union': + ll = ll / (len(target) + len(prefix)) + elif self.nll_type == 'ar_ftb' or self.nll_type == 'ar_btf': + ll = -self._eval_target_nll_ar(prefix, target) + else: + raise NotImplementedError(self.nll_type) + + is_target_greedy_dec = False + out.append((ll, 1.0 if is_target_greedy_dec else 0.0)) + return out + + def loglikelihood_rolling(self, requests: List[Instance]) -> List[float]: + raise NotImplementedError \ No newline at end of file diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/models/dummy.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/models/dummy.py new file mode 100644 index 0000000000000000000000000000000000000000..014ad49ee36f756acd0428340f945312d79590e8 --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/models/dummy.py @@ -0,0 +1,41 @@ +import random + +from tqdm import tqdm + +from lm_eval.api.model import LM +from lm_eval.api.registry import register_model + + +@register_model("dummy") +class DummyLM(LM): + def __init__(self) -> None: + super().__init__() + + @classmethod + def create_from_arg_string(cls, arg_string, additional_config=None): + return cls() + + def loglikelihood(self, requests, disable_tqdm: bool = False): + res = [] + + for _ in tqdm(requests, disable=disable_tqdm): + res.append((-random.random(), False)) + + return res + + def generate_until(self, requests, disable_tqdm: bool = False): + res = [] + + for request in tqdm(requests, disable=disable_tqdm): + res.append("lol") + assert request.arguments[0].strip() != "" + + return res + + def loglikelihood_rolling(self, requests, disable_tqdm: bool = False): + res = [] + + for _ in tqdm(requests, disable=disable_tqdm): + res.append(-random.random()) + + return res diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/models/hts_sampler.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/models/hts_sampler.py new file mode 100644 index 0000000000000000000000000000000000000000..550f443390f2fb98efbe3c050e97357170748146 --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/models/hts_sampler.py @@ -0,0 +1,256 @@ +import torch +import torch.nn.functional as F +import numpy as np +from .verifier import CodeVerifier +import logging +import re +import math + +logger = logging.getLogger(__name__) + +class HTSSampler: + def __init__(self, model, tokenizer, device="cuda"): + self.model = model + self.tokenizer = tokenizer + self.device = device + self.verifier = CodeVerifier(model, tokenizer, device) + + def _get_num_transfer_tokens(self, block_length, steps): + if steps == 0: return torch.tensor([], dtype=torch.int64) + base = block_length // steps + remainder = block_length % steps + num_transfer_tokens = torch.full((steps,), base, dtype=torch.int64) + num_transfer_tokens[:remainder] += 1 + return num_transfer_tokens + + def _sample_with_temperature(self, logits, temperature, top_k, top_p): + logits = logits.to(torch.float32) + orig_probs = torch.softmax(logits, dim=-1) + x0_p, _ = torch.max(orig_probs, dim=-1) + + if temperature > 0.0: + noise = torch.rand_like(logits, dtype=torch.float32) + gumbel_noise = -torch.log(-torch.log(noise + 1e-10) + 1e-10) + logits = logits / temperature + gumbel_noise + + if top_k is not None and top_k > 0: + indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, None] + logits[indices_to_remove] = -float('Inf') + + x0 = torch.argmax(logits, dim=-1) + return x0, x0_p + + def _safe_scalar(self, val): + if isinstance(val, torch.Tensor): + if val.numel() > 1: return val.mean().item() + return val.item() + return float(val) + + def _analyze_structure(self, text, task_type="code"): + score = 0.0 + stripped = text.strip() + if task_type == "code": + if len(stripped) < 5: return -0.1 + keywords = ["return", "print", "yield", "lambda", "class ", "def "] + if any(k in stripped for k in keywords): score += 0.05 + if ":" in stripped: score += 0.02 + if " " in text: score += 0.03 + elif task_type == "math": + if "\\boxed{" in stripped: score += 0.1 + if "The answer is" in stripped: score += 0.05 + return score + + def _chunked_forward(self, x, chunk_size=32, slice_indices=None): + total_batch = x.shape[0] + logits_list = [] + for i in range(0, total_batch, chunk_size): + end_idx = min(i + chunk_size, total_batch) + sub_x = x[i:end_idx] + with torch.no_grad(): + with torch.amp.autocast('cuda', dtype=torch.bfloat16): + outputs = self.model(sub_x, 'full') + sub_logits = outputs.logits + sub_logits = torch.cat([sub_logits[:, :1, :], sub_logits[:, :-1, :]], dim=1) + if slice_indices is not None: + s_start, s_end = slice_indices + sub_logits = sub_logits[:, s_start:s_end, :] + logits_list.append(sub_logits.detach().clone()) + return torch.cat(logits_list, dim=0) + + def _branch_and_resample(self, x, conf_scores, survivor_indices, target_width, mask_id, + prompt_length, resample_window=6, task_type="code"): + num_survivors = len(survivor_indices) + if num_survivors == 0: return x[:target_width].clone(), conf_scores[:target_width].clone() + + base_repeat = target_width // num_survivors + remainder = target_width % num_survivors + new_x_list, new_conf_list = [], [] + + for i in range(num_survivors): + count = base_repeat + (1 if i < remainder else 0) + if count == 0: continue + survivor_x = x[survivor_indices[i]] + survivor_conf = conf_scores[survivor_indices[i]] + + new_x_list.append(survivor_x.unsqueeze(0)) + new_conf_list.append(survivor_conf.unsqueeze(0)) + + if count > 1: + gen_part = survivor_x[prompt_length:] + gen_conf = survivor_conf[prompt_length:] + non_mask_indices = (gen_part != mask_id).nonzero(as_tuple=True)[0] + for _ in range(count - 1): + perturbed_x = survivor_x.clone() + perturbed_conf = survivor_conf.clone() + if len(non_mask_indices) > 0: + pool_size = min(resample_window * 2, len(non_mask_indices)) + current_token_confs = gen_conf[non_mask_indices] + _, candidate_pool = torch.topk(current_token_confs, k=pool_size, largest=False) + + num_to_perturb = min(resample_window, pool_size) + rand_indices = torch.randperm(pool_size, device=self.device)[:num_to_perturb] + selected_sub_indices = candidate_pool[rand_indices] + + target_idx_in_x = prompt_length + non_mask_indices[selected_sub_indices] + perturbed_x[target_idx_in_x] = mask_id + perturbed_conf[target_idx_in_x] = 0.0 + new_x_list.append(perturbed_x.unsqueeze(0)) + new_conf_list.append(perturbed_conf.unsqueeze(0)) + + return torch.cat(new_x_list, dim=0), torch.cat(new_conf_list, dim=0) + + @torch.no_grad() + def generate_hts(self, prompt_text, input_ids, problem_data=None, + initial_N=1, final_K=1, survivor_K=None, + prune_step_pct=0.0, reward_mode="confidence", + temperature=0.7, block_length=32, steps=64, gen_length=1024, + top_p=0.95, top_k=None, minimal_topk=1, threshold=0.9, + eos_id=151643, mask_id=151666, + hts_mode=False, hts_start_pct=0.1, hts_end_pct=0.6, decay_factor=1.2, + hts_survivor_k=4, task_type="code", until=None, pruning_interval=20): + + input_ids = input_ids.to(self.device) + prompt_length = input_ids.shape[1] + total_length = prompt_length + gen_length + + x = torch.full((initial_N, total_length), mask_id, dtype=torch.long, device=self.device) + x[:, :prompt_length] = input_ids.repeat(initial_N, 1) + conf_scores = torch.zeros((initial_N, total_length), dtype=torch.float32, device=self.device) + conf_scores[:, :prompt_length] = 1.0 + + schedule = self._get_num_transfer_tokens(gen_length, steps) + current_bsz = initial_N + schedule_map = {} + ts_start, tr_end = 0, 0 + + if hts_mode: + ts_start, tr_end = int(steps * hts_start_pct), int(steps * hts_end_pct) + else: + final_K_list = [final_K] if not isinstance(final_K, list) else final_K + prune_pct_list = [prune_step_pct] if not isinstance(prune_step_pct, list) else prune_step_pct + for pct, width in zip(prune_pct_list, final_K_list): + if pct > 0: schedule_map[int(steps * pct)] = width + + stats = { + "initial_n": initial_N, + "final_k": final_K if not isinstance(final_K, list) else final_K[-1], + "nfe": 0, + "svf_calls": 0, + "pruning_history": [], + "entropy_history": [], + "final_scores": [] + } + + next_allowed_pruning_step = ts_start + + for step in range(steps): + perform_pruning = False + num_parents_to_select = hts_survivor_k + + if hts_mode and ts_start <= step < tr_end and step >= next_allowed_pruning_step: + target_width = max(stats["final_k"], math.ceil(initial_N * (decay_factor ** -(step - ts_start)))) + if current_bsz > target_width: + perform_pruning = True + elif not hts_mode and step in schedule_map: + target_width = schedule_map[step] + num_parents_to_select = target_width + if current_bsz > target_width: + perform_pruning = True + + if perform_pruning: + stats["svf_calls"] += current_bsz + full_logits = self._chunked_forward(x[:current_bsz, :], slice_indices=(prompt_length, total_length)) + rough_ids = torch.argmax(full_logits, dim=-1) + rough_codes = self.tokenizer.batch_decode(rough_ids, skip_special_tokens=True) + + candidates = [] + for i in range(current_bsz): + s = self._safe_scalar(self.verifier.get_reward(prompt_text, rough_codes[i], mode=reward_mode, current_logits=full_logits[i], task_type=task_type)) + s += self._analyze_structure(rough_codes[i], task_type=task_type) + clean_text = rough_codes[i].strip().replace(" ", "").replace("\n", "") + content_key = hash(clean_text[:150] + clean_text[-150:]) if clean_text else i + candidates.append({'score': s, 'idx': i, 'key': content_key}) + + stats["pruning_history"].append({"step": step, "scores": [c['score'] for c in candidates]}) + candidates.sort(key=lambda c: c['score'], reverse=True) + + selected_indices, seen_keys = [], set() + for cand in candidates: + if len(selected_indices) >= num_parents_to_select: break + if cand['key'] not in seen_keys: + selected_indices.append(cand['idx']); seen_keys.add(cand['key']) + for cand in candidates: + if len(selected_indices) >= num_parents_to_select: break + if cand['idx'] not in selected_indices: selected_indices.append(cand['idx']) + + top_indices = torch.tensor(selected_indices, device=self.device) + x, conf_scores = self._branch_and_resample(x, conf_scores, top_indices, target_width, mask_id, prompt_length, task_type=task_type) + + current_bsz = target_width + next_allowed_pruning_step = step + pruning_interval + + active_mask = (x[:current_bsz, prompt_length:] == mask_id) + + stats["nfe"] += current_bsz + logits = self._chunked_forward(x[:current_bsz, :], slice_indices=(prompt_length, total_length)) + + with torch.no_grad(): + probs = torch.softmax(logits.float(), dim=-1) + entropy = -(probs * torch.log(probs + 1e-10)).sum(dim=-1).mean().item() + stats["entropy_history"].append(entropy) + + x0, x0_p = self._sample_with_temperature(logits, temperature, top_k, top_p) + num_transfer = schedule[step].item() + + confidence = torch.where(active_mask, x0_p, -torch.inf) + transfer_idx = torch.zeros_like(x0, dtype=torch.bool) + + for b in range(current_bsz): + k = min(num_transfer, active_mask[b].sum().item()) + if k <= 0: continue + high_conf_mask = (confidence[b] > threshold) + if high_conf_mask.sum() >= k: + transfer_idx[b] = high_conf_mask + else: + _, topk_ids = torch.topk(confidence[b], k=k) + transfer_idx[b, topk_ids] = True + + if transfer_idx.any(): + x[:current_bsz, prompt_length:][transfer_idx] = x0[transfer_idx] + conf_scores[:current_bsz, prompt_length:][transfer_idx] = x0_p[transfer_idx] + + final_codes = self.tokenizer.batch_decode(x[:current_bsz, prompt_length:], skip_special_tokens=True) + final_candidates = [] + for i, code in enumerate(final_codes): + txt = code.split(self.tokenizer.eos_token)[0] + if until: + for term in until: + if term in txt: txt = txt.split(term)[0] + s = self._safe_scalar(self.verifier.get_reward(prompt_text, txt, mode=reward_mode, task_type=task_type)) + final_candidates.append({'resp': txt, 'score': s}) + + final_candidates.sort(key=lambda c: c['score'], reverse=True) + stats["final_scores"] = [c['score'] for c in final_candidates] + stats["all_trajectories"] = [{"rank": i+1, "resp": c['resp'], "score": c['score']} for i, c in enumerate(final_candidates)] + + return [c['resp'] for c in final_candidates], stats \ No newline at end of file diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/models/huggingface.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/models/huggingface.py new file mode 100644 index 0000000000000000000000000000000000000000..f7e098a2e88d1c63276216a42b427ab28d77d835 --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/models/huggingface.py @@ -0,0 +1,1459 @@ +import copy +import logging +import os +from datetime import timedelta +from pathlib import Path +from typing import Dict, List, Literal, Optional, Tuple, Union + +import jinja2 +import torch +import torch.nn.functional as F +import transformers +from accelerate import ( + Accelerator, + InitProcessGroupKwargs, + find_executable_batch_size, +) +from accelerate.utils import get_max_memory +from huggingface_hub import HfApi +from packaging import version +from peft import PeftModel +from peft import __version__ as PEFT_VERSION +from tqdm import tqdm +from transformers.models.auto.modeling_auto import ( + MODEL_FOR_CAUSAL_LM_MAPPING_NAMES, + MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES, +) + +from lm_eval import utils +from lm_eval.api.instance import Instance +from lm_eval.api.model import TemplateLM +from lm_eval.api.registry import register_model +from lm_eval.models.utils import ( + Collator, + clear_torch_cache, + configure_pad_token, + get_dtype, + handle_stop_sequences, + pad_and_concat, + stop_sequences_criteria, +) + + +eval_logger = logging.getLogger(__name__) + + +@register_model("hf-auto", "hf", "huggingface") +class HFLM(TemplateLM): + """ + An abstracted Huggingface model class. Enables usage with both models of + `transformers.AutoModelForCausalLM` and `transformers.AutoModelForSeq2SeqLM` classes. + + Supports data-parallel multi-GPU with HF Accelerate. + """ + + AUTO_MODEL_CLASS = None + _DEFAULT_MAX_LENGTH = 2048 + + def __init__( + self, + pretrained: Union[str, transformers.PreTrainedModel], + backend: Literal["default", "causal", "seq2seq"] = "default", + # override whether the model should be treated as decoder-only (causal) or encoder-decoder (seq2seq) + revision: Optional[str] = "main", + subfolder: Optional[str] = None, + tokenizer: Optional[ + Union[ + str, + transformers.PreTrainedTokenizer, + transformers.PreTrainedTokenizerFast, + ] + ] = None, + truncation: Optional[bool] = False, + logits_cache: bool = True, + max_length: Optional[int] = None, + device: Optional[str] = "cuda", + dtype: Optional[Union[str, torch.dtype]] = "auto", + batch_size: Optional[Union[int, str]] = 1, + max_batch_size: Optional[int] = 64, + trust_remote_code: Optional[bool] = False, + use_fast_tokenizer: Optional[bool] = True, + add_bos_token: Optional[bool] = False, + prefix_token_id: Optional[int] = None, + # arguments used for splitting a model across GPUs naively. + # only used if `parallelize=True`. + parallelize: Optional[bool] = False, + max_memory_per_gpu: Optional[Union[int, str]] = None, + max_cpu_memory: Optional[Union[int, str]] = None, + offload_folder: Optional[Union[str, os.PathLike]] = "./offload", + # PEFT, delta weights and quantization options + peft: Optional[str] = None, + delta: Optional[str] = None, + autogptq: Optional[Union[bool, str]] = False, + gptqmodel: Optional[bool] = False, + gguf_file: Optional[str] = None, + **kwargs, + ) -> None: + super().__init__() + # optionally: take in an already-initialized transformers.PreTrainedModel + if not isinstance(pretrained, str): + eval_logger.warning( + "`pretrained` model kwarg is not of type `str`. Many other model arguments may be ignored. Please do not launch via accelerate or use `parallelize=True` if passing an existing model this way." + ) + assert not parallelize, ( + "`parallelize=True` is not compatible with passing pre-initialized model to `pretrained`" + ) + self._model = pretrained + self._device = self._model.device + self._config = self._model.config + gpus = 0 + + else: + assert isinstance(device, str) + assert isinstance(pretrained, str) + assert isinstance(batch_size, (int, str)) + + gpus = torch.cuda.device_count() + accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52)) + accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs]) + if accelerator.num_processes > 1: + self.accelerator = accelerator + + if "npu" in accelerator.device.type: + gpus = torch.npu.device_count() + + # using one process with no model parallelism + if not (parallelize or accelerator.num_processes > 1): + # use user-passed device + device_list = set( + ["cuda", "cpu"] + + [f"cuda:{i}" for i in range(gpus)] + + ["mps", "mps:0"] + + [f"npu:{i}" for i in range(gpus)] + ) + if device and device in device_list: + self._device = torch.device(device) + eval_logger.info(f"Using device '{device}'") + if device in ("mps", "mps:0") and version.parse( + torch.__version__ + ) < version.parse("2.1"): + raise RuntimeError( + f"mps requires torch >= 2.1. You have {torch.__version__}" + ) + else: + eval_logger.info("Device not specified") + eval_logger.info(f"Cuda Available? {torch.cuda.is_available()}") + self._device = ( + torch.device("cuda") + if torch.cuda.is_available() + else torch.device("cpu") + ) + else: # Parallelism managed by accelerate + if device != "cuda": + eval_logger.info( + f"Using `accelerate launch` or `parallelize=True`, device '{device}' will be overridden when placing model." + ) + # TODO: include in warning that `load_in_8bit` etc. affect this too + self._device = ( + self.accelerator.device + if hasattr(self, "accelerator") + else torch.device(device) + ) + + revision = str(revision) # cast to string if not already one + # TODO: update this to be less of a hack once subfolder is fixed in HF + revision = revision + ("/" + subfolder if subfolder is not None else "") + + self._get_config( + pretrained, + revision=revision, + trust_remote_code=trust_remote_code, + gguf_file=gguf_file, + ) + + # determine which of 'causal' and 'seq2seq' backends to use for HF models + self._get_backend( + config=self.config, backend=backend, trust_remote_code=trust_remote_code + ) + + # load tokenizer so we know tokenizer vocabulary size before loading model and PEFT + self._create_tokenizer( + pretrained, + tokenizer, + revision=revision, + trust_remote_code=trust_remote_code, + use_fast_tokenizer=use_fast_tokenizer, + gguf_file=gguf_file, + add_bos_token=add_bos_token, + ) + + # if we passed `pretrained` as a string, initialize our model now + if isinstance(pretrained, str): + self._create_model( + pretrained=pretrained, + revision=revision, + dtype=dtype, + trust_remote_code=trust_remote_code, + parallelize=parallelize, + gpus=gpus, + max_memory_per_gpu=max_memory_per_gpu, + max_cpu_memory=max_cpu_memory, + offload_folder=offload_folder, + peft=peft, + delta=delta, + autogptq=autogptq, + gptqmodel=gptqmodel, + gguf_file=gguf_file, + **kwargs, + ) + + # access self._model through self.model property outside this method + if isinstance(self.model, torch.nn.Module): + self.model.eval() + self.model.tie_weights() + + self.truncation = truncation + self.logits_cache = logits_cache + self.vocab_size = self.tokenizer.vocab_size + # select (or create) a pad token to use + self.tokenizer = configure_pad_token(self.tokenizer, model_config=self.config) + + self.add_bos_token = add_bos_token + if "gemma" in getattr(self.config, "model_type", ""): + self.add_bos_token = True + eval_logger.info( + f"Model type is '{self.config.model_type}', part of the Gemma family--a BOS token will be used as Gemma underperforms without it." + ) + + self._max_length = max_length + self.pretrained = pretrained + self.delta = delta + self.peft = peft + self.revision = revision + self.batch_schedule = 1 + self.batch_sizes = {} + self.max_batch_size = max_batch_size + + if str(batch_size).startswith("auto"): + batch_size = batch_size.split(":") + self.batch_size_per_gpu = batch_size[0] + self.batch_schedule = float(batch_size[1]) if len(batch_size) > 1 else 1 + else: + self.batch_size_per_gpu = int(batch_size) + + if isinstance(pretrained, str): + if gpus >= 1 or str(self.device) == "mps": + # TODO: can remove this whole snippet except in the mps case, perhaps? + if not (parallelize or autogptq or hasattr(self, "accelerator")): + # place model onto device requested manually, + # if not using HF Accelerate or device_map + # or any other option that preloads model onto device + try: + self.model.to(self.device) + except ValueError: + eval_logger.debug( + "Failed to place model onto specified device. This may be because the model is quantized via `bitsandbytes` or `device_map` is provided. If the desired GPU is being used, this message is safe to ignore." + ) + # multigpu data-parallel support when launched with accelerate + if gpus > 1: + if accelerator.num_processes > 1: + if parallelize: + eval_logger.warning( + "You are both using a HF Accelerate `device_map` (`--model_args parallelize=True`) and launching via `accelerate launch`. This will attempt to do model and data parallelism depending on the resources available." + ) + elif gpus > accelerator.num_processes: + eval_logger.warning( + "WARNING: The number of total system GPUs does not match the number of spawned processes. " + "If you would like to use data parallelism, please launch the script " + "with 'accelerate launch *script*'. " + f"Current run will proceed with {accelerator.num_processes} devices." + ) + if self.accelerator.is_local_main_process: + eval_logger.info( + f"Using {gpus} devices with data parallelism" + ) + + self._device = torch.device(f"{accelerator.device}") + self.accelerator = accelerator + + self._rank = self.accelerator.local_process_index + self._world_size = self.accelerator.num_processes + else: + # if we aren't launching via accelerate, ditch + self._rank = 0 + self._world_size = 1 + else: + # if a PreTrainedModel was passed into HFLM, we forgo distributed setup. + eval_logger.warning( + "Passed an already-initialized model through `pretrained`, assuming single-process call to evaluate() or custom distributed integration" + ) + self._rank = 0 + self._world_size = 1 + + self.custom_prefix_token_id = prefix_token_id + if prefix_token_id is not None: + eval_logger.info( + f"Loglikelihood prefix token id used in evaluation: {self.prefix_token_id}" + ) + + def _get_accelerate_args( + self, + parallelize: Optional[bool] = None, + device_map: Optional[str] = "auto", + max_memory_per_gpu: Optional[Union[int, str]] = None, + max_cpu_memory: Optional[Union[int, str]] = None, + offload_folder: Optional[str] = "./offload", + gpus: Optional[int] = None, + ) -> dict: + """Returns the kwargs needed to apply `accelerate` in `AutoModel.from_pretrained`.""" + num_local_processes = int(os.environ.get("LOCAL_WORLD_SIZE", 1)) + num_machines = int(os.environ.get("WORLD_SIZE", 0)) // num_local_processes + if ( + num_machines == 0 + and hasattr(self, "accelerator") + and self.accelerator is not None + ): + eval_logger.info( + "We are not in a distributed setting for accelerate. Setting model_parallel to False." + ) + parallelize = False + + if parallelize is None: + # If parallelism is unset by the user, we automatically assign model parallelism + # if enough extra GPUs are available + max_memory_all_gpus = get_max_memory() + # We just want gpu, not cpu, max memory + if "cpu" in max_memory_all_gpus: + del max_memory_all_gpus["cpu"] + parallelize = bool(num_local_processes < len(max_memory_all_gpus)) + eval_logger.info( + f"Setting model parallel to {parallelize} since " + f"the number of local processes is {num_local_processes} " + f"and the number of GPUs is {len(max_memory_all_gpus)}" + ) + + args = {} + if parallelize: # Model parallelism will be used + max_memory = {} + if max_memory_per_gpu is not None: # Using the provided memory requirements + max_memory_per_gpu_map = { + device_idx: max_memory_per_gpu for device_idx in range(gpus) + } + else: # Estimating the possible memory requirements + max_memory_all_gpus = get_max_memory() + if "cpu" in max_memory_all_gpus: + del max_memory_all_gpus["cpu"] + if not hasattr(self, "accelerator"): + max_memory_per_gpu_map = { + k: v for k, v in max_memory_all_gpus.items() + } + else: + # use only 1 / num_processes of the GPUs if we are running under accelerate launch + max_memory_per_gpu_map = { + k: v + for k, v in max_memory_all_gpus.items() + if k % num_local_processes + == (self.accelerator.process_index % num_local_processes) + } + args["max_memory"] = max_memory_per_gpu_map + args["device_map"] = "auto" if device_map is None else device_map + eval_logger.info( + f"Model parallel was set to True, setting max memory per GPU to {max_memory_per_gpu_map} and device map to {args.get('device_map')}" + ) + + if max_cpu_memory is not None: + max_memory["cpu"] = max_cpu_memory + + args["offload_folder"] = offload_folder + elif ( + device_map is None + ): # No model parallelism, we use the default provided device for our model + if hasattr(self, "accelerator"): + device_map = {"": f"{self.accelerator.device}"} + else: + device_map = {"": str(self.device)} + args["max_memory"] = None + args["device_map"] = device_map + eval_logger.info( + f"Model parallel was set to False, max memory was not set, and device map was set to {device_map}" + ) + else: + args["max_memory"] = None + args["device_map"] = None + eval_logger.info("Model parallel was set to False.") + + return args + + @property + def config(self): + # return the associated transformers.AutoConfig for the given pretrained model. + return self._config + + @property + def model(self): + # returns the model, unwrapping it if using Accelerate + if hasattr(self, "accelerator"): + return self.accelerator.unwrap_model(self._model) + else: + return self._model + + @property + def eot_token_id(self): + # we use EOT because end of *text* is more accurate for what we're doing than end of *sentence* + return self.tokenizer.eos_token_id + + @property + def prefix_token_id(self): + # it is used as prefix for loglikelihood + if self.custom_prefix_token_id is not None: + return self.custom_prefix_token_id + if self.tokenizer.bos_token_id is not None: + return self.tokenizer.bos_token_id + return self.tokenizer.eos_token_id + + @property + def max_length(self): + if self._max_length: # if max length manually set, return it + return self._max_length + seqlen_config_attrs = ("n_positions", "max_position_embeddings", "n_ctx") + for attr in seqlen_config_attrs: + if hasattr(self.model.config, attr): + return getattr(self.model.config, attr) + if hasattr(self.tokenizer, "model_max_length"): + if self.tokenizer.model_max_length == 1000000000000000019884624838656: + return self._DEFAULT_MAX_LENGTH + return self.tokenizer.model_max_length + return self._DEFAULT_MAX_LENGTH + + @property + def max_gen_toks(self) -> int: + return 256 + + @property + def batch_size(self): + return self.batch_size_per_gpu + + @property + def device(self): + return self._device + + @property + def rank(self): + return self._rank + + @property + def world_size(self): + return self._world_size + + @property + def tokenizer_name(self) -> str: + return self.tokenizer.name_or_path.replace("/", "__") + + def _get_backend( + self, + config: Union[transformers.PretrainedConfig, transformers.AutoConfig], + backend: Literal["default", "causal", "seq2seq"] = "default", + trust_remote_code: Optional[bool] = False, + ) -> None: + """ + Helper method during initialization. + Determines the backend ("causal" (decoder-only) or "seq2seq" (encoder-decoder)) model type to be used. + sets `self.AUTO_MODEL_CLASS` appropriately if not already set. + + **If not calling HFLM.__init__() or HFLM._get_backend() within a subclass of HFLM, + user must set `self.backend` to be either "causal" or "seq2seq" manually!** + """ + + assert backend in ["default", "causal", "seq2seq"] + + if backend != "default": + # if we've settled on non-default backend, use that manually + if backend == "causal": + self.backend = backend + elif backend == "seq2seq": + self.backend = backend + eval_logger.info( + f"Overrode HF model backend type, and using type '{self.backend}'" + ) + else: + # determine and use the default HF backend for this model, based on its config + metadata. + if ( + getattr(config, "model_type") + in MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES + ): + # first check if model type is listed under seq2seq models, since some + # models like MBart are listed in both seq2seq and causal mistakenly in HF transformers. + # these special cases should be treated as seq2seq models. + self.backend = "seq2seq" + eval_logger.debug(f"Using model type '{self.backend}'") + elif ( + getattr(self.config, "model_type") in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES + ): + self.backend = "causal" + eval_logger.debug(f"Using model type '{self.backend}'") + else: + if not trust_remote_code: + eval_logger.warning( + "HF model type is neither marked as CausalLM or Seq2SeqLM. \ + This is expected if your model requires `trust_remote_code=True` but may be an error otherwise." + "Setting backend to causal" + ) + # if model type is neither in HF transformers causal or seq2seq model registries + # then we default to assuming AutoModelForCausalLM + self.backend = "causal" + eval_logger.info( + f"Model type cannot be determined. Using default model type '{self.backend}'" + ) + + if self.AUTO_MODEL_CLASS is None: + if self.backend == "causal": + self.AUTO_MODEL_CLASS = transformers.AutoModelForCausalLM + elif self.backend == "seq2seq": + self.AUTO_MODEL_CLASS = transformers.AutoModelForSeq2SeqLM + + def _get_config( + self, + pretrained: str, + revision: str = "main", + trust_remote_code: bool = False, + gguf_file: Optional[str] = None, + ) -> None: + """Return the model config for HuggingFace models""" + self._config = transformers.AutoConfig.from_pretrained( + pretrained, + revision=revision, + trust_remote_code=trust_remote_code, + gguf_file=gguf_file, + ) + + def _create_model( + self, + pretrained: str, + revision: Optional[str] = "main", + dtype: Optional[Union[str, torch.dtype]] = "auto", + trust_remote_code: Optional[bool] = False, + # arguments used for splitting a model across GPUs naively. + # only used if `parallelize=True`. + # (accelerate naive PP (device_map) options) + parallelize: Optional[bool] = False, + gpus: Optional[int] = None, + max_memory_per_gpu: Optional[Union[int, str]] = None, + max_cpu_memory: Optional[Union[int, str]] = None, + offload_folder: Optional[str] = "./offload", + # PEFT, delta weights and quantization options + peft: Optional[str] = None, + delta: Optional[str] = None, + autogptq: Optional[Union[bool, str]] = False, + gptqmodel: Optional[bool] = False, + gguf_file: Optional[str] = None, + **kwargs, + ) -> None: + """ + Initializes an HF or HF-compatible PreTrainedModel from scratch + inside HFLM, using the kwargs passed into self.__init__(). + + Also handles functionality such as AutoGPTQ usage and PEFT wrapping. + + For future similar extensions to AutoGPTQ that are not core to HF's ecosystem, + (such as PyTorch models that are nearly, but not quite, fully mirroring + HF's public interface relied on in this HFLM class) + please consider subclassing HFLM and overriding this and other methods as needed. + """ + + model_kwargs = kwargs if kwargs else {} + + model_kwargs.update( + self._get_accelerate_args( + parallelize=parallelize, + device_map=kwargs.get("device_map", None), + max_memory_per_gpu=max_memory_per_gpu, + max_cpu_memory=max_cpu_memory, + offload_folder=offload_folder, + gpus=gpus, + ) + ) + + if not autogptq and not gptqmodel: + if model_kwargs.get("load_in_4bit", None): + assert transformers.__version__ >= "4.30.0", ( + "load_in_4bit requires transformers >= 4.30.0" + ) + if transformers.__version__ >= "4.30.0": + if model_kwargs.get("load_in_4bit", None): + if model_kwargs.get("bnb_4bit_compute_dtype", None): + model_kwargs["bnb_4bit_compute_dtype"] = get_dtype( + model_kwargs["bnb_4bit_compute_dtype"] + ) + + self._model = self.AUTO_MODEL_CLASS.from_pretrained( + pretrained, + revision=revision, + torch_dtype=get_dtype(dtype), + trust_remote_code=trust_remote_code, + gguf_file=gguf_file, + **model_kwargs, + ) + else: + if autogptq and gptqmodel: + raise ValueError( + "Cannot use both 'autogptq' and 'gptqmodel' options at the same time." + ) + + if autogptq: + try: + from auto_gptq import AutoGPTQForCausalLM + except ModuleNotFoundError as exception: + raise type(exception)( + "Tried to load auto_gptq, but auto-gptq is not installed ", + "please install auto-gptq via pip install lm-eval[gptq] or pip install -e .[gptq]", + ) + + self._model = AutoGPTQForCausalLM.from_quantized( + pretrained, + trust_remote_code=trust_remote_code, + model_basename=None if autogptq is True else Path(autogptq).stem, + use_safetensors=True + if autogptq is True + else autogptq.endswith(".safetensors"), + **model_kwargs, + ) + + if gptqmodel: + try: + from gptqmodel import GPTQModel + except ModuleNotFoundError as exception: + raise type(exception)( + "Tried to load gptqmodel, but gptqmodel is not installed ", + "please install gptqmodel via `pip install gptqmodel --no-build-isolation` or `pip install lm-eval[gptqmodel] --no-build-isolation`", + ) + + self._model = GPTQModel.from_quantized( + pretrained, trust_remote_code=trust_remote_code, **model_kwargs + ) + + if peft and delta: + raise ValueError( + "Cannot use both 'peft' and 'delta' options at the same time." + ) + + if peft: + if model_kwargs.get("load_in_4bit", None): + if version.parse(PEFT_VERSION) < version.parse("0.4.0"): + raise AssertionError("load_in_4bit requires peft >= 0.4.0") + if self._model.config.vocab_size != len(self.tokenizer): + # resize model for LoRAs with added tokens + eval_logger.info( + f"Model config indicates vocab_size='{self._model.config.vocab_size}', but found tokenizer with vocab size '{len(self.tokenizer)}'. Resizing model embedding layer..." + ) + self._model.resize_token_embeddings(len(self.tokenizer)) + self._model = PeftModel.from_pretrained( + self._model, peft, revision=revision + ) + elif delta: + if autogptq: + eval_logger.warning( + "Delta weights might trigger unexpected behavior when used with AutoGPTQ." + ) + _model_delta = self.AUTO_MODEL_CLASS.from_pretrained( + delta, + revision=revision, + torch_dtype=get_dtype(dtype), + trust_remote_code=trust_remote_code, + **model_kwargs, + ) + for name, param in self._model.state_dict().items(): + try: + param.data += _model_delta.state_dict()[name] + except KeyError: + raise KeyError(f"Delta model is missing weights for layer: {name}") + except Exception as e: + raise RuntimeError( + f"Failed to add delta weights to layer {name}. Error: {e}" + ) + + del _model_delta + + return None + + def _create_tokenizer( + self, + pretrained: Union[str, transformers.PreTrainedModel], + tokenizer: Optional[ + Union[ + str, + transformers.PreTrainedTokenizer, + transformers.PreTrainedTokenizerFast, + ] + ], + revision: Optional[str] = "main", + trust_remote_code: Optional[bool] = False, + use_fast_tokenizer: Optional[bool] = True, + gguf_file: Optional[str] = None, + add_bos_token: Optional[bool] = False, + ) -> None: + """ + Helper method during initialization. + + Create a tokenizer object corresponding to the correct + tokenizer for value of `pretrained`, or use the pre-initialized tokenizer passed. + """ + kwargs = { + "revision": revision, + "trust_remote_code": trust_remote_code, + } + + # gguf format embeds tokenizer and is not compatible with hf tokenizer `use_fast` param + if gguf_file is not None: + kwargs["gguf_file"] = gguf_file + else: + kwargs["use_fast"] = use_fast_tokenizer + + if add_bos_token: + kwargs["add_bos_token"] = True + + if tokenizer: + if isinstance(tokenizer, str): + self.tokenizer = transformers.AutoTokenizer.from_pretrained( + tokenizer, **kwargs + ) + else: + assert isinstance( + tokenizer, transformers.PreTrainedTokenizer + ) or isinstance(tokenizer, transformers.PreTrainedTokenizerFast) + self.tokenizer = tokenizer + else: + # Get tokenizer based on 'pretrained' + if isinstance(pretrained, str): + model_name = pretrained + else: + # get the HF hub name via accessor on model + model_name = self.model.name_or_path + self.tokenizer = transformers.AutoTokenizer.from_pretrained( + model_name, **kwargs + ) + return None + + def _detect_batch_size(self, requests=None, pos: int = 0): + if requests: + _, context_enc, continuation_enc = requests[pos] + max_length = len( + (context_enc + continuation_enc)[-(self.max_length + 1) :][:-1] + ) + max_context_enc = len(context_enc[-(self.max_length + 1) :]) + max_cont_enc = len(continuation_enc[-(self.max_length + 1) :]) + else: + max_length = self.max_length + max_context_enc = max_length + max_cont_enc = max_length + + # if OOM, then halves batch_size and tries again + @find_executable_batch_size(starting_batch_size=self.max_batch_size) + def forward_batch(batch_size): + if self.backend == "seq2seq": + length = max(max_context_enc, max_cont_enc) + batched_conts = torch.ones( + (batch_size, length), device=self.device + ).long() + test_batch = torch.ones((batch_size, length), device=self.device).long() + call_kwargs = { + "attn_mask": test_batch, + "labels": batched_conts, + } + else: + call_kwargs = {} + test_batch = torch.ones( + (batch_size, max_length), device=self.device + ).long() + for _ in range(5): + out = F.log_softmax(self._model_call(test_batch, **call_kwargs), dim=-1) # noqa: F841 + + return batch_size + + try: + batch_size = forward_batch() + except RuntimeError as e: + if "No executable batch size found" in str(e): + batch_size = 1 + else: + raise + + if self.world_size > 1: + # if multi-GPU, always take minimum over all selected batch sizes + max_rnk_bs = torch.tensor([batch_size], device=self.device) + gathered = ( + self.accelerator.gather(max_rnk_bs).cpu().detach().numpy().tolist() + ) + batch_size = min(gathered) + clear_torch_cache() + return batch_size + + clear_torch_cache() + return batch_size + + def tok_encode( + self, string: str, left_truncate_len=None, add_special_tokens=None + ) -> List[int]: + """ """ + # default for None - empty dict, use predefined tokenizer param + # used for all models except for CausalLM or predefined value + special_tokens_kwargs = {} + + # by default for CausalLM - false or self.add_bos_token is set + if add_special_tokens is None: + if self.backend == "causal": + special_tokens_kwargs = { + "add_special_tokens": False or self.add_bos_token + } + # otherwise the method explicitly defines the value + else: + special_tokens_kwargs = {"add_special_tokens": add_special_tokens} + + encoding = self.tokenizer.encode(string, **special_tokens_kwargs) + + # left-truncate the encoded context to be at most `left_truncate_len` tokens long + if left_truncate_len: + encoding = encoding[-left_truncate_len:] + + return encoding + + def tok_batch_encode( + self, + strings: List[str], + padding_side: str = "left", + left_truncate_len: int = None, + truncation: bool = False, + ) -> Tuple[torch.Tensor, torch.Tensor]: + # encode a batch of strings. converts to tensors and pads automatically, unlike tok_encode. + old_padding_side = self.tokenizer.padding_side + self.tokenizer.padding_side = padding_side + + add_special_tokens = {} + if self.backend == "causal": + add_special_tokens = {"add_special_tokens": False or self.add_bos_token} + + encoding = self.tokenizer( + strings, + truncation=truncation, + padding="longest", + return_tensors="pt", + **add_special_tokens, + ) + if left_truncate_len: + original_lengths = encoding["input_ids"].size(1) + if original_lengths > left_truncate_len: + eval_logger.warn( + f"Left truncation applied. Original sequence length was {original_lengths}, " + f"truncating to last {left_truncate_len} tokens. Some content will be lost.", + ) + encoding["input_ids"] = encoding["input_ids"][:, -left_truncate_len:] + encoding["attention_mask"] = encoding["attention_mask"][ + :, -left_truncate_len: + ] + self.tokenizer.padding_side = old_padding_side + + return encoding["input_ids"], encoding["attention_mask"] + + def tok_decode(self, tokens, skip_special_tokens=True): + return self.tokenizer.decode(tokens, skip_special_tokens=skip_special_tokens) + + def _model_call(self, inps, attn_mask=None, labels=None): + """ + :param inps: torch.Tensor + A torch tensor of shape [batch, (sequence_ctx + sequence_cont)] or of shape + [batch, sequence_ctx]. the size of sequence may vary from call to call + :param attn_mask: torch.Tensor, optional + A torch tensor of shape [batch, (sequence_ctx + sequence_cont)]. Only passed + (and must be passed) if self.AUTO_MODEL_CLASS is transformers.AutoModelForSeq2SeqLM + :param labels: torch.Tensor, optional + A torch tensor of shape [batch, (sequence_ctx + sequence_cont)]. Only passed + (and must be passed) if self.AUTO_MODEL_CLASS is transformers.AutoModelForSeq2SeqLM + :return + A torch tensor of shape [batch, sequence, vocab] with the + logits returned from the model's decoder + """ + with torch.no_grad(): + if attn_mask is not None or labels is not None: + assert attn_mask is not None and labels is not None + assert self.AUTO_MODEL_CLASS == transformers.AutoModelForSeq2SeqLM + return self.model( + input_ids=inps, attention_mask=attn_mask, labels=labels + ).logits + else: + assert self.AUTO_MODEL_CLASS == transformers.AutoModelForCausalLM + return self.model(inps).logits + + def _model_generate(self, context, max_length, stop, **generation_kwargs): + # temperature = 0.0 if not set + # if do_sample is false and temp==0.0: + # remove temperature, as do_sample=False takes care of this + # and we don't want a warning from HF + generation_kwargs["temperature"] = generation_kwargs.get("temperature", 0.0) + do_sample = generation_kwargs.get("do_sample", None) + + # The temperature has to be a strictly positive float -- if it is 0.0, use greedy decoding strategies + if generation_kwargs.get("temperature") == 0.0 and do_sample is None: + generation_kwargs["do_sample"] = do_sample = False + + if do_sample is False and generation_kwargs.get("temperature") == 0.0: + generation_kwargs.pop("temperature") + # build stopping criteria + stopping_criteria = stop_sequences_criteria( + self.tokenizer, stop, context.shape[1], context.shape[0] + ) + return self.model.generate( + input_ids=context, + max_length=max_length, + stopping_criteria=stopping_criteria, + pad_token_id=self.tokenizer.pad_token_id, + use_cache=True, + **generation_kwargs, + ) + + def _select_cont_toks( + self, logits: torch.Tensor, contlen: int = None, inplen: int = None + ) -> torch.Tensor: + if self.backend == "causal": + assert contlen and inplen, ( + "Must pass input len and cont. len to select scored logits for causal LM" + ) + # discard right-padding. + # also discard the input/context tokens. we'll only score continuations. + logits = logits[inplen - contlen : inplen] + elif self.backend == "seq2seq": + assert contlen and not inplen, ( + "Selecting scored logits for Seq2SeqLM requires only cont. len" + ) + # only discard right-padding. + # the logits input to this fn only contain decoder-side tokens. + logits = logits[:contlen] + + return logits + + def loglikelihood_rolling( + self, requests: List[Instance], disable_tqdm: bool = False + ) -> List[float]: + adaptive_batch_size = None + if self.batch_size == "auto": + # using rolling window with maximum context + print("Passed argument batch_size = auto. Detecting largest batch size") + batch_size = self._detect_batch_size() + print(f"Determined Largest batch size: {batch_size}") + adaptive_batch_size = batch_size + + # First, collect all windows from all requests + all_windows = [] # List of (request_idx, window) tuples + request_window_counts = [] # Track number of windows per request + + for req_idx, (string,) in enumerate( + tqdm( + [req.args for req in requests], + disable=(disable_tqdm or (self.rank != 0)), + ) + ): + rolling_token_windows: List[Tuple[List[int], List[int]]] = list( + map( + utils.make_disjoint_window, + utils.get_rolling_token_windows( + token_list=self.tok_encode(string), + prefix_token=self.prefix_token_id, + max_seq_len=self.max_length, + context_len=1, + ), + ) + ) + + # TODO: Right now, we pass single EOT token to the Encoder and the full context to the decoder, in seq2seq case + windows = [(None,) + x for x in rolling_token_windows] + + # Store windows with their request index + all_windows.extend((req_idx, window) for window in windows) + request_window_counts.append(len(windows)) + + # Handle distributed case padding + pad_amnt = 0 + if self.world_size > 1: + mytensor = torch.tensor(len(all_windows), device=self.device) + gathered = self.accelerator.gather(mytensor).cpu().detach().numpy().tolist() + pad_amnt = max(gathered) - gathered[self.rank] + if pad_amnt > 0: + all_windows += pad_amnt * [all_windows[0]] + + all_nlls = [] + batch_size = adaptive_batch_size or self.batch_size + for i in range(0, len(all_windows), batch_size): + batch = all_windows[i : i + batch_size] + # Extract just the windows for processing, keeping track of request indices + batch_indices, batch_windows = zip(*batch) + + batch_nlls = self._loglikelihood_tokens( + requests=batch_windows, + disable_tqdm=False, + override_bs=len(batch_windows), + ) + # Store results with their request indices + all_nlls.extend(zip(batch_indices, batch_nlls)) + + # Remove padding if necessary + if (self.world_size > 1) and (pad_amnt > 0): + all_nlls = all_nlls[:-pad_amnt] + + # Reconstruct per-request loglikelihoods + loglikelihoods = [] + current_idx = 0 + for window_count in request_window_counts: + # Get all nlls for this request + request_nlls = all_nlls[current_idx : current_idx + window_count] + # Sum up the nlls for this request (discarding is_greedy) + request_total = sum(nll[0] for _, nll in request_nlls) + loglikelihoods.append(request_total) + current_idx += window_count + + string = requests[len(loglikelihoods) - 1].args[0] + self.cache_hook.add_partial( + "loglikelihood_rolling", (string,), request_total + ) + + return loglikelihoods + + def _batch_scheduler(self, pos, n_reordered_requests): + sched = pos // int(len(n_reordered_requests) / self.batch_schedule) + if sched in self.batch_sizes: + return self.batch_sizes[sched] + if (len(self.batch_sizes) > 1) and ( + self.batch_sizes[sched - 1] == self.max_batch_size + ): + # if previous batch size is already maximal, skip recomputation + self.batch_sizes[sched] = self.max_batch_size + return self.batch_sizes[sched] + print( + f"Passed argument batch_size = auto:{self.batch_schedule}. Detecting largest batch size" + ) + self.batch_sizes[sched] = self._detect_batch_size(n_reordered_requests, pos) + print(f"Determined largest batch size: {self.batch_sizes[sched]}") + return self.batch_sizes[sched] + + def _loglikelihood_tokens( + self, + requests: List[Tuple[Tuple[str, str], List[int], List[int]]], + disable_tqdm: bool = False, + override_bs: int = None, + ) -> List[Tuple[float, bool]]: + # TODO: implement some kind of efficient-request-middleware that lumps together requests with the same context + res = [] + + def _collate(req: Tuple[Tuple[str, str], List[int], List[int]]): + """Defines the key for the sorted method""" + # the negative sign on len(toks) sorts descending - this has a few advantages: + # - time estimates will always be over not underestimates, which is more useful for planning + # - to know the size of a batch when going through the list, you know the first one is always the batch + # padded context length. this is useful to simplify the batching logic and more importantly to make + # automatic adaptive batches much much easier to implement + # - any OOMs will happen right away rather than near the end + + toks = req[1] + req[2] + return -len(toks), tuple(toks) + + def _lookup_one_token_cont(req: Tuple[Tuple[str, str], List[int], List[int]]): + """Defines the key to group and lookup one-token continuations""" + # Use with group_by="contexts" (optional)" + # allows for the creation of a lookup, so we can reuse logits in case of one-token continuations. + # speeds up some multiple-choice tasks proportionally to the number of choices. + # groups requests by context+continuation[:-1] and infer on one request/group. + return req[-2] + req[-1][:-1] + + re_ord = Collator( + requests, + sort_fn=_collate, + group_by="contexts" + if self.backend == "causal" and self.logits_cache + else None, + group_fn=_lookup_one_token_cont, + ) + + # automatic (variable) batch size detection for vectorization + # pull longest context sample from request + n_reordered_requests = len(re_ord) + batch_size = ( + self.batch_size + if self.batch_size != "auto" + else override_bs + if override_bs is not None + else 0 + ) + batch_fn = ( + self._batch_scheduler + if self.batch_size == "auto" + and n_reordered_requests > 0 + and not override_bs + else None + ) + + chunks = re_ord.get_batched(n=batch_size, batch_fn=batch_fn) + pbar = tqdm( + total=len(requests), + disable=(disable_tqdm or (self.rank != 0)), + desc="Running loglikelihood requests", + ) + for chunk in chunks: + inps = [] + cont_toks_list = [] + inplens = [] + + conts = [] + encoder_attns = [] + + padding_len_inp = None + padding_len_cont = None + # because vectorizing is annoying, we first convert each (context, continuation) pair to padded + # tensors, then we pack them together into a batch, call the model, and then pick it all apart + # again because vectorizing is annoying + + for _, context_enc, continuation_enc in chunk: + # sanity check + assert len(context_enc) > 0 + assert len(continuation_enc) > 0 + assert len(continuation_enc) <= self.max_length + + # how this all works (illustrated on a causal decoder-only setup): + # CTX CONT + # inp 0 1 2 3|4 5 6 7 8 9 <- last token is deleted by inp[:, :-1] + # model \ \ + # logits 1 2 3|4 5 6 7 8 9 <- the ctx half gets tossed out by the + # cont_toks 4 5 6 7 8 9 [:, -len(continuation_enc):, :self.vocab_size] slice + + # when too long to fit in context, truncate from the left + if self.backend == "causal": + total_length = len(context_enc) + len(continuation_enc) + if total_length > self.max_length + 1: + eval_logger.warn( + f"Combined length of context ({len(context_enc)}) and continuation ({len(continuation_enc)}) " + f"exceeds model's maximum length ({self.max_length}). " + f"Truncating {total_length - self.max_length + 1} tokens from the left." + ) + inp = torch.tensor( + (context_enc + continuation_enc)[-(self.max_length + 1) :][:-1], + dtype=torch.long, + device=self.device, + ) + (inplen,) = inp.shape + elif self.backend == "seq2seq": + inp = torch.tensor( + (context_enc)[-self.max_length :], + dtype=torch.long, + device=self.device, + ) + (inplen,) = inp.shape + + # build encoder attn masks + encoder_attns.append(torch.ones_like(inp)) + + cont = torch.tensor( + (continuation_enc)[-self.max_length :], + # TODO: left-shift these? + # TODO: our code assumes we never end up truncating conts for either model type + dtype=torch.long, + device=self.device, + ) + (contlen,) = cont.shape + + conts.append(cont) + + padding_len_cont = ( + max(padding_len_cont, contlen) + if padding_len_cont is not None + else contlen + ) + + padding_len_inp = ( + max(padding_len_inp, inplen) + if padding_len_inp is not None + else inplen + ) + + inps.append(inp) # [1, inp_length] + cont_toks_list.append(continuation_enc) + inplens.append(inplen) + + # create encoder attn mask and batched conts, if seq2seq + call_kwargs = {} + if self.backend == "causal": + batched_inps = pad_and_concat( + padding_len_inp, inps, padding_side="right" + ) # [batch, padding_len_inp] + elif self.backend == "seq2seq": + # TODO: left-pad encoder inps and mask? + batched_inps = pad_and_concat( + padding_len_inp, inps + ) # [batch, padding_len_inp] + batched_conts = pad_and_concat( + padding_len_cont, conts + ) # [batch, padding_len_cont] + batched_encoder_mask = pad_and_concat( + padding_len_inp, encoder_attns + ) # [batch, padding_len_inp] + call_kwargs = { + "attn_mask": batched_encoder_mask, + "labels": batched_conts, + } + + multi_logits = F.log_softmax( + self._model_call(batched_inps, **call_kwargs), dim=-1 + ) # [batch, padding_length (inp or cont), vocab] + + for (request_str, ctx_tokens, _), logits, inplen, cont_toks in zip( + chunk, multi_logits, inplens, cont_toks_list + ): + # Slice to original seq length + contlen = len(cont_toks) + # take only logits in the continuation + # (discard context toks if decoder-only ; discard right-padding) + # also discards + checks for "virtual tokens" in the causal LM's input window + # from prompt/prefix tuning tokens, if applicable + ctx_len = ( + inplen + (logits.shape[0] - padding_len_inp) + if self.backend == "causal" + else None + ) + logits = self._select_cont_toks(logits, contlen=contlen, inplen=ctx_len) + logits = logits.unsqueeze(0) # [1, seq, vocab] + + # Check if per-token argmax is exactly equal to continuation + greedy_tokens = logits.argmax(dim=-1) + + # check for one-token continuation cache hits. + # noop in case group_by != "contexts" or no cache hit and returns the + # original args. Otherwise, expands the logits batch dimension and yields each + # batch along with matching continuation tokens and prompt strings. + # logits -> [1, seq, vocab] + for request_str, cont_toks, logits in re_ord.get_cache( + req_str=request_str, + cxt_toks=ctx_tokens, + cont_toks=cont_toks, + logits=logits, + ): + cont_toks = torch.tensor( + cont_toks, dtype=torch.long, device=self.device + ).unsqueeze(0) # [1, seq] + max_equal = (greedy_tokens == cont_toks).all() + + # Obtain log-probs at the corresponding continuation token indices + # last_token_slice = logits[:, -1, :].squeeze(0).tolist() + logits = torch.gather(logits, 2, cont_toks.unsqueeze(-1)).squeeze( + -1 + ) # [1, seq] + + # Answer: (log prob, is-exact-match) + answer = (float(logits.sum()), bool(max_equal)) + + res.append(answer) + + if request_str is not None: + # special case: loglikelihood_rolling produces a number of loglikelihood requests + # all with cache key None. instead do add_partial on the per-example level + # in the loglikelihood_rolling() function for those. + self.cache_hook.add_partial( + "loglikelihood", request_str, answer + ) + pbar.update(1) + + pbar.close() + + return re_ord.get_original(res) + + def generate_until( + self, requests: List[Instance], disable_tqdm: bool = False + ) -> List[str]: + res = [] + + def _collate(req: Tuple[str, dict]): + """Defines the key for the sorted method""" + # the negative sign on len(toks) sorts descending - this has a few advantages: + # - time estimates will always be over not underestimates, which is more useful for planning + # - to know the size of a batch when going through the list, you know the first one is always the batch + # padded context length. this is useful to simplify the batching logic and more importantly to make + # automatic adaptive batches much much easier to implement + # - any OOMs will happen right away rather than near the end + toks = self.tok_encode(req[0]) + return -len(toks), req[0] + + pbar = tqdm( + total=len(requests), + disable=(disable_tqdm or (self.rank != 0)), + desc="Running generate_until requests", + ) + adaptive_batch_size = None + if self.batch_size == "auto": + # using rolling window with maximum context + print("Passed argument batch_size = auto. Detecting largest batch size") + batch_size = self._detect_batch_size() + print(f"Determined Largest batch size: {batch_size}") + adaptive_batch_size = batch_size + # for each different set of kwargs, we execute all requests, by batch. + batch_size = ( + self.batch_size + if self.batch_size != "auto" + else adaptive_batch_size + if adaptive_batch_size is not None + else 0 + ) + batch_fn = ( + self._batch_scheduler + if self.batch_size == "auto" and not adaptive_batch_size + else None + ) + + # we group requests by their generation_kwargs, + # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling + # in the same batch. + # group_fn=lambda x: x[1] -> x=(context, gen_kwargs) + re_ords = Collator( + [reg.args for reg in requests], + sort_fn=_collate, + group_by="gen_kwargs", + group_fn=lambda x: x[1], + ) + chunks = re_ords.get_batched(n=batch_size, batch_fn=batch_fn) + eos = self.tok_decode(self.eot_token_id, skip_special_tokens=False) + for chunk in chunks: + contexts, all_gen_kwargs = zip(*chunk) + # we assume all gen kwargs in the batch are the same + # this is safe to assume because the `grouper` object ensures it. + gen_kwargs = all_gen_kwargs[0] + # unpack our keyword arguments. + if isinstance(gen_kwargs, dict): + kwargs = copy.deepcopy(gen_kwargs) # edge case for repeats > 1 + # add EOS token to stop sequences + until = handle_stop_sequences(kwargs.pop("until", None), eos=eos) + else: + raise ValueError( + f"Expected `kwargs` to be of type `dict` but got {type(gen_kwargs)}" + ) + if "max_gen_toks" in kwargs.keys(): + max_gen_toks = kwargs.pop("max_gen_toks") + else: + max_gen_toks = self.max_gen_toks + + # set the max length in tokens of inputs ("context_enc") + if self.backend == "causal": + # max len for inputs = max length, minus room to generate the max new tokens + max_ctx_len = self.max_length - max_gen_toks + assert max_ctx_len > 0, ( + f"Invalid configuration: requested max tokens to generate ({max_gen_toks}) must be less than model's maximum sequence length ({self.max_length})." + ) + elif self.backend == "seq2seq": + # max len for inputs = encoder's whole max_length + max_ctx_len = self.max_length + + # encode, pad, and truncate contexts for this batch + context_enc, attn_masks = self.tok_batch_encode( + contexts, + left_truncate_len=max_ctx_len, + truncation=self.truncation, + ) + context_enc = context_enc.to(self.device) + attn_masks = attn_masks.to(self.device) + + if "max_length" not in kwargs: + kwargs["max_length"] = context_enc.shape[1] + max_gen_toks + + # perform batched generation + cont = self._model_generate( + context=context_enc, + attention_mask=attn_masks, + stop=until, + **kwargs, + ) + + cont_toks_list = cont.tolist() + for cont_toks, context in zip(cont_toks_list, contexts): + # discard context + left-padding toks if using causal decoder-only LM + if self.backend == "causal": + cont_toks = cont_toks[context_enc.shape[1] :] + + s = self.tok_decode(cont_toks) + + # use secondary stop seqs to cut off should-have-been-stopped content post-hoc + for term in until: + if len(term) > 0: + # ignore '' separator, + # for seq2seq case where self.tok_decode(self.eot_token_id) = '' + s = s.split(term)[0] + + res.append(s) + + self.cache_hook.add_partial("generate_until", (context, gen_kwargs), s) + pbar.update(1) + # reorder this group of results back to original unsorted form + res = re_ords.get_original(res) + + pbar.close() + + return res + + def apply_chat_template( + self, chat_history: List[Dict[str, str]], add_generation_prompt: bool = True + ) -> str: + """ + Method to apply a chat template to a list of chat history between user and model. + """ + try: + chat_templated = self.tokenizer.apply_chat_template( + chat_history, + tokenize=False, + add_generation_prompt=add_generation_prompt, + continue_final_message=not add_generation_prompt, + ) + except jinja2.exceptions.TemplateError: + eval_logger.warning( + "Failed to apply chat template. removing the system role in chat history." + ) + chat_history = [msg for msg in chat_history if msg["role"] != "system"] + chat_templated = self.tokenizer.apply_chat_template( + chat_history, + tokenize=False, + add_generation_prompt=add_generation_prompt, + continue_final_message=not add_generation_prompt, + ) + + return chat_templated + + def get_model_info(self) -> dict: + """ + Method to get Hugging Face model information for experiment reproducibility. + """ + + def get_model_num_params(model) -> int: + if hasattr(model, "num_parameters"): + return model.num_parameters() + if hasattr(model, "parameters"): + return sum(p.numel() for p in model.parameters()) + else: + return -1 + + def get_model_dtype(model) -> str: + if hasattr(model, "dtype"): + return model.dtype + else: + return "" + + def get_model_sha(pretrained: str, revision: str) -> str: + try: + model_info = HfApi().model_info(repo_id=pretrained, revision=revision) + return model_info.sha + except Exception as e: + eval_logger.debug( + f"Failed to get model SHA for {pretrained} at revision {revision}. Error: {e}" + ) + return "" + + model_info = { + "model_num_parameters": get_model_num_params(self._model), + "model_dtype": get_model_dtype(self._model), + "model_revision": self.revision, + "model_sha": get_model_sha(self.pretrained, self.revision), + } + if self.peft: + model_info["peft_sha"] = get_model_sha(self.peft, self.revision) + if self.delta: + model_info["delta_sha"] = get_model_sha(self.delta, self.revision) + return model_info diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/models/utils.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/models/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..2878de6ea500b1e4f6723c9ef63f7d6b90be5711 --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/models/utils.py @@ -0,0 +1,731 @@ +import collections +import fnmatch +import gc +import itertools +import logging +import time +from functools import wraps +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Iterable, + Iterator, + List, + Literal, + Optional, + Tuple, + Type, + Union, +) + +import torch +import transformers + + +eval_logger = logging.getLogger(__name__) + + +if TYPE_CHECKING: + from transformers import PreTrainedTokenizerBase + from transformers.configuration_utils import PretrainedConfig + + +def chunks(iter, n: int = 0, fn=None): + """ + Divides an iterable into chunks of specified size or based on a given function. + Useful for batching + + Parameters: + - iter: The input iterable to be divided into chunks. + - n: An integer representing the size of each chunk. Default is 0. + - fn: A function that takes the current index and the iterable as arguments and returns the size of the chunk. Default is None. + + Returns: + An iterator that yields chunks of the input iterable. + + Example usage: + ``` + data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + for chunk in chunks(data, 3): + print(chunk) + ``` + Output: + ``` + [1, 2, 3] + [4, 5, 6] + [7, 8, 9] + [10] + ``` + """ + arr = [] + for i, x in enumerate(iter): + arr.append(x) + if len(arr) == (fn(i, iter) if fn else n): + yield arr + arr = [] + + if arr: + yield arr + + +class MultiChoice: + def __init__(self, choices) -> None: + self.choices = choices + + # Simple wildcard support (linux filename patterns) + def __contains__(self, values) -> bool: + for value in values.split(","): + if len(fnmatch.filter(self.choices, value)) == 0: + eval_logger.info("Available tasks to choose:") + for choice in self.choices: + eval_logger.info(f" - {choice}") + raise ValueError("'{}' is not in task list".format(value)) + return True + + def __iter__(self) -> Iterator: + for choice in self.choices: + yield choice + + +class Grouper: + """ + takes an array `arr` and function `fn` and returns a dictionary + with keys fn(ob) for each ob in `arr` and with values `self.arr[key]` a list of all + objects in `arr` satisfying `key == fn(ob)`. + """ + + def __init__(self, arr, fn) -> None: + # self.orig_arr = arr + self.size = len(arr) + arr = list(enumerate(arr)) + + def group_return_dict(arr, fn): + res = collections.defaultdict(list) + + for ob in arr: + res[fn(ob)].append(ob) + return res + + arr = group_return_dict(arr, lambda x: fn(x[1])) + + # self.arr has format Dict[Tuple[int, ]] + self.arr = arr + self._grouped = None + + def get_grouped(self): + # return the contents but not indices for our grouped dict. + if self._grouped: + return self._grouped + grouped = {} + for key in self.arr.keys(): + # drop the index from each element of self.arr + grouped[key] = [y[1] for y in self.arr[key]] + self._grouped = grouped + return grouped + + def get_original(self, grouped_dict): + # take in a grouped dictionary with e.g. results for each key listed + # in the same order as the instances in `self.arr`, and + # return the results in the same (single list) order as `self.orig_arr`. + res = [None] * self.size + cov = [False] * self.size + # orig = [None] * self.size + + assert grouped_dict.keys() == self.arr.keys() + + for key in grouped_dict.keys(): + for (ind, _), v in zip(self.arr[key], grouped_dict[key]): + res[ind] = v + cov[ind] = True + # orig[ind] = _ + + assert all(cov) + # assert orig == self.orig_arr + + return res + + +def pad_and_concat( + max_length: int, + tensors: List[torch.Tensor], + padding_side: Literal["right", "left"] = "right", +): + """ + Method for padding a list of tensors given the maximum tensor + length in the batch. Used for batching inputs and continuations in + seq2seq models. + """ + assert padding_side == "left" or padding_side == "right", ( + f"Unrecognized padding type: '{padding_side}' not 'left' or 'right'" + ) + + for i, tensor in enumerate(tensors): + if len(tensor.shape) == 2: + tensor = tensor.squeeze(0) # squeeze, in case passed [1, seq] size + tensor_len = tensor.shape[0] + if tensor_len < max_length: + if padding_side == "right": + # right-pad + tensors[i] = torch.cat( + [ + tensor, # [seq] + torch.zeros( + max_length - tensor_len, + dtype=torch.long, + device=tensor.device, + ), # [padding_length - seq] + ], + dim=0, + ).unsqueeze(0) + else: + # left-pad + tensors[i] = torch.cat( + [ + torch.zeros( + max_length - tensor_len, + dtype=torch.long, + device=tensor.device, + ), # [padding_length - seq] + tensor, # [seq] + ], + dim=0, + ).unsqueeze(0) + else: + tensors[i] = tensor.unsqueeze(0) + + return torch.cat(tensors, dim=0) + + +def clear_torch_cache() -> None: + gc.collect() + torch.cuda.empty_cache() + + +def get_dtype(dtype: Union[str, torch.dtype]) -> torch.dtype: + """Converts `dtype` from `str` to torch.dtype when possible. Does not use an instantiated HF AutoConfig""" + if isinstance(dtype, str) and dtype != "auto": + # Convert `str` args torch dtype: `float16` -> `torch.float16` + _torch_dtype = getattr(torch, dtype) + else: + _torch_dtype = dtype + return _torch_dtype + + +class MultiTokenEOSCriteria(transformers.StoppingCriteria): + """Criteria to stop on the specified multi-token sequence.""" + + def __init__( + self, + sequence: str, + tokenizer: transformers.PreTrainedTokenizer, + initial_decoder_input_length: int, + batch_size: int, + ) -> None: + self.initial_decoder_input_length = initial_decoder_input_length + self.done_tracker = [False] * batch_size + self.sequence = sequence + self.sequence_ids = tokenizer.encode(sequence, add_special_tokens=False) + # print(sequence, self.sequence_ids) + # we look back for 2 more tokens than it takes to encode our stop sequence + # because tokenizers suck, and a model might generate `['\n', '\n']` but our `sequence` is `['\n\n']` + # and we don't want to mistakenly not stop a generation because our + # (string) stop sequence was output in a different tokenization + + # NOTE: there is a minor danger that this will end up looking back 2 tokens into the past, into the inputs to the model, + # and stopping generation immediately as a result. With only 2 extra tokens of lookback, this risk is minimized + # Additionally, in lookback_ids_batch we should prevent ever looking back into the inputs as described. + self.sequence_id_len = len(self.sequence_ids) + 2 + self.tokenizer = tokenizer + + def __call__(self, input_ids, scores, **kwargs) -> bool: + # For efficiency, we compare the last n tokens where n is the number of tokens in the stop_sequence + lookback_ids_batch = input_ids[:, self.initial_decoder_input_length :] + + lookback_ids_batch = lookback_ids_batch[:, -self.sequence_id_len :] + + lookback_tokens_batch = self.tokenizer.batch_decode(lookback_ids_batch) + + for i, done in enumerate(self.done_tracker): + if not done: + self.done_tracker[i] = self.sequence in lookback_tokens_batch[i] + return False not in self.done_tracker + + +def stop_sequences_criteria( + tokenizer: transformers.PreTrainedTokenizer, + stop_sequences: List[str], + initial_decoder_input_length: int, + batch_size: int, +) -> transformers.StoppingCriteriaList: + return transformers.StoppingCriteriaList( + [ + *[ + MultiTokenEOSCriteria( + sequence, tokenizer, initial_decoder_input_length, batch_size + ) + for sequence in stop_sequences + ], + ] + ) + + +def undistribute(iterable): + """ + Undoes https://more-itertools.readthedocs.io/en/stable/api.html#more_itertools.distribute . + + Re-interleaves results that have been split using more_itertools.distribute: + >>> group_1, group_2 = distribute(2, [1, 2, 3, 4, 5, 6]) + >>> list(group_1) + [1, 3, 5] + >>> list(group_2) + [2, 4, 6] + >>> undistribute([group_1, group_2]) + [1, 2, 3, 4, 5, 6] + + Handles non-uniform component lengths: + + >>> children = distribute(3, [1, 2, 3, 4, 5, 6, 7]) + >>> [list(c) for c in children] + [[1, 4, 7], [2, 5], [3, 6]] + >>> undistribute(children) + [1, 2, 3, 4, 5, 6, 7] + + Also handles when some iterables are empty: + + >>> children = distribute(5, [1, 2, 3]) + >>> [list(c) for c in children] + [[1], [2], [3], [], []] + >>> undistribute(children) + [1, 2, 3] + + """ + + return [ + x + for x in itertools.chain.from_iterable( + itertools.zip_longest(*[list(x) for x in iterable]) + ) + if x is not None + ] + + +def retry_on_specific_exceptions( + on_exceptions: List[Type[Exception]], + max_retries: Optional[int] = None, + backoff_time: float = 3.0, + backoff_multiplier: float = 1.5, + on_exception_callback: Optional[Callable[[Exception, float], Any]] = None, +): + """Retry on an LLM Provider's rate limit error with exponential backoff + For example, to use for OpenAI, do the following: + ``` + from openai import RateLimitError + + # Recommend specifying max_retries to avoid infinite loops! + @retry_on_specific_exceptions([RateLimitError], max_retries=3) + def completion(...): + # Wrap OpenAI completion function here + ... + ``` + """ + + def decorator(func: Callable): + @wraps(func) + def wrapper(*args, **kwargs): + sleep_time = backoff_time + attempt = 0 + while max_retries is None or attempt < max_retries: + try: + return func(*args, **kwargs) + except tuple(on_exceptions) as e: + if on_exception_callback is not None: + on_exception_callback(e, sleep_time) + time.sleep(sleep_time) + sleep_time *= backoff_multiplier + attempt += 1 + + return wrapper + + return decorator + + +class Collator: + """ + A class for reordering and batching elements of an array. + + This class allows for sorting an array based on a provided sorting function, grouping elements based on a grouping function, and generating batches from the sorted and grouped data. + + Objects of this class have the group_by attribute which determines the method for grouping + the data while batching it. Three options include "gen_kwargs", "contexts", or None: + If group_by == "gen_kwargs" then requests will be grouped by gen_kwargs + If group_by == "contexts" then requests will be grouped by context + cont[:-1] + If None then requests will just be reordered by length descending. + """ + + def __init__( + self, + arr: List, + sort_fn: Callable = lambda x: x, + group_fn: Callable = lambda x: x[1], + group_by: Union[Literal["gen_kwargs", "contexts"], None] = None, + ) -> None: + self._group_by = group_by + # 0 indices are enumerated indices. Apply functions to original arr. + self._sort_fn = lambda x: sort_fn(x[1]) + self._group_fn = lambda x: group_fn(x[1]) + self._reorder_indices: List = [] + self._size = len(arr) + self._arr_with_indices: Union[Dict, Tuple[Tuple[int, Any], ...]] = tuple( + enumerate(arr) + ) # [indices, (arr)] + if self._group_by == "contexts": + self._group_by_context() + elif self._group_by == "gen_kwargs": + self._group_by_index() + + def _group_by_index(self) -> None: + """Group the elements of a list based on their indices.""" + self._arr_with_indices = self.group( + self._arr_with_indices, fn=self._group_fn, group_by="gen_kwargs" + ) + + def _group_by_context(self) -> None: + """Group the array with indices by context.""" + self._arr_with_indices = self.group( + self._arr_with_indices, fn=self._group_fn, group_by="contexts" + ) + + def get_batched(self, n: int = 1, batch_fn: Optional[Callable] = None) -> Iterator: + """ + Generates and yields batches from the reordered array. The method of grouping and batching + depends on the parameter `group_by`. + If `group_by` is set to "gen_kwargs", it will batch the + re-ordered values with same gen_kwargs for each batch. + If `group_by` is "contexts", it caches the requests by context before batching. + If `group_by` is neither "gen_kwargs" nor "contexts", it yields the reordered array + + Parameters: + - n (int): The size of each batch. Defaults to 1. + - batch_fn ([Callable[[int, Iterable], int]] | None): A function to determine the size of + each batch. Optional, defaults to None. + + Returns: + Iterator: An iterator over batches of reordered elements grouped as per the `group_by` + attribute. + + Yields: + List of batched elements according to the `group_by` attribute. + """ + if self._group_by == "gen_kwargs": + for ( + key, + values, + ) in self._arr_with_indices.items(): # type: ignore + values = self._reorder(values) + batch = self.get_chunks(values, n=n, fn=batch_fn) + yield from batch + elif self._group_by == "contexts": + # Get one sample from each key + values = self._reorder( + [value[0] for value in self._arr_with_indices.values()] + ) + batch = self.get_chunks(values, n=n, fn=batch_fn) + yield from batch + else: + values = self._reorder(self._arr_with_indices) # type: ignore + batch = self.get_chunks(values, n=n, fn=batch_fn) + yield from batch + + def get_cache( + self, + req_str: Tuple[str, str] = None, + cxt_toks: List[int] = None, + cont_toks: List[int] = None, + logits: torch.Tensor = None, + ) -> Iterator[Tuple[Tuple[str, str], List[int], torch.Tensor]]: + """ + Retrieves cached single-token continuations and their associated arguments, updating indices as necessary. + + The behavior of this function varies depending on how the `group_by` attribute is set: + + - When `group_by` is "contexts": + The function identifies single-token continuations by checking for keys that equate to + [context+continuation][-1] and logs the indices for re-ordering. + In this mode, this function can work in two scenarios: + + 1. Cache Hit - Single Match: + If a single matching context-continuation pair is found in the cache, + the function yields the original arguments. + + 2. Cache Hit - Multiple Matches: + If multiple matching context-continuation pairs are found in the cache, + the function expands the logits batch dimension to match the number of cache hits. + It updates the original requests and continuation tokens. + + - When `group_by` is not set to "contexts": + This method yields the original arguments, logits and continuation tokens, + without checking for one-token continuations. + + Parameters: + - req_str (tuple[str, str]): Original strings used for CachingLM. + - cxt_toks (list[int]): Full context tokens used for lookup. + - cont_toks (list[int]): Continuation tokens for which logits were generated. + - logits (torch.Tensor [1, seq_length, vocab_size]): Logits generated by the model given context and continuation keys. + + Yields: + - Iterator: + - req_str (tuple[str, str]): strings used for CachingLM. + - cont_toks (list[int]) : continuation tokens. + - logits (torch.Tensor [1, seq_length, vocab_size]): The original logits (repeated cache hit times) + """ + if self._group_by == "contexts": + cache_hit: List[ + Tuple[int, Tuple[Tuple[str, str], List[int], List[int]]] + ] = self._arr_with_indices.pop(tuple(cxt_toks + cont_toks[:-1])) + if (cache_size := len(cache_hit)) == 1: + self._reorder_indices.extend(x[0] for x in cache_hit) + yield req_str, cont_toks, logits + else: + # If we have matching requests then expand the batch dimension (no-op) and + # yield each along with its corresponding args. + multilogits = logits.expand(cache_size, -1, -1).chunk(cache_size) + indices, req_str, cont_toks = zip( + *[(x[0], x[1][0], x[-1][-1]) for x in cache_hit] + ) + self._reorder_indices.extend(indices) + for c_key, cont_tok, logit in zip(req_str, cont_toks, multilogits): + yield c_key, cont_tok, logit + else: + yield req_str, cont_toks, logits + + def _reorder(self, arr: Union[List, Tuple[Tuple[int, Any], ...]]) -> Iterator: + """ + Reorders the elements in the array based on the sorting function. + + Parameters: + - arr (list | tuple[tuple[int, Any], ...]]): The array or iterable to be reordered. + + Yields: + Iterator + """ + arr = sorted(arr, key=self._sort_fn) + if not self._group_by == "contexts": + # If grouped by contexts then indices will be set in get_cache() + self._reorder_indices.extend([x[0] for x in arr]) + yield from [x[1] for x in arr] + + def get_original(self, newarr: List) -> List: + """ + Restores the original order of elements from the reordered list. + + Parameters: + - newarr (list): The reordered array. + + Returns: + list: The array with elements restored to their original order. + """ + res = [None] * self._size + cov = [False] * self._size + + for ind, v in zip(self._reorder_indices, newarr): + res[ind] = v + cov[ind] = True + + assert all(cov) + + return res + + def __len__(self): + return self._size + + @staticmethod + def group( + arr: Iterable, + fn: Callable, + group_by: Literal["gen_kwargs", "contexts"] = "gen_kwargs", + ) -> dict: + """ + Groups elements of an iterable based on a provided function. + + + The `group_by` parameter determines the method of grouping. + If `group_by` is "contexts", the elements are grouped by [context + cont][:-1]. + If `group_by` is "gen_kwargs", the elements are grouped based on the gen_kwargs dict. + + Parameters: + - arr (Iterable): The iterable to be grouped. + - fn (Callable): The function to determine the grouping. + - values (bool): If True, returns the values of the group. Defaults to False. + + Returns: + Iterator: An iterable of grouped elements. + """ + res = collections.defaultdict(list) + for ob in arr: + # where ob == [context + cont] + if group_by == "contexts": + res[tuple(fn(ob))].append(ob) + else: + try: + hashable_dict = tuple( + ( + key, + tuple(value) + if isinstance(value, collections.abc.Iterable) + else value, + ) + for key, value in sorted(fn(ob).items()) + ) + res[hashable_dict].append(ob) + except (TypeError, AttributeError): + res[tuple(fn(ob))].append(ob) + return res + + @staticmethod + def get_chunks(_iter, n: int = 0, fn=None): + """ + Divides an iterable into chunks of specified size or based on a given function. + Useful for batching + + Parameters: + - iter: The input iterable to be divided into chunks. + - n: An integer representing the size of each chunk. Default is 0. + - fn: A function that takes the current index and the iterable as arguments and returns the size of the chunk. Default is None. + + Returns: + An iterator that yields chunks of the input iterable. + + Example usage: + ``` + data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + for chunk in chunks(data, 3): + print(chunk) + ``` + Output: + ``` + [1, 2, 3] + [4, 5, 6] + [7, 8, 9] + [10] + ``` + """ + arr = [] + _iter = tuple(_iter) + for i, x in enumerate(_iter): + arr.append(x) + if len(arr) == (fn(i, _iter) if fn else n): + yield arr + arr = [] + + if arr: + yield arr + + +def configure_pad_token( + tokenizer: "PreTrainedTokenizerBase", + model_config: Optional["PretrainedConfig"] = None, +) -> "PreTrainedTokenizerBase": + """ + This function checks if the (Hugging Face) tokenizer has a padding token and sets it if not present. + Some tokenizers require special handling. + + Args: + tokenizer: The tokenizer for which the padding token is to be handled. + model_config: The configuration of the model. Default is None. + + Returns: + The tokenizer after the padding token has been handled. + + Raises: + AssertionError: If the tokenizer is of type RWKVWorldTokenizer or Rwkv5Tokenizer and the padding token id is not 0. + """ + if tokenizer.pad_token: + pass + elif tokenizer.unk_token: + tokenizer.pad_token_id = tokenizer.unk_token_id + elif tokenizer.eos_token: + tokenizer.pad_token_id = tokenizer.eos_token_id + else: + # handle special cases + if model_config and getattr(model_config, "model_type", None) == "qwen": + # Qwen's trust_remote_code tokenizer does not allow for adding special tokens + tokenizer.pad_token = "<|endoftext|>" + elif ( + tokenizer.__class__.__name__ == "RWKVWorldTokenizer" + or tokenizer.__class__.__name__ == "Rwkv5Tokenizer" + ): + # The RWKV world tokenizer, does not allow for adding special tokens / setting the pad token (which is set as 0) + # The additional tokenizer name check is needed, as there exists rwkv4 models with neox tokenizer + # --- + # Note that the world tokenizer class name, might change in the future for the final huggingface merge + # https://github.com/huggingface/transformers/pull/26963 + assert tokenizer.pad_token_id == 0 + else: + tokenizer.add_special_tokens({"pad_token": "<|pad|>"}) + + return tokenizer + + +def replace_placeholders( + string: str, default_placeholder: str, image_token: str, max_images: int +): + """ + A utility function used for local multimodal models. It locates all `placeholder` string + occurrences in the given input `string_` and replaces the first `max_count` instances with + `replacement`, and all subsequent occurrences with the empty string. + + This is used to replace placeholder tags by model-specific image tokens like <|image_pad|> + and to allow for only the first `max_count` images to be passed to a model if desired. + + :param string: The original string containing placeholders. + :param default_placeholder: The placeholder text to be replaced. + :param image_token: The token to replace the placeholder with. + :param max_images: The maximum number of replacements to make. + :return: The string with placeholders replaced. + """ + count = 0 + result = [] + + parts = string.split(default_placeholder) + for part in parts[:-1]: # Iterate through all but the last part + result.append(part) + if count < max_images: + result.append(image_token) + count += 1 + elif default_placeholder != image_token: + result.append(default_placeholder) + + # Add the last part of the string + result.append(parts[-1]) + return "".join(result) + + +def flatten_image_list(images: List[List]): + """ + Takes in a list of lists of images, and returns a single list of all images in order. + Used for some multimodal models like Llava-1.5 which expects this flattened-list format for its image processor. + + :param images: A list of lists of PIL images. + :return: a list of PIL images, via concatenating all the sub-lists in order. + """ + return [image for image_list in images for image in image_list] + + +def handle_stop_sequences( + until: Union[str, List[str], None], eos: Optional[str] +) -> List[str]: + """Ensures that the `until` parameter is a list of stop sequences and includes the EOS token.""" + if isinstance(until, str): + until = [until] + elif until is None: + until = [] + elif not isinstance(until, list): + raise ValueError( + f"Expected `kwargs['until']` to be of type Union[str,list] but got {until}" + ) + + if eos is not None and eos not in until: + until.append(eos) + return until diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/models/verifier.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/models/verifier.py new file mode 100644 index 0000000000000000000000000000000000000000..664cf9159e3233528671db6e60d5f521fae2da4b --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/models/verifier.py @@ -0,0 +1,155 @@ +import torch +import logging +import ast +import re +import numpy as np +import textwrap + +logger = logging.getLogger(__name__) + +class CodeVerifier: + def __init__(self, model, tokenizer, device="cuda"): + self.model = model + self.tokenizer = tokenizer + self.device = device + + self.yes_ids, self.no_ids = [], [] + for t in ["Yes", " Yes", "YES"]: + ids = self.tokenizer.encode(t, add_special_tokens=False) + if len(ids) > 0: self.yes_ids.append(ids[-1]) + for t in ["No", " No", "NO"]: + ids = self.tokenizer.encode(t, add_special_tokens=False) + if len(ids) > 0: self.no_ids.append(ids[-1]) + + self.yes_ids = list(set(self.yes_ids)) + self.no_ids = list(set(self.no_ids)) + + def _extract_python_code(self, text): + text = text.strip() + match = re.search(r"```python\s*(.*?)```", text, re.DOTALL) + if match: return match.group(1) + match_generic = re.search(r"```\s*(.*?)```", text, re.DOTALL) + if match_generic: return match_generic.group(1) + return text + + def check_syntax(self, code_str): + clean_code = self._extract_python_code(code_str) + try: + if len(clean_code.strip()) < 5: return False + ast.parse(clean_code) + return True + except: + return False + + def compute_confidence(self, logits): + if logits is None: return 0.0 + probs = torch.softmax(logits, dim=-1) + max_probs, _ = torch.max(probs, dim=-1) + log_probs = torch.log(max_probs + 1e-10) + return torch.exp(torch.mean(log_probs)).item() + + def svf_score(self, prompt, code_str, task_type="code"): + + max_len = 2000 + if len(code_str) > max_len: + if task_type == "reasoning": + truncated_code = code_str[:500] + "\n...[truncated]...\n" + code_str[-(max_len-500):] + else: + truncated_code = code_str[-max_len:] + else: + truncated_code = code_str + + if task_type == "code": + prompt_template = f""" + You are an expert programming contest judge. Your task is to evaluate a generated solution for a given problem based on correctness, efficiency, and adherence to constraints. + + [Problem Statement] + {prompt} + [/Problem Statement] + + [Proposed Python Solution] + ```python + {truncated_code} + ``` + [/Proposed Python Solution] + + **Analysis Steps:** + 1. Correctness: Does the core algorithm correctly solve the problem? + 2. Efficiency: Is the time complexity acceptable for the given constraints? + 3. Edge Cases & Constraints: Does the code handle all rules and edge cases? + + **Conclusion**: Based on your analysis, is the solution likely to be fully correct? Answer with a single word: Yes or No. + **Answer:** """ + + elif task_type == "math": + prompt_template = f""" + You are an expert mathematician and competition judge. Your task is to evaluate a proposed mathematical solution for a given problem based on its logical rigor and accuracy. + + [Math Problem] + {prompt} + [/Math Problem] + + [Proposed Mathematical Solution] + {truncated_code} + [/Proposed Mathematical Solution] + + **Analysis Steps:** + 1. Reasoning Validity: Are the logical steps and mathematical properties applied correctly? + 2. Calculation Accuracy: Are the intermediate calculations or algebraic manipulations accurate? + 3. Goal Alignment: Does the current reasoning path directly lead toward the final answer required by the problem? + + **Conclusion**: Based on your analysis, is this solution path sound and likely to result in the correct final answer? Answer with a single word: Yes or No. + **Answer:** """ + + elif task_type == "reasoning": + prompt_template = f""" + You are an expert reading comprehension and faithfulness judge. Your task is to evaluate a generated answer based on the provided context and question. + + [Context and Question] + {prompt} + [/Context and Question] + + [Proposed Answer] + {truncated_code} + [/Proposed Answer] + + **Analysis Steps :** + 1. Faithfulness: Is the answer an exact, literal span from the context? + 2. Relevance: Does the answer directly address the specific question asked without hallucinating external information? + 3. Accuracy: Does the provided context strictly support this answer? + + **Conclusion**: Based on your analysis, is the answer fully faithful to the context and correct? Answer with a single word: Yes or No. + **Answer:** """ + + else: + prompt_template = f"Is the following answer correct?\nQuestion: {prompt}\nAnswer: {truncated_code}\nAnswer Yes or No.\nAnswer:" + + verify_text = textwrap.dedent(prompt_template).strip() + input_ids = self.tokenizer(verify_text, return_tensors="pt").input_ids.to(self.device) + + max_pos = getattr(self.model.config, "max_position_embeddings", + getattr(self.model.config, "n_positions", + getattr(self.model.config, "max_sequence_length", 20480))) + + if input_ids.shape[1] > max_pos - 16: + logger.warning("Verifier input is too long, truncating from the left.") + input_ids = input_ids[:, -(max_pos - 16):] + + with torch.no_grad(): + with torch.amp.autocast('cuda', dtype=torch.bfloat16): + outputs = self.model(input_ids, 'full') + logits = outputs.logits[0, -1, :] + + yes_score = max((logits[i].item() for i in self.yes_ids if i < logits.shape[-1]), default=-float('inf')) + no_score = max((logits[i].item() for i in self.no_ids if i < logits.shape[-1]), default=-float('inf')) + + if yes_score == -float('inf') and no_score == -float('inf'): return 0.5 + + probs = torch.softmax(torch.tensor([yes_score, no_score]), dim=0) + return probs[0].item() + + def get_reward(self, prompt, code_str, mode="confidence", problem_data=None, current_logits=None, task_type="code"): + if mode == "svf": + return self.svf_score(prompt, code_str, task_type=task_type) + else: + return self.compute_confidence(current_logits) \ No newline at end of file diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/prompts/__init__.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/prompts/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e1ad14105c1b2cd6f9985a9cba796face12b9bca --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/prompts/__init__.py @@ -0,0 +1,128 @@ +import ast +import logging +import os +from typing import Dict + +from lm_eval import utils + + +eval_logger = logging.getLogger(__name__) + +# Prompt library. +# Stores prompts in a dictionary indexed by 2 levels: +# prompt category name, and prompt name. +# This allows us to access prompts +PROMPT_REGISTRY: Dict[str, Dict[str, str]] = { + "qa-basic": { + "question-newline-answer": "Question: {{question}}\nAnswer:", + "q-newline-a": "Q: {{question}}\nA:", + }, +} + + +def get_prompt(prompt_id: str, dataset_name: str = None, subset_name: str = None): + # unpack prompt name + category_name, prompt_name = prompt_id.split(":") + if subset_name is None: + dataset_full_name = dataset_name + else: + dataset_full_name = f"{dataset_name}-{subset_name}" + eval_logger.info(f"Loading prompt from {category_name} for {dataset_full_name}") + if category_name == "promptsource": + try: + from promptsource.templates import DatasetTemplates + except ModuleNotFoundError as exception: + raise type(exception)( + "Tried to load a Promptsource template, but promptsource is not installed ", + "please install promptsource via pip install lm-eval[promptsource] or pip install -e .[promptsource]", + ) + try: + if subset_name is None: + prompts = DatasetTemplates(dataset_name=dataset_name) + else: + prompts = DatasetTemplates( + dataset_name=dataset_name, subset_name=subset_name + ) + except Exception: + raise ValueError(f"{dataset_name} and {subset_name} not found") + if prompt_name in prompts.all_template_names: + return prompts[prompt_name] + else: + raise ValueError( + f"{prompt_name} not in prompt list {prompts.all_template_names}" + ) + elif ".yaml" in category_name: + import yaml + + with open(category_name, "rb") as file: + prompt_yaml_file = yaml.full_load(file) + + prompt_string = prompt_yaml_file["prompts"][prompt_name] + return PromptString(prompt_string) + else: + try: + return PROMPT_REGISTRY[category_name][prompt_name] + except Exception: + raise ValueError( + f"expected only a single `:` as separator between \ + prompt category and name, but got `{prompt_id}` instead" + ) + + +def load_prompt_list( + use_prompt: str, dataset_name=None, subset_name=None, yaml_path=None, **kwargs +): + category_name, prompt_name = use_prompt.split(":") + + if category_name == "promptsource": + from promptsource.templates import DatasetTemplates + + if subset_name is None: + prompts = DatasetTemplates(dataset_name=dataset_name) + else: + prompts = DatasetTemplates( + dataset_name=dataset_name, subset_name=subset_name + ) + + prompt_list = utils.pattern_match(prompt_name, prompts.all_template_names) + + elif ".yaml" in category_name: + import yaml + + if yaml_path is not None: + category_name = os.path.realpath(os.path.join(yaml_path, category_name)) + + with open(category_name, "rb") as file: + prompt_yaml_file = yaml.full_load(file) + + prompt_list = utils.pattern_match( + prompt_name, prompt_yaml_file["prompts"].keys() + ) + + # category_name, *prompt_name = use_prompt.split(":") + # TODO allow to multiple prompt naming + # if len(prompt_name) > 1: + # prompt_list = [] + # for prompt in prompt_name: + # prompt_list.append(utils.pattern_match(prompt_name, prompts.all_template_names)) + # else: + # prompt_list = utils.pattern_match(prompt_name, prompts.all_template_names) + return [":".join([category_name, prompt]) for prompt in prompt_list] + + +class PromptString: + def __init__(self, prompt_string): + self.prompt_string = prompt_string + + def apply(self, doc): + doc_to_text = self.prompt_string["doc_to_text"] + doc_to_target = self.prompt_string["doc_to_target"] + + # TODO need a way to process doc_to_choice + if "doc_to_choice" in self.prompt_string: + raise NotImplementedError("Not yet implemented to accept doc_to_choice") + + text_string = utils.apply_template(doc_to_text, doc) + target_string = utils.apply_template(doc_to_target, doc) + + return [text_string, target_string] diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/tasks/README.md b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/tasks/README.md new file mode 100644 index 0000000000000000000000000000000000000000..dd6403428d6492332077b195853fbfca574634ef --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/tasks/README.md @@ -0,0 +1,165 @@ + +# Tasks + + A list of supported tasks and task groupings can be viewed with `lm-eval --tasks list`. + + For more information, including a full list of task names and their precise meanings or sources, follow the links provided to the individual README.md files for each subfolder. + +| Task Family | Description | Language(s) | +|--------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------| +| [aclue](aclue/README.md) | Tasks focusing on ancient Chinese language understanding and cultural aspects. | Ancient Chinese | +| [aexams](aexams/README.md) | Tasks in Arabic related to various academic exams covering a range of subjects. | Arabic | +| [agieval](agieval/README.md) | Tasks involving historical data or questions related to history and historical texts. | English, Chinese | +| [anli](anli/README.md) | Adversarial natural language inference tasks designed to test model robustness. | English | +| [arabic_leaderboard_complete](arabic_leaderboard_complete/README.md) | A full version of the tasks in the Open Arabic LLM Leaderboard, focusing on the evaluation of models that reflect the characteristics of Arabic language understanding and comprehension, culture, and heritage. Note that some of these tasks are machine-translated. | Arabic (Some MT) | +| [arabic_leaderboard_light](arabic_leaderboard_light/README.md) | A light version of the tasks in the Open Arabic LLM Leaderboard (i.e., 10% samples of the test set in the original benchmarks), focusing on the evaluation of models that reflect the characteristics of Arabic language understanding and comprehension, culture, and heritage. Note that some of these tasks are machine-translated. | Arabic (Some MT) | +| [arabicmmlu](arabicmmlu/README.md) | Localized Arabic version of MMLU with multiple-choice questions from 40 subjects. | Arabic | +| [AraDICE](aradice/README.md) | A collection of multiple tasks carefully designed to evaluate dialectal and cultural capabilities in large language models (LLMs). | Arabic | +| [arc](arc/README.md) | Tasks involving complex reasoning over a diverse set of questions. | English | +| [arithmetic](arithmetic/README.md) | Tasks involving numerical computations and arithmetic reasoning. | English | +| [asdiv](asdiv/README.md) | Tasks involving arithmetic and mathematical reasoning challenges. | English | +| [babi](babi/README.md) | Tasks designed as question and answering challenges based on simulated stories. | English | +| [basque_bench](basque_bench/README.md) | Collection of tasks in Basque encompassing various evaluation areas. | Basque | +| [basqueglue](basqueglue/README.md) | Tasks designed to evaluate language understanding in Basque language. | Basque | +| [bbh](bbh/README.md) | Tasks focused on deep semantic understanding through hypothesization and reasoning. | English, German | +| [bbq](bbq/README.md) | A question-answering benchmark designed to measure social biases in language models across various demographic categories and contexts. | English | +| [belebele](belebele/README.md) | Language understanding tasks in a variety of languages and scripts. | Multiple (122 languages) | +| benchmarks | General benchmarking tasks that test a wide range of language understanding capabilities. | | +| [bertaqa](bertaqa/README.md) | Local Basque cultural trivia QA tests in English and Basque languages. | English, Basque, Basque (MT) | +| [bigbench](bigbench/README.md) | Broad tasks from the BIG-bench benchmark designed to push the boundaries of large models. | Multiple | +| [blimp](blimp/README.md) | Tasks testing grammatical phenomena to evaluate language model's linguistic capabilities. | English | +| [careqa](careqa/README.md) | Multiple choice and open-ended medical question answering based on the Spanish Specialised Healthcare Training (MIR) exams. | English, Spanish | +| [catalan_bench](catalan_bench/README.md) | Collection of tasks in Catalan encompassing various evaluation areas. | Catalan | +| [ceval](ceval/README.md) | Tasks that evaluate language understanding and reasoning in an educational context. | Chinese | +| [cmmlu](cmmlu/README.md) | Multi-subject multiple choice question tasks for comprehensive academic assessment. | Chinese | +| code_x_glue | Tasks that involve understanding and generating code across multiple programming languages. | Go, Java, JS, PHP, Python, Ruby | +| [commonsense_qa](commonsense_qa/README.md) | CommonsenseQA, a multiple-choice QA dataset for measuring commonsense knowledge. | English | +| [copal_id](copal_id/README.md) United States | Indonesian causal commonsense reasoning dataset that captures local nuances. | Indonesian | +| [coqa](coqa/README.md) | Conversational question answering tasks to test dialog understanding. | English | +| [crows_pairs](crows_pairs/README.md) | Tasks designed to test model biases in various sociodemographic groups. | English, French | +| csatqa | Tasks related to SAT and other standardized testing questions for academic assessment. | Korean | +| [drop](drop/README.md) | Tasks requiring numerical reasoning, reading comprehension, and question answering. | English | +| [eq_bench](eq_bench/README.md) | Tasks focused on equality and ethics in question answering and decision-making. | English | +| [eus_exams](eus_exams/README.md) | Tasks based on various professional and academic exams in the Basque language. | Basque | +| [eus_proficiency](eus_proficiency/README.md) | Tasks designed to test proficiency in the Basque language across various topics. | Basque | +| [eus_reading](eus_reading/README.md) | Reading comprehension tasks specifically designed for the Basque language. | Basque | +| [eus_trivia](eus_trivia/README.md) | Trivia and knowledge testing tasks in the Basque language. | Basque | +| [evalita-LLM](evalita-LLM/README.md) | A native Italian benchmark with diverse tasks formats and multiple prompts. | Italian | +| [fda](fda/README.md) | Tasks for extracting key-value pairs from FDA documents to test information extraction. | English | +| [fld](fld/README.md) | Tasks involving free-form and directed dialogue understanding. | English | +| [french_bench](french_bench/README.md) | Set of tasks designed to assess language model performance in French. | French | +| [galician_bench](galician_bench/README.md) | Collection of tasks in Galician encompassing various evaluation areas. | Galician | +| [global_mmlu](global_mmlu/README.md) | Collection of culturally sensitive and culturally agnostic MMLU tasks in 15 languages with human translations or post-edits. | Multiple (15 languages) | +| [glue](glue/README.md) | General Language Understanding Evaluation benchmark to test broad language abilities. | English | +| [gpqa](gpqa/README.md) | Tasks designed for general public question answering and knowledge verification. | English | +| [gsm8k](gsm8k/README.md) | A benchmark of grade school math problems aimed at evaluating reasoning capabilities. | English | +| [groundcocoa](groundcocoa/README.md) | A benchmark evaluating the conditional and compositional reasoning of language models using a grounding task. | English | +| [haerae](haerae/README.md) | Tasks focused on assessing detailed factual and historical knowledge. | Korean | +| [headqa](headqa/README.md) | A high-level education-based question answering dataset to test specialized knowledge. | Spanish, English | +| [hellaswag](hellaswag/README.md) | Tasks to predict the ending of stories or scenarios, testing comprehension and creativity. | English | +| [hendrycks_ethics](hendrycks_ethics/README.md) | Tasks designed to evaluate the ethical reasoning capabilities of models. | English | +| [hendrycks_math](hendrycks_math/README.md) | Mathematical problem-solving tasks to test numerical reasoning and problem-solving. | English | +| [histoires_morales](histoires_morales/README.md) | A dataset of structured narratives that describe normative and norm-divergent actions taken by individuals to accomplish certain intentions in concrete situations. | French (Some MT) | +| [hrm8k](hrm8k/README.md) | A challenging bilingual math reasoning benchmark for Korean and English. | Korean (Some MT), English (Some MT) | +| [humaneval](humaneval/README.md) | Code generation task that measure functional correctness for synthesizing programs from docstrings. | Python | +| [ifeval](ifeval/README.md) | Interactive fiction evaluation tasks for narrative understanding and reasoning. | English | +| [inverse_scaling](inverse_scaling/README.md) | Multiple-choice tasks from the Inverse Scaling Prize, designed to find settings where larger language models perform worse. | English | +| [japanese_leaderboard](japanese_leaderboard/README.md) | Japanese language understanding tasks to benchmark model performance on various linguistic aspects. | Japanese | +| [kbl](kbl/README.md) | Korean Benchmark for Legal Language Understanding. | Korean | +| [kmmlu](kmmlu/README.md) | Knowledge-based multi-subject multiple choice questions for academic evaluation. | Korean | +| [kobest](kobest/README.md) | A collection of tasks designed to evaluate understanding in Korean language. | Korean | +| [kormedmcqa](kormedmcqa/README.md) | Medical question answering tasks in Korean to test specialized domain knowledge. | Korean | +| [lambada](lambada/README.md) | Tasks designed to predict the endings of text passages, testing language prediction skills. | English | +| [lambada_cloze](lambada_cloze/README.md) | Cloze-style LAMBADA dataset. | English | +| [lambada_multilingual](lambada_multilingual/README.md) | Multilingual LAMBADA dataset. This is a legacy version of the multilingual dataset, and users should instead use `lambada_multilingual_stablelm`. | German, English, Spanish, French, Italian | +| [lambada_multilingual_stablelm](lambada_multilingual_stablelm/README.md) | Multilingual LAMBADA dataset. Users should prefer evaluating on this version of the multilingual dataset instead of on `lambada_multilingual`. | German, English, Spanish, French, Italian, Dutch, Portuguese | +| [leaderboard](leaderboard/README.md) | Task group used by Hugging Face's [Open LLM Leaderboard v2](https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard). Those tasks are static and will not change through time | English | +| [lingoly](lingoly/README.md) | Challenging logical reasoning benchmark in low-resource languages with controls for memorization | English, Multilingual | +| [logiqa](logiqa/README.md) | Logical reasoning tasks requiring advanced inference and deduction. | English, Chinese | +| [logiqa2](logiqa2/README.md) | Large-scale logical reasoning dataset adapted from the Chinese Civil Service Examination. | English, Chinese | +| [mastermind](mastermind/README.md) | Reasoning benchmark based on the board game of Mastermind. | English | +| [mathqa](mathqa/README.md) | Question answering tasks involving mathematical reasoning and problem-solving. | English | +| [mbpp](mbpp/README.md) | A benchmark designed to measure the ability to synthesize short Python programs from natural language descriptions. | Python | +| [meddialog](meddialog/README.md) | Medical open-ended QA and Question Entailment stemming from the MedDialog dataset. | English | +| [medtext](medtext/README.md) | Medical open-ended QA from the MedText Clinical Notes dataset. | English | +| [mimic_repsum](mimic_repsum/README.md) | Medical report summarization from the MIMIC-III dataset. | English | +| [mc_taco](mc_taco/README.md) | Question-answer pairs that require temporal commonsense comprehension. | English | +| [med_concepts_qa](med_concepts_qa/README.md) | Benchmark for evaluating LLMs on their abilities to interpret medical codes and distinguish between medical concept. | English | +| [metabench](metabench/README.md) | Distilled versions of six popular benchmarks which are highly predictive of overall benchmark performance and of a single general ability latent trait. | English | +| [mediqa_qa2019](mediqa_qa2019/README.md) | Open-ended healthcare question answering benchmark from the MEDIQA 2019 challenge. | English | +| medmcqa | Medical multiple choice questions assessing detailed medical knowledge. | English | +| medqa | Multiple choice question answering based on the United States Medical License Exams. | | +| [meqsum](meqsum/README.md) | Healtcare Question Entailment benchmark from the MeqSum dataset. | | +| [mgsm](mgsm/README.md) | Benchmark of multilingual grade-school math problems. | Spanish, French, German, Russian, Chinese, Japanese, Thai, Swahili, Bengali, Telugu | +| [minerva_math](minerva_math/README.md) | Mathematics-focused tasks requiring numerical reasoning and problem-solving skills. | English | +| [mlqa](mlqa/README.md) | MultiLingual Question Answering benchmark dataset for evaluating cross-lingual question answering performance. | English, Arabic, German, Spanish, Hindi, Vietnamese, Simplified Chinese | +| [mmlu](mmlu/README.md) | Massive Multitask Language Understanding benchmark for broad domain language evaluation. Several variants are supported. | English | +| [mmlu_pro](mmlu_pro/README.md) | A refined set of MMLU, integrating more challenging, reasoning-focused questions and expanding the choice set from four to ten options. | English | +| [mmlu-pro-plus](mmlu-pro-plus/README.md) | A new test set for evaluating shortcut learning and higher-order reasoning of LLMs. | English | +| [mmlu_prox](mmlu_prox/README.md) | A multilingual benchmark that extends MMLU-Pro to multiple typologically diverse languages with human validation. | English, Japanese, Chinese, Korean, French, German, Spanish, Portuguese, Swahili, Thai, Arabic, Hindi, Bengali | +| [mmlusr](mmlusr/README.md) | Variation of MMLU designed to be more rigorous. | English | +| model_written_evals | Evaluation tasks auto-generated for evaluating a collection of AI Safety concerns. | | +| [moral_stories](moral_stories/README.md) | A crowd-sourced dataset of structured narratives that describe normative and norm-divergent actions taken by individuals to accomplish certain intentions in concrete situations. | English +| [mts_dialog](mts_dialog/README.md) | Open-ended healthcare QA from the MTS-Dialog dataset. | English | +| [mutual](mutual/README.md) | A retrieval-based dataset for multi-turn dialogue reasoning. | English | +| [nq_open](nq_open/README.md) | Open domain question answering tasks based on the Natural Questions dataset. | English | +| [okapi/arc_multilingual](okapi/arc_multilingual/README.md) | Tasks that involve reading comprehension and information retrieval challenges. | Multiple (31 languages) **Machine Translated.** | +| [okapi/hellaswag_multilingual](okapi/hellaswag_multilingual/README.md) | Tasks that involve reading comprehension and information retrieval challenges. | Multiple (30 languages) **Machine Translated.** | +| okapi/mmlu_multilingual | Tasks that involve reading comprehension and information retrieval challenges. | Multiple (34 languages) **Machine Translated.** | +| [okapi/truthfulqa_multilingual](okapi/truthfulqa_multilingual/README.md) | Tasks that involve reading comprehension and information retrieval challenges. | Multiple (31 languages) **Machine Translated.** | +| [olaph](olaph/README.md) | Open-ended medical factuality Question Answering from the OLAPH dataset. | English | +| [openbookqa](openbookqa/README.md) | Open-book question answering tasks that require external knowledge and reasoning. | English | +| [paloma](paloma/README.md) | Paloma is a comprehensive benchmark designed to evaluate open language models across a wide range of domains, ranging from niche artist communities to mental health forums on Reddit. | English | +| [paws-x](paws-x/README.md) | Paraphrase Adversaries from Word Scrambling, focusing on cross-lingual capabilities. | English, French, Spanish, German, Chinese, Japanese, Korean | +| [pile](pile/README.md) | Open source language modelling data set that consists of 22 smaller, high-quality datasets. | English | +| [pile_10k](pile_10k/README.md) | The first 10K elements of The Pile, useful for debugging models trained on it. | English | +| [piqa](piqa/README.md) | Physical Interaction Question Answering tasks to test physical commonsense reasoning. | English | +| [polemo2](polemo2/README.md) | Sentiment analysis and emotion detection tasks based on Polish language data. | Polish | +| [portuguese_bench](portuguese_bench/README.md) | Collection of tasks in European Portuguese encompassing various evaluation areas. | Portuguese | +| [prost](prost/README.md) | Tasks requiring understanding of professional standards and ethics in various domains. | English | +| [pubmedqa](pubmedqa/README.md) | Question answering tasks based on PubMed research articles for biomedical understanding. | English | +| [qa4mre](qa4mre/README.md) | Question Answering for Machine Reading Evaluation, assessing comprehension and reasoning. | English | +| [qasper](qasper/README.md) | Question Answering dataset based on academic papers, testing in-depth scientific knowledge. | English | +| [race](race/README.md) | Reading comprehension assessment tasks based on English exams in China. | English | +| realtoxicityprompts | Tasks to evaluate language models for generating text with potential toxicity. | | +| [ruler](ruler/README.md) | RULER is a benchmark for testing how well language models handle long pieces of text. Requires custom arg (see readme) | English | +| [sciq](sciq/README.md) | Science Question Answering tasks to assess understanding of scientific concepts. | English | +| [score](score/README.md) | Systematic consistency and robustness evaluation for LLMs on 3 datasets(MMLU-Pro, Agi Eval and MATH) | English | +| [scrolls](scrolls/README.md) | Tasks that involve long-form reading comprehension across various domains. | English | +| [simple_cooccurrence_bias](simple_cooccurrence_bias/README.md) | A metric that evaluates language models for biases based on stereotypical word associations and co-occurrences in text. | English | +| [siqa](siqa/README.md) | Social Interaction Question Answering to evaluate common sense and social reasoning. | English | +| [spanish_bench](spanish_bench/README.md) | Collection of tasks in Spanish encompassing various evaluation areas. | Spanish | +| [squad_completion](squad_completion/README.md) | A variant of the SQuAD question answering task designed for zero-shot evaluation of small LMs. | English | +| [squadv2](squadv2/README.md) | Stanford Question Answering Dataset version 2, a reading comprehension benchmark. | English | +| [storycloze](storycloze/README.md) | Tasks to predict story endings, focusing on narrative logic and coherence. | English | +| [super_glue](super_glue/README.md) | A suite of challenging tasks designed to test a range of language understanding skills. | English | +| [swag](swag/README.md) | Situations With Adversarial Generations, predicting the next event in videos. | English | +| [swde](swde/README.md) | Information extraction tasks from semi-structured web pages. | English | +| [tinyBenchmarks](tinyBenchmarks/README.md) | Evaluation of large language models with fewer examples using tiny versions of popular benchmarks. | English | +| [tmmluplus](tmmluplus/README.md) | An extended set of tasks under the TMMLU framework for broader academic assessments. | Traditional Chinese | +| [toxigen](toxigen/README.md) | Tasks designed to evaluate language models on their propensity to generate toxic content. | English | +| [translation](translation/README.md) | Tasks focused on evaluating the language translation capabilities of models. | Arabic, English, Spanish, Basque, Hindi, Indonesian, Burmese, Russian, Swahili, Telugu, Chinese | +| [triviaqa](triviaqa/README.md) | A large-scale dataset for trivia question answering to test general knowledge. | English | +| [truthfulqa](truthfulqa/README.md) | A QA task aimed at evaluating the truthfulness and factual accuracy of model responses. | English | +| [turkishmmlu](turkishmmlu/README.md) | A multiple-choice QA test modeled after MMLU, written in Turkish based on Turkish high-school level exams. | Turkish | +| [unitxt](unitxt/README.md) | A number of tasks implemented using the unitxt library for flexible, shareable, and reusable data preparation and evaluation for generative AI. | English | +| [unscramble](unscramble/README.md) | Tasks involving the rearrangement of scrambled sentences to test syntactic understanding. | English | +| [webqs](webqs/README.md) | Web-based question answering tasks designed to evaluate internet search and retrieval. | English | +| [wikitext](wikitext/README.md) | Tasks based on text from Wikipedia articles to assess language modeling and generation. | English | +| [winogender](winogender/README.md) | A diagnostic dataset that tests for gender bias in coreference resolution by measuring how models associate pronouns with different occupations. | English | +| [winogrande](winogrande/README.md) | A large-scale dataset for coreference resolution, inspired by the Winograd Schema Challenge. | English | +| [wmdp](wmdp/README.md) | A benchmark with the objective of minimizing performance, based on potentially-sensitive multiple-choice knowledge questions. | English | +| [wmt2016](wmt2016/README.md) | Tasks from the WMT 2016 shared task, focusing on translation between multiple languages. | English, Czech, German, Finnish, Russian, Romanian, Turkish | +| [wsc273](wsc273/README.md) | The Winograd Schema Challenge, a test of commonsense reasoning and coreference resolution. | English | +| [xcopa](xcopa/README.md) | Cross-lingual Choice of Plausible Alternatives, testing reasoning in multiple languages. | Estonian, Haitian, Indonesian, Italian, Quechua, Swahili, Tamil, Thai, Turkish, Vietnamese, Chinese | +| [xnli](xnli/README.md) | Cross-Lingual Natural Language Inference to test understanding across different languages. | Arabic, Bulgarian, German, Greek, English, Spanish, French, Hindi, Russian, Swahili, Thai, Turkish, Urdu, Vietnamese, Chinese | +| [xnli_eu](xnli_eu/README.md) | Cross-lingual Natural Language Inference tasks in Basque. | Basque | +| [xquad](xquad/README.md) | Cross-lingual Question Answering Dataset in multiple languages. | Arabic, German, Greek, English, Spanish, Hindi, Romanian, Russian, Thai, Turkish, Vietnamese, Chinese | +| [xstorycloze](xstorycloze/README.md) | Cross-lingual narrative understanding tasks to predict story endings in multiple languages. | Russian, Simplified Chinese, Spanish, Arabic, Hindi, Indonesian, Telugu, Swahili, Basque, Burmese | +| [xwinograd](xwinograd/README.md) | Cross-lingual Winograd schema tasks for coreference resolution in multiple languages. | English, French, Japanese, Portuguese, Russian, Chinese | + +## Multilingual Tasks +| Task Family | Description | Modality | +|------------------------------|---------------------------------------------------------------------------------------------------------|-------------| +| [chartqa](chartqa/README.md) | A benchmark for question answering about charts that requires both visual and logical reasoning. | Image, Text | +| [mmmu](mmmu/README.md) | Evaluate multimodal models on massive multi-discipline tasks demanding college-level subject knowledge. | Image, Text | diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/tasks/__init__.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/tasks/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..602337a4355944a68953390ce21911f0f782e393 --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/tasks/__init__.py @@ -0,0 +1,669 @@ +import collections +import inspect +import logging +import os +from functools import partial +from typing import Dict, List, Mapping, Optional, Union + +from lm_eval import utils +from lm_eval.api.group import ConfigurableGroup, GroupConfig +from lm_eval.api.task import ConfigurableTask, Task +from lm_eval.evaluator_utils import get_subtask_list + + +GROUP_ONLY_KEYS = list(GroupConfig().to_dict().keys()) + +eval_logger = logging.getLogger(__name__) + + +class TaskManager: + """TaskManager indexes all tasks from the default `lm_eval/tasks/` + and an optional directory if provided. + + """ + + def __init__( + self, + verbosity: Optional[str] = None, + include_path: Optional[Union[str, List]] = None, + include_defaults: bool = True, + metadata: Optional[dict] = None, + ) -> None: + if verbosity is not None: + utils.setup_logging(verbosity) + self.include_path = include_path + self.metadata = metadata + self._task_index = self.initialize_tasks( + include_path=include_path, include_defaults=include_defaults + ) + self._all_tasks = sorted(list(self._task_index.keys())) + + self._all_groups = sorted( + [x for x in self._all_tasks if self._task_index[x]["type"] == "group"] + ) + self._all_subtasks = sorted( + [ + x + for x in self._all_tasks + if self._task_index[x]["type"] in ["task", "python_task"] + ] + ) + self._all_tags = sorted( + [x for x in self._all_tasks if self._task_index[x]["type"] == "tag"] + ) + + self.task_group_map = collections.defaultdict(list) + + def initialize_tasks( + self, + include_path: Optional[Union[str, List]] = None, + include_defaults: bool = True, + ) -> dict[str, dict]: + """Creates a dictionary of tasks indexes. + + :param include_path: Union[str, List] = None + An additional path to be searched for tasks recursively. + Can provide more than one such path as a list. + :param include_defaults: bool = True + If set to false, default tasks (those in lm_eval/tasks/) are not indexed. + return + Dictionary of task names as key and task metadata + """ + if include_defaults: + all_paths = [os.path.dirname(os.path.abspath(__file__)) + "/"] + else: + all_paths = [] + if include_path is not None: + if isinstance(include_path, str): + include_path = [include_path] + all_paths.extend(include_path) + + task_index = {} + for task_dir in all_paths: + tasks = self._get_task_and_group(task_dir) + task_index = {**tasks, **task_index} + + return task_index + + @property + def all_tasks(self): + return self._all_tasks + + @property + def all_groups(self): + return self._all_groups + + @property + def all_subtasks(self): + return self._all_subtasks + + @property + def all_tags(self): + return self._all_tags + + @property + def task_index(self): + return self._task_index + + def list_all_tasks( + self, list_groups=True, list_tags=True, list_subtasks=True + ) -> str: + from pytablewriter import MarkdownTableWriter + + def sanitize_path(path): + # don't print full path if we are within the lm_eval/tasks dir ! + # if we aren't though, provide the full path. + if "lm_eval/tasks/" in path: + return "lm_eval/tasks/" + path.split("lm_eval/tasks/")[-1] + else: + return path + + group_table = MarkdownTableWriter() + group_table.headers = ["Group", "Config Location"] + gt_values = [] + for g in self.all_groups: + path = self.task_index[g]["yaml_path"] + if path == -1: + path = "---" + else: + path = sanitize_path(path) + gt_values.append([g, path]) + group_table.value_matrix = gt_values + + tag_table = MarkdownTableWriter() + tag_table.headers = ["Tag"] + tag_table.value_matrix = [[t] for t in self.all_tags] + + subtask_table = MarkdownTableWriter() + subtask_table.headers = ["Task", "Config Location", "Output Type"] + st_values = [] + for t in self.all_subtasks: + path = self.task_index[t]["yaml_path"] + + output_type = "" + + # read the yaml file to determine the output type + if path != -1: + config = utils.load_yaml_config(path, mode="simple") + if "output_type" in config: + output_type = config["output_type"] + elif ( + "include" in config + ): # if no output type, check if there is an include with an output type + include_path = path.split("/")[:-1] + config["include"] + include_config = utils.load_yaml_config(include_path, mode="simple") + if "output_type" in include_config: + output_type = include_config["output_type"] + + if path == -1: + path = "---" + else: + path = sanitize_path(path) + st_values.append([t, path, output_type]) + subtask_table.value_matrix = st_values + + result = "\n" + if list_groups: + result += group_table.dumps() + "\n\n" + if list_tags: + result += tag_table.dumps() + "\n\n" + if list_subtasks: + result += subtask_table.dumps() + "\n\n" + return result + + def match_tasks(self, task_list: list[str]) -> list[str]: + return utils.pattern_match(task_list, self.all_tasks) + + def _name_is_registered(self, name: str) -> bool: + if name in self.all_tasks: + return True + return False + + def _name_is_task(self, name: str) -> bool: + if self._name_is_registered(name) and (self.task_index[name]["type"] == "task"): + return True + return False + + def _name_is_tag(self, name: str) -> bool: + if self._name_is_registered(name) and (self.task_index[name]["type"] == "tag"): + return True + return False + + def _name_is_group(self, name: str) -> bool: + if self._name_is_registered(name) and ( + self.task_index[name]["type"] == "group" + ): + return True + return False + + def _name_is_python_task(self, name: str) -> bool: + if self._name_is_registered(name) and ( + self.task_index[name]["type"] == "python_task" + ): + return True + return False + + def _config_is_task(self, config: dict) -> bool: + if ("task" in config) and isinstance(config["task"], str): + return True + return False + + def _config_is_group(self, config: dict) -> bool: + if ("task" in config) and isinstance(config["task"], list): + return True + return False + + def _config_is_python_task(self, config: dict) -> bool: + if "class" in config: + return True + return False + + def _get_yaml_path(self, name: str): + if name not in self.task_index: + raise ValueError + return self.task_index[name]["yaml_path"] + + def _get_config(self, name): + if name not in self.task_index: + raise ValueError + yaml_path = self._get_yaml_path(name) + if yaml_path == -1: + return {} + else: + return utils.load_yaml_config(yaml_path, mode="full") + + def _get_tasklist(self, name): + if self._name_is_task(name): + raise ValueError + return self.task_index[name]["task"] + + def _process_alias(self, config, group=None): + # If the group is not the same as the original + # group which the group alias was intended for, + # Set the group_alias to None instead. + if ("group_alias" in config) and ("group" in config) and group is not None: + if config["group"] != group: + config["group_alias"] = None + return config + + def _class_has_config_in_constructor(self, cls): + constructor = getattr(cls, "__init__", None) + return ( + "config" in inspect.signature(constructor).parameters + if constructor + else False + ) + + def _load_individual_task_or_group( + self, + name_or_config: Optional[Union[str, dict]] = None, + parent_name: Optional[str] = None, + update_config: Optional[dict] = None, + ) -> Mapping: + def _load_task(config, task): + if "include" in config: + config = { + **utils.load_yaml_config( + yaml_path=None, + yaml_config={"include": config.pop("include")}, + mode="full", + ), + **config, + } + if self._config_is_python_task(config): + if self._class_has_config_in_constructor(config["class"]): + task_object = config["class"](config=config) + else: + task_object = config["class"]() + if isinstance(task_object, ConfigurableTask): + # very scuffed: set task name here. TODO: fixme? + task_object.config.task = task + else: + if self.metadata is not None: + config["metadata"] = config.get("metadata", {}) | self.metadata + else: + config["metadata"] = config.get("metadata", {}) + task_object = ConfigurableTask(config=config) + + return {task: task_object} + + def _get_group_and_subtask_from_config( + config: dict, + ) -> tuple[ConfigurableGroup, list[str]]: + if self.metadata is not None: + config["metadata"] = config.get("metadata", {}) | self.metadata + group_name = ConfigurableGroup(config=config) + subtask_list = [] + for task in group_name.config["task"]: + if isinstance(task, str) and self._name_is_tag(task): + subtask_list.extend(self._get_tasklist(task)) + else: + subtask_list.append(task) + return group_name, subtask_list + + def _process_group_config( + config: dict, update_config: dict = None + ) -> tuple[dict, dict]: + if update_config is not None: + config = {**config, **update_config} + _update_config = { + k: v for k, v in config.items() if k not in GROUP_ONLY_KEYS + } + if not bool(_update_config): + _update_config = None + + group_config = {k: v for k, v in config.items() if k in GROUP_ONLY_KEYS} + return group_config, _update_config + + if isinstance(name_or_config, str): + if update_config is not None: + # Process name_or_config as a dict instead + name_or_config = {"task": name_or_config, **update_config} + elif self._name_is_task(name_or_config) or self._name_is_python_task( + name_or_config + ): + task_config = self._get_config(name_or_config) + return _load_task(task_config, task=name_or_config) + else: + subtask_list = self._get_tasklist(name_or_config) + if subtask_list == -1: + group_config = self._get_config(name_or_config) + group_config, update_config = _process_group_config(group_config) + group_name, subtask_list = _get_group_and_subtask_from_config( + group_config + ) + else: + if self._name_is_tag(name_or_config): + fn = partial( + self._load_individual_task_or_group, + update_config=name_or_config + if isinstance(name_or_config, dict) + else None, + ) + return dict( + collections.ChainMap(*map(fn, reversed(subtask_list))) + ) + else: + group_name = ConfigurableGroup( + config={"group": name_or_config, "task": subtask_list} + ) + + if isinstance(name_or_config, dict): + if self._config_is_task(name_or_config): + name = name_or_config.pop("task") + if update_config is not None: + name_or_config = {**name_or_config, **update_config} + # If the name is registered as a group + if self._name_is_group(name): + group_config = self._get_config(name) + + group_config, update_config = _process_group_config( + group_config, name_or_config + ) + group_name, subtask_list = _get_group_and_subtask_from_config( + group_config + ) + elif self._name_is_tag(name): + subtask_list = self._get_tasklist(name) + fn = partial( + self._load_individual_task_or_group, + update_config=name_or_config, + ) + return dict(collections.ChainMap(*map(fn, reversed(subtask_list)))) + else: + if self._name_is_registered(name): + base_task_config = self._get_config(name) + + # Check if this is a duplicate. + if parent_name is not None: + num_duplicate = len( + list( + filter( + lambda x: x.startswith(name), + self.task_group_map[parent_name], + ) + ) + ) + if num_duplicate > 0: + name = f"{name}-{num_duplicate}" + self.task_group_map[parent_name].append(name) + + task_config = { + **base_task_config, + **name_or_config, + } + else: + task_config = name_or_config + return _load_task(task_config, task=name) + else: + group_config, update_config = _process_group_config(name_or_config) + group_name, subtask_list = _get_group_and_subtask_from_config( + group_config + ) + + fn = partial( + self._load_individual_task_or_group, + parent_name=group_name, + update_config=update_config, + ) + return { + group_name: dict(collections.ChainMap(*map(fn, reversed(subtask_list)))) + } + + def load_task_or_group(self, task_list: Optional[Union[str, list]] = None) -> dict: + """Loads a dictionary of task objects from a list + + :param task_list: Union[str, list] = None + Single string or list of string of task names to be loaded + + :return + Dictionary of task objects + """ + if isinstance(task_list, str): + task_list = [task_list] + + all_loaded_tasks = dict( + collections.ChainMap( + *map( + lambda task: self._load_individual_task_or_group(task), + task_list, + ) + ) + ) + return all_loaded_tasks + + def load_config(self, config: Dict): + return self._load_individual_task_or_group(config) + + def _get_task_and_group(self, task_dir: str): + """Creates a dictionary of tasks index with the following metadata, + - `type`, that can be either `task`, `python_task`, `group` or `tags`. + `task` refer to regular task configs, `python_task` are special + yaml files that only consists of `task` and `class` parameters. + `group` are group configs. `tags` are labels that can be assigned + to tasks to assist in sorting and calling tasks of certain themes. + - `yaml_path`, path to the yaml file. If the entry is a `group` that + was configured through a task config, the yaml_path will be -1 + and all subtasks will be listed in `task` (see below) + - `task`, reserved for entries with `type` as `group`. This will list + all subtasks. When a group config is created (as opposed to task + config having `group` parameter set), this will be set to -1 to + avoid recursive indexing. The whole list of subtasks will be loaded + at evaluation. + + :param task_dir: str + A directory to check for tasks + + :return + Dictionary of task names as key and task metadata + """ + + def _populate_tags_and_groups(config, task, tasks_and_groups, print_info): + # TODO: remove group in next release + if "tag" in config: + attr_list = config["tag"] + if isinstance(attr_list, str): + attr_list = [attr_list] + + for tag in attr_list: + if tag not in tasks_and_groups: + tasks_and_groups[tag] = { + "type": "tag", + "task": [task], + "yaml_path": -1, + } + elif tasks_and_groups[tag]["type"] != "tag": + eval_logger.info( + f"The tag '{tag}' is already registered as a group, this tag will not be registered. " + "This may affect tasks you want to call." + ) + break + else: + tasks_and_groups[tag]["task"].append(task) + + # TODO: remove group in next release + print_info = True + ignore_dirs = [ + "__pycache__", + ".ipynb_checkpoints", + ] + tasks_and_groups = collections.defaultdict() + for root, dirs, file_list in os.walk(task_dir): + dirs[:] = [d for d in dirs if d not in ignore_dirs] + for f in file_list: + if f.endswith(".yaml"): + yaml_path = os.path.join(root, f) + config = utils.load_yaml_config(yaml_path, mode="simple") + if self._config_is_python_task(config): + # This is a python class config + task = config["task"] + tasks_and_groups[task] = { + "type": "python_task", + "yaml_path": yaml_path, + } + _populate_tags_and_groups( + config, task, tasks_and_groups, print_info + ) + elif self._config_is_group(config): + # This is a group config + tasks_and_groups[config["group"]] = { + "type": "group", + "task": -1, # This signals that + # we don't need to know + # the task list for indexing + # as it can be loaded + # when called. + "yaml_path": yaml_path, + } + + # # Registered the level 1 tasks from a group config + # for config in config["task"]: + # if isinstance(config, dict) and self._config_is_task(config): + # task = config["task"] + # tasks_and_groups[task] = { + # "type": "task", + # "yaml_path": yaml_path, + # } + + elif self._config_is_task(config): + # This is a task config + task = config["task"] + tasks_and_groups[task] = { + "type": "task", + "yaml_path": yaml_path, + } + _populate_tags_and_groups( + config, task, tasks_and_groups, print_info + ) + else: + eval_logger.debug(f"File {f} in {root} could not be loaded") + + return tasks_and_groups + + +def get_task_name_from_config(task_config: Dict[str, str]) -> str: + if "task" in task_config: + return task_config["task"] + if "dataset_name" in task_config: + return "{dataset_path}_{dataset_name}".format(**task_config) + else: + return "{dataset_path}".format(**task_config) + + +def get_task_name_from_object(task_object): + if hasattr(task_object, "config"): + return task_object._config["task"] + + # TODO: scrap this + # this gives a mechanism for non-registered tasks to have a custom name anyways when reporting + return ( + task_object.EVAL_HARNESS_NAME + if hasattr(task_object, "EVAL_HARNESS_NAME") + else type(task_object).__name__ + ) + + +def _check_duplicates(task_dict: dict) -> None: + """helper function solely used in validating get_task_dict output. + Takes the output of lm_eval.evaluator_utils.get_subtask_list and + returns a list of all leaf subtasks contained within, and errors if any such leaf subtasks are + "oversubscribed" to several disjoint groups. + """ + subtask_names = [] + for key, value in task_dict.items(): + subtask_names.extend(value) + + duplicate_tasks = { + task_name for task_name in subtask_names if subtask_names.count(task_name) > 1 + } + + # locate the potentially problematic groups that seem to 'compete' for constituent subtasks + competing_groups = [ + group + for group in task_dict.keys() + if len(set(task_dict[group]).intersection(duplicate_tasks)) > 0 + ] + + if len(duplicate_tasks) > 0: + raise ValueError( + f"Found 1 or more tasks while trying to call get_task_dict() that were members of more than 1 called group: {list(duplicate_tasks)}. Offending groups: {competing_groups}. Please call groups which overlap their constituent tasks in separate evaluation runs." + ) + + +def get_task_dict( + task_name_list: Union[str, List[Union[str, Dict, Task]]], + task_manager: Optional[TaskManager] = None, +): + """Creates a dictionary of task objects from either a name of task, config, or prepared Task object. + + :param task_name_list: List[Union[str, Dict, Task]] + Name of model or LM object, see lm_eval.models.get_model + :param task_manager: TaskManager = None + A TaskManager object that stores indexed tasks. If not set, + task_manager will load one. This should be set by the user + if there are additional paths that want to be included + via `include_path` + + :return + Dictionary of task objects + """ + + task_name_from_string_dict = {} + task_name_from_config_dict = {} + task_name_from_object_dict = {} + + if isinstance(task_name_list, str): + task_name_list = [task_name_list] + elif isinstance(task_name_list, list): + if not all([isinstance(task, (str, dict, Task)) for task in task_name_list]): + raise TypeError( + "Expected all list items to be of types 'str', 'dict', or 'Task', but at least one entry did not match." + ) + else: + raise TypeError( + f"Expected a 'str' or 'list' but received {type(task_name_list)}." + ) + + string_task_name_list = [task for task in task_name_list if isinstance(task, str)] + others_task_name_list = [ + task for task in task_name_list if not isinstance(task, str) + ] + if len(string_task_name_list) > 0: + if task_manager is None: + task_manager = TaskManager() + + task_name_from_string_dict = task_manager.load_task_or_group( + string_task_name_list + ) + + for task_element in others_task_name_list: + if isinstance(task_element, dict): + task_name_from_config_dict = { + **task_name_from_config_dict, + **task_manager.load_config(config=task_element), + } + + elif isinstance(task_element, Task): + task_name_from_object_dict = { + **task_name_from_object_dict, + get_task_name_from_object(task_element): task_element, + } + + if not set(task_name_from_string_dict.keys()).isdisjoint( + set(task_name_from_object_dict.keys()) + ): + raise ValueError + + final_task_dict = { + **task_name_from_string_dict, + **task_name_from_config_dict, + **task_name_from_object_dict, + } + + # behavior can get odd if one tries to invoke several groups that "compete" for the same task. + # (notably, because one could request several num_fewshot values at once in GroupConfig overrides for the subtask + # and we'd be unsure which to use and report.) + # we explicitly check and error in this case. + _check_duplicates(get_subtask_list(final_task_dict)) + + return final_task_dict diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/tasks/humaneval/humaneval_5.yaml b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/tasks/humaneval/humaneval_5.yaml new file mode 100644 index 0000000000000000000000000000000000000000..01be8ec130a1c1f6669e6b5dc0a0245bbc246e16 --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/tasks/humaneval/humaneval_5.yaml @@ -0,0 +1,19 @@ +include: humaneval.yaml +task: humaneval_5 +repeats: 5 +metric_list: + - metric: !function utils.pass_at_k + aggregation: mean + higher_is_better: true + k: [1,2,3,4,5] +generation_kwargs: + until: + - "\nclass" + - "\ndef" + - "\n#" + - "\nif" + - "\nprint" + max_gen_toks: 1024 + do_sample: true + temperature: 0.2 + top_p: 0.95 diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/tasks/humaneval/humaneval_5_instruct_noprefix.yaml b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/tasks/humaneval/humaneval_5_instruct_noprefix.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6cb0f51cfc1cd4c38f266518a1ad1931dec5d9d3 --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/tasks/humaneval/humaneval_5_instruct_noprefix.yaml @@ -0,0 +1,15 @@ +include: humaneval_5.yaml +task: humaneval_5_instruct_noprefix +doc_to_text: "Write a solution to the following problem and make sure that it passes the tests:\n```{{prompt}}" +gen_prefix: "```python\n" +generation_kwargs: + until: + - "\nassert" + - "\n# Test" +filter_list: + - name: "create_test" + filter: + - function: "custom" + filter_fn: !function utils.build_predictions_instruct +metadata: + version: 2.0 diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/tasks/humaneval/humaneval_64.yaml b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/tasks/humaneval/humaneval_64.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1720ae7c77c1d1b0bae96a4b1153ea6b0893c64d --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/tasks/humaneval/humaneval_64.yaml @@ -0,0 +1,19 @@ +include: humaneval.yaml +task: humaneval_64 +repeats: 64 +metric_list: + - metric: !function utils.pass_at_k + aggregation: mean + higher_is_better: true + k: [2,8,16,32,64] +generation_kwargs: + until: + - "\nclass" + - "\ndef" + - "\n#" + - "\nif" + - "\nprint" + max_gen_toks: 1024 + do_sample: true + temperature: 0.2 + top_p: 0.95 diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/tasks/humaneval/humaneval_instruct.yaml b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/tasks/humaneval/humaneval_instruct.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0dcbd49e46760d63f1d72fe09194c4102b6a0415 --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/tasks/humaneval/humaneval_instruct.yaml @@ -0,0 +1,11 @@ +include: humaneval.yaml +task: humaneval_instruct +doc_to_text: "Write a solution to the following problem and make sure that it passes the tests:\n```{{prompt}}" +gen_prefix: "Here is the completed function:\n```python\n{{prompt}}\n" +filter_list: + - name: "create_test" + filter: + - function: "custom" + filter_fn: !function utils.build_predictions_instruct +metadata: + version: 2.0 \ No newline at end of file diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/tasks/humaneval/humaneval_instruct_noprefix.yaml b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/tasks/humaneval/humaneval_instruct_noprefix.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2f11a7aea0985234d5d39ce8d1f163eea9803bdb --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/tasks/humaneval/humaneval_instruct_noprefix.yaml @@ -0,0 +1,15 @@ +include: humaneval.yaml +task: humaneval_instruct_noprefix +doc_to_text: "Write a solution to the following problem and make sure that it passes the tests:\n```{{prompt}}```" +gen_prefix: "```python\n" +generation_kwargs: + until: + - "\nassert" + - "\n# Test" +filter_list: + - name: "create_test" + filter: + - function: "custom" + filter_fn: !function utils.build_predictions_instruct +metadata: + version: 2.0 diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/tasks/humaneval/sanitize_utils.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/tasks/humaneval/sanitize_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..a8acf070cc2d47a4aee9fe37e0b73ecda22fd9ea --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/tasks/humaneval/sanitize_utils.py @@ -0,0 +1,121 @@ +import ast +import traceback + +from typing import Dict, List, Optional, Set, Tuple + +def refine_text(text: str) -> str: + text = text.replace("\t", " ") + text = text.replace("\r\n", "\n").replace("\r", "\n") + return text.strip() + "\n" + +def syntax_check(code, verbose = False): + try: + ast.parse(code) + return True + except (SyntaxError, MemoryError): + if verbose: + traceback.print_exc() + return False + +def extract_longest_valid_code(text: str) -> str: + lines = text.splitlines() + + if len(lines) > 100: + lines = lines[:100] + max_valid_lines = 0 + max_valid_snippet = "" + + for i in range(len(lines)): + for j in range(i, len(lines)): + current_snippet = "\n".join(lines[i:j+1]) + if syntax_check(current_snippet): + valid_line_count = sum(1 for line in lines[i:j+1] if line.strip()) + if valid_line_count > max_valid_lines: + max_valid_lines = valid_line_count + max_valid_snippet = current_snippet + + return max_valid_snippet + +def get_deps(nodes: List[Tuple[str, ast.AST]]) -> Dict[str, Set[str]]: + name2deps = {} + for name, node in nodes: + deps = set() + stack = [node] + while stack: + current = stack.pop() + for child in ast.iter_child_nodes(current): + if isinstance(child, ast.Name): + deps.add(child.id) + elif isinstance(child, ast.Attribute): + deps.add(child.attr) + else: + stack.append(child) + name2deps[name] = deps + return name2deps + +def get_function_dependency(entrypoint: str, call_graph: Dict[str, Set[str]]) -> Set[str]: + visited = set() + to_visit = [entrypoint] + + while to_visit: + current = to_visit.pop(0) + if current not in visited: + visited.add(current) + to_visit.extend(call_graph.get(current, set()) - visited) + + return visited + +def get_definition_name(node: ast.AST) -> Optional[str]: + if isinstance(node, (ast.FunctionDef, ast.ClassDef)): + return node.name + elif isinstance(node, ast.Assign): + targets = node.targets + if targets and isinstance(targets[0], ast.Name): + return targets[0].id + return None + +def has_return_statement(node: ast.AST) -> bool: + return any(isinstance(n, ast.Return) for n in ast.walk(node)) + +def sanitize(text: str, entrypoint: Optional[str] = None) -> str: + + text = refine_text(text) + + # text = python_extract(text) + + code = extract_longest_valid_code(text) + tree = ast.parse(code) + + definitions = {} + + imports = [] + + for node in tree.body: + if isinstance(node, (ast.Import, ast.ImportFrom)): + imports.append(node) + elif isinstance(node, ast.ClassDef): + name = node.name + definitions[name] = ('class', node) + elif isinstance(node, ast.FunctionDef): + name = node.name + if has_return_statement(node): + definitions[name] = ('function', node) + elif isinstance(node, ast.Assign): + name = get_definition_name(node) + if name: + definitions[name] = ('variable', node) + + if entrypoint: + name2deps = get_deps([(name, node) for name, (_, node) in definitions.items()]) + reachable = get_function_dependency(entrypoint, name2deps) + + sanitized_output = [] + + for node in imports: + sanitized_output.append(ast.unparse(node)) + + for name, (_, node) in definitions.items(): + if not entrypoint or name in reachable: + sanitized_output.append(ast.unparse(node)) + + return "\n".join(sanitized_output) \ No newline at end of file diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/tasks/humaneval/utils.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/tasks/humaneval/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fc98f5d4f71963e8d93b0ccd6aec39d0691960d2 --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/tasks/humaneval/utils.py @@ -0,0 +1,52 @@ +import evaluate as hf_evaluate + +from lm_eval.tasks.humaneval.sanitize_utils import sanitize + + +try: + compute_ = hf_evaluate.load("code_eval") + test_cases = ["assert add(2, 3)==5"] + candidates = [["def add(a,b): return a*b"]] + results = compute_.compute(references=test_cases, predictions=candidates, k=[1]) +except Exception as e: + raise e + + +def pass_at_k(references: list[str], predictions: list[list[str]], k: list[int] = None): + global compute_ + assert k is not None + if isinstance(k, int): + k = [k] + + processed_predictions = [] + for preds in predictions: + processed_preds = [] + for p in preds: + processed_preds.append(p.strip("```")[0] if "```" in p else p) + processed_predictions.append(processed_preds) + + res = compute_.compute( + references=references, + predictions=predictions, + k=k, + ) + return res[0] + + +def build_predictions(resps: list[list[str]], docs: list[dict]) -> list[list[str]]: + return [[doc["prompt"] + r for r in resp] for resp, doc in zip(resps, docs)] + + +def build_predictions_instruct( + resps: list[list[str]], docs: list[dict] +) -> list[list[str]]: + return [ + [ + sanitize( + doc["prompt"] + "\n" + r.split('```python\n', 1)[-1].split('```')[0], + doc["entry_point"] + ) + for r in resp + ] + for resp, doc in zip(resps, docs) + ] \ No newline at end of file diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/utils.py b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..bca19866ce7ac86c8c34bfac3f1f07a2c2345565 --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/lm_eval/utils.py @@ -0,0 +1,552 @@ +import collections +import fnmatch +import functools +import hashlib +import importlib.util +import inspect +import json +import logging +import os +import re +from dataclasses import asdict, is_dataclass +from itertools import islice +from pathlib import Path +from typing import Any, Callable, Generator, List, Optional, Tuple + +import numpy as np +import yaml +from jinja2 import BaseLoader, Environment, StrictUndefined + + +SPACING = " " * 47 + +HIGHER_IS_BETTER_SYMBOLS = { + True: "↑", + False: "↓", +} + + +def setup_logging(verbosity=logging.INFO): + # Configure the root logger + class CustomFormatter(logging.Formatter): + def format(self, record): + if record.name.startswith("lm_eval."): + record.name = record.name[len("lm_eval.") :] + return super().format(record) + + formatter = CustomFormatter( + "%(asctime)s %(levelname)-8s [%(name)s:%(lineno)d] %(message)s", + datefmt="%Y-%m-%d:%H:%M:%S", + ) + + log_level = os.environ.get("LOGLEVEL", verbosity) or verbosity + + level_map = { + "DEBUG": logging.DEBUG, + "INFO": logging.INFO, + "WARNING": logging.WARNING, + "ERROR": logging.ERROR, + "CRITICAL": logging.CRITICAL, + } + + log_level = level_map.get(str(log_level).upper(), logging.INFO) + + if not logging.root.handlers: + handler = logging.StreamHandler() + handler.setFormatter(formatter) + + root_logger = logging.getLogger() + root_logger.addHandler(handler) + root_logger.setLevel(log_level) + + if log_level == logging.DEBUG: + third_party_loggers = ["urllib3", "filelock", "fsspec"] + for logger_name in third_party_loggers: + logging.getLogger(logger_name).setLevel(logging.INFO) + else: + logging.getLogger().setLevel(log_level) + + +def hash_string(string: str) -> str: + return hashlib.sha256(string.encode("utf-8")).hexdigest() + + +def escaped_split(text, sep_char, maxsplit=-1): + """Split text into a list on occurrences of the given separation + character `sep_char`. The separation character may be escaped by a + backslash to avoid splitting at that location. + + The separation character must be a string of size 1. + + If `maxsplit` is given, at most `maxsplit` splits are done (thus, + the list will have at most `maxsplit + 1` elements). If `maxsplit` + is not specified or less than 0, then there is no limit on the + number of splits (all possible splits are made). + """ + assert len(sep_char) == 1, ( + "separation string must be a single character for escaped splitting" + ) + + if maxsplit == 0: + return text + maxsplit = max(0, maxsplit) + + return re.split(r"(? dict: + """ + Parses something like + args1=val1,arg2=val2 + Into a dictionary + """ + if args_string is None: + return {} + args_string = args_string.strip() + if not args_string: + return {} + arg_list = [arg for arg in args_string.split(",") if arg] + args_dict = { + kv[0]: handle_arg_string("=".join(kv[1:])) + for kv in [arg.split("=") for arg in arg_list] + } + return args_dict + + +def join_iters(iters): + for iter in iters: + yield from iter + + +def group(arr, fn): + res = collections.defaultdict(list) + + for ob in arr: + res[fn(ob)].append(ob) + + return list(res.values()) + + +# Returns a list containing all values of the source_list that +# match at least one of the patterns +def pattern_match(patterns, source_list): + if isinstance(patterns, str): + patterns = [patterns] + + task_names = set() + for pattern in patterns: + for matching in fnmatch.filter(source_list, pattern): + task_names.add(matching) + return sorted(list(task_names)) + + +def softmax(x) -> np.ndarray: + """Compute softmax values for each sets of scores in x.""" + e_x = np.exp(x - np.max(x)) + return e_x / e_x.sum() + + +def general_detokenize(string) -> str: + string = string.replace(" n't", "n't") + string = string.replace(" )", ")") + string = string.replace("( ", "(") + string = string.replace('" ', '"') + string = string.replace(' "', '"') + string = re.sub(r" (['.,])", r"\1", string) + return string + + +def get_file_task_name(filename: str) -> str: + """ + Given the sample results filenames, extracts and returns the task name. + """ + return filename[filename.find("_") + 1 : filename.rfind("_")] + + +def get_file_datetime(filename: str) -> str: + """ + Given the results and sample results filenames, extracts and returns the datetime. + """ + return filename[filename.rfind("_") + 1 :].replace(".jsonl", "") + + +def sanitize_model_name(model_name: str) -> str: + """ + Given the model name, returns a sanitized version of it. + """ + return re.sub(r"[\"<>:/\|\\?\*\[\]]+", "__", model_name) + + +def sanitize_task_name(task_name: str) -> str: + """ + Given the task name, returns a sanitized version of it. + """ + return re.sub(r"\W", "_", task_name) + + +def get_latest_filename(filenames: List[str]) -> str: + """ + Given a list of filenames, returns the filename with the latest datetime. + """ + return max(filenames, key=lambda f: get_file_datetime(f)) + + +def get_results_filenames(filenames: List[str]) -> List[str]: + """ + Extracts filenames that correspond to aggregated results. + """ + return [f for f in filenames if "/results_" in f and ".json" in f] + + +def get_sample_results_filenames(filenames: List[str]) -> List[str]: + """ + Extracts filenames that correspond to sample results. + """ + return [f for f in filenames if "/samples_" in f and ".json" in f] + + +def get_rolling_token_windows( + token_list: List[int], prefix_token: int, max_seq_len: int, context_len: int +) -> Generator[Tuple[List[int], List[int]], None, None]: + """ + - context_len allows for a rolling window context, allowing each prediction window to potentially + condition on some context + + :param token_list: list + List of tokens to be PREDICTED + :param max_seq_len: int + max_seq_len of model (or max_seq_len we want to use) + :param context_len: int + Amount of desired token context for prediction. Needs to be at least 1. + :param prefix_token: token + Dummy token like so the first token has something to condition on + :return: generator + Generator of tuples + (input_tokens, pred_tokens) + Note: Score only the last len(pred_tokens) logits of the LM + """ + assert 1 <= context_len <= max_seq_len + if not token_list: + return + # +1 offset, going from input->preds + pred_len = max_seq_len - context_len + 1 + predicted = 0 + + # Special handling for first window: predict all tokens + first_seq_len = min(max_seq_len, len(token_list)) + yield [prefix_token] + token_list[: first_seq_len - 1], token_list[:first_seq_len] + predicted += first_seq_len + + while predicted < len(token_list): + window_pred_len = min(len(token_list) - predicted, pred_len) + window_end = predicted + window_pred_len + + yield ( + token_list[window_end - max_seq_len - 1 : window_end - 1], + token_list[window_end - window_pred_len : window_end], + ) + predicted += window_pred_len + + +def make_disjoint_window( + pair: Tuple[List[int], List[int]], +) -> Tuple[List[int], List[int]]: + """Takes output from get_rolling_token_windows and makes the context not overlap with the continuation""" + a, b = pair + return a[: len(a) - (len(b) - 1)], b + + +class EnhancedJSONEncoder(json.JSONEncoder): + """ + Provides a proper json encoding for the loggers and trackers json dumps. + Notably manages the json encoding of dataclasses. + """ + + def default(self, o): + if is_dataclass(o): + return asdict(o) + return super().default(o) + + +class Reorderer: + def __init__(self, arr: List[Any], fn: Callable) -> None: + """Reorder an array according to some function + + Args: + arr (List[Any]): The initial array + fn (Callable[[Any], Any]): A function to determine the priority of elements + """ + self.size = len(arr) + arr = list(enumerate(arr)) + arr = group(arr, lambda x: fn(x[1])) + # arr = [([y[0] for y in x], x[0][1]) for x in arr] + # TODO: overhaul reorderer. It currently grouped requests by content but we don't want this + arr = [([y[0]], x[0][1]) for x in arr for y in x] + arr.sort(key=lambda x: fn(x[1])) + + self.arr = arr + + def get_reordered(self): + """Gets the reordered array + + Returns: + List[Any]: The reordered array + """ + return [x[1] for x in self.arr] + + def get_original(self, newarr): + """Restores the original order of a new array based on the old array's order + + Args: + newarr (List[Any]): The array to be restored + + Returns: + List[Any]: The array restored to the original order + """ + res = [None] * self.size + cov = [False] * self.size + + for (inds, _), v in zip(self.arr, newarr): + for ind in inds: + res[ind] = v + cov[ind] = True + + assert all(cov) + + return res + + +def make_table(result_dict, column: str = "results", sort_results: bool = False): + """Generate table of results.""" + from pytablewriter import LatexTableWriter, MarkdownTableWriter + + if column == "results": + column_name = "Tasks" + elif column == "groups": + column_name = "Groups" + + all_headers = [ + column_name, + "Version", + "Filter", + "n-shot", + "Metric", + "", + "Value", + "", + "Stderr", + ] + + md_writer = MarkdownTableWriter() + latex_writer = LatexTableWriter() + md_writer.headers = all_headers + latex_writer.headers = all_headers + + values = [] + + keys = result_dict[column].keys() + if sort_results: + # sort entries alphabetically by task or group name. + # NOTE: we default here to false, because order matters for multi-level table printing a la mmlu. + # sorting here would mess that up + keys = sorted(keys) + for k in keys: + dic = result_dict[column][k] + version = result_dict["versions"].get(k, " N/A") + n = str(result_dict.get("n-shot", " ").get(k, " ")) + higher_is_better = result_dict.get("higher_is_better", {}).get(k, {}) + + if "alias" in dic: + k = dic.pop("alias") + + metric_items = dic.items() + metric_items = sorted(metric_items) + + for (mf), v in metric_items: + m, _, f = mf.partition(",") + if m.endswith("_stderr"): + continue + + hib = HIGHER_IS_BETTER_SYMBOLS.get(higher_is_better.get(m), "") + + v = "%.4f" % v if isinstance(v, float) else v + + if m + "_stderr" + "," + f in dic: + se = dic[m + "_stderr" + "," + f] + se = " N/A" if se == "N/A" else "%.4f" % se + values.append([k, version, f, n, m, hib, v, "±", se]) + else: + values.append([k, version, f, n, m, hib, v, "", ""]) + k = "" + version = "" + md_writer.value_matrix = values + latex_writer.value_matrix = values + + # todo: make latex table look good + # print(latex_writer.dumps()) + + return md_writer.dumps() + + +def positional_deprecated(fn): + """ + A decorator to nudge users into passing only keyword args (`kwargs`) to the + wrapped function, `fn`. + """ + + @functools.wraps(fn) + def _wrapper(*args, **kwargs): + if len(args) != 1 if inspect.ismethod(fn) else 0: + print( + f"WARNING: using {fn.__name__} with positional arguments is " + "deprecated and will be disallowed in a future version of " + "lm-evaluation-harness!" + ) + return fn(*args, **kwargs) + + return _wrapper + + +def ignore_constructor(loader, node): + return node + + +def import_function(loader: yaml.Loader, node, yaml_path: Path): + function_name = loader.construct_scalar(node) + + *module_name, function_name = function_name.split(".") + if isinstance(module_name, list): + module_name = ".".join(module_name) + module_path = yaml_path.parent / f"{module_name}.py" + + spec = importlib.util.spec_from_file_location(module_name, module_path.as_posix()) + + if spec is None: + raise ImportError(f"Could not import module {module_name} from {module_path}.") + module = importlib.util.module_from_spec(spec) + + if spec.loader is None: + raise ImportError(f"Module loader is None, {module_name} from {module_path}.") + spec.loader.exec_module(module) + + function = getattr(module, function_name) + return function + + +def load_yaml_config(yaml_path=None, yaml_config=None, yaml_dir=None, mode="full"): + if mode == "simple": + constructor_fn = ignore_constructor + elif mode == "full": + if yaml_path is None: + raise ValueError("yaml_path must be provided if mode is 'full'.") + # Attach yaml_path to the import function so that it can be used later + constructor_fn = functools.partial(import_function, yaml_path=Path(yaml_path)) + + loader = yaml.CLoader if yaml.__with_libyaml__ else yaml.FullLoader + # Add the import_function constructor to the YAML loader + yaml.add_constructor("!function", constructor_fn, Loader=loader) + if yaml_config is None: + with open(yaml_path, "rb") as file: + yaml_config = yaml.load(file, Loader=loader) + + if yaml_dir is None: + yaml_dir = os.path.dirname(yaml_path) + + assert yaml_dir is not None + + if "include" in yaml_config: + include_path = yaml_config["include"] + del yaml_config["include"] + + if isinstance(include_path, str): + include_path = [include_path] + + # Load from the last one first + include_path.reverse() + final_yaml_config = {} + for path in include_path: + # Assumes that path is a full path. + # If not found, assume the included yaml + # is in the same dir as the original yaml + if not os.path.isfile(path): + path = os.path.join(yaml_dir, path) + + try: + included_yaml_config = load_yaml_config(yaml_path=path, mode=mode) + final_yaml_config.update(included_yaml_config) + except Exception as ex: + # If failed to load, ignore + raise ex + + final_yaml_config.update(yaml_config) + return final_yaml_config + return yaml_config + + +def regex_replace(string, pattern, repl, count: int = 0): + """Implements the `re.sub` function as a custom Jinja filter.""" + return re.sub(pattern, repl, string, count=count) + + +env = Environment( + loader=BaseLoader, undefined=StrictUndefined, keep_trailing_newline=True +) +env.filters["regex_replace"] = regex_replace + + +def apply_template(template: str, doc: dict) -> str: + rtemplate = env.from_string(template) + return rtemplate.render(**doc) + + +def create_iterator(raw_iterator, *, rank=0, world_size=1, limit=None): + """ + Method for creating a (potentially) sliced and limited + iterator from a raw document iterator. Used for splitting data + among ranks in multigpu setting or only pulling a sample of documents + """ + return islice(raw_iterator, rank, limit, world_size) + + +def weighted_f1_score(items): + from sklearn.metrics import f1_score + + unzipped_list = list(zip(*items)) + golds = unzipped_list[0] + preds = unzipped_list[1] + fscore = f1_score(golds, preds, average="weighted") + return fscore diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/pyproject.toml b/Prism/Dream/Dream_Baseline/eval_instruct/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..888b91d1ea7ee461452ebd44b9ce10103818770c --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/pyproject.toml @@ -0,0 +1,134 @@ +[build-system] +requires = ["setuptools>=40.8.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "lm_eval" +version = "0.4.8" +authors = [ + {name="EleutherAI", email="contact@eleuther.ai"} +] +description = "A framework for evaluating language models" +readme = "README.md" +classifiers = [ + "Development Status :: 3 - Alpha", + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", +] +requires-python = ">=3.9" +license = { "text" = "MIT" } +dependencies = [ + "accelerate>=0.26.0", + "evaluate", + "datasets>=2.16.0", + "evaluate>=0.4.0", + "jsonlines", + "numexpr", + "peft>=0.2.0", + "pybind11>=2.6.2", + "pytablewriter", + "rouge-score>=0.0.4", + "sacrebleu>=1.5.0", + "scikit-learn>=0.24.1", + "sqlitedict", + "torch>=1.8", + "tqdm-multiprocess", + "transformers>=4.1", + "zstandard", + "dill", + "word2number", + "more_itertools", +] + +[tool.setuptools.packages.find] +include = ["lm_eval*"] + +# required to include yaml files in pip installation +[tool.setuptools.package-data] +lm_eval = ["**/*.yaml", "tasks/**/*"] + +[project.scripts] +lm-eval = "lm_eval.__main__:cli_evaluate" +lm_eval = "lm_eval.__main__:cli_evaluate" + +[project.urls] +Homepage = "https://github.com/EleutherAI/lm-evaluation-harness" +Repository = "https://github.com/EleutherAI/lm-evaluation-harness" + +[project.optional-dependencies] +api = ["requests", "aiohttp", "tenacity", "tqdm", "tiktoken"] +audiolm_qwen = ["librosa", "soundfile"] +deepsparse = ["deepsparse-nightly[llm]>=1.8.0.20240404"] +dev = ["pytest", "pytest-cov", "pytest-xdist", "pre-commit", "mypy", "unitxt"] +gptq = ["auto-gptq[triton]>=0.6.0"] +gptqmodel = ["gptqmodel>=1.0.9"] +hf_transfer = ["hf_transfer"] +ibm_watsonx_ai = ["ibm_watsonx_ai>=1.1.22", "python-dotenv"] +ifeval = ["langdetect", "immutabledict", "nltk>=3.9.1"] +ipex = ["optimum"] +japanese_leaderboard = ["emoji==2.14.0", "neologdn==0.5.3", "fugashi[unidic-lite]", "rouge_score>=0.1.2"] +longbench=["jeiba", "fuzzywuzzy", "rouge"] +mamba = ["mamba_ssm", "causal-conv1d==1.0.2"] +math = ["sympy>=1.12", "antlr4-python3-runtime==4.11", "math_verify[antlr4_11_0]"] +multilingual = ["nagisa>=0.2.7", "jieba>=0.42.1", "pycountry"] +neuronx = ["optimum[neuronx]"] +optimum = ["optimum[openvino]"] +promptsource = ["promptsource>=0.2.3"] +ruler = ["nltk", "wonderwords", "scipy"] +sae_lens = ["sae_lens"] +sentencepiece = ["sentencepiece>=0.1.98"] +sparseml = ["sparseml-nightly[llm]>=1.8.0.20240404"] +sparsify = ["sparsify"] +testing = ["pytest", "pytest-cov", "pytest-xdist"] +vllm = ["vllm>=0.4.2"] +wandb = ["wandb>=0.16.3", "pandas", "numpy"] +zeno = ["pandas", "zeno-client"] +all = [ + "lm_eval[api]", + "lm_eval[audiolm_qwen]", + "lm_eval[deepsparse]", + "lm_eval[dev]", + "lm_eval[gptq]", + "lm_eval[gptqmodel]", + "lm_eval[hf_transfer]", + "lm_eval[ibm_watsonx_ai]", + "lm_eval[ifeval]", + "lm_eval[ipex]", + "lm_eval[japanese_leaderboard]", + "lm_eval[longbench]", + "lm_eval[mamba]", + "lm_eval[math]", + "lm_eval[multilingual]", + "lm_eval[neuronx]", + "lm_eval[optimum]", + "lm_eval[promptsource]", + "lm_eval[ruler]", + "lm_eval[sae_lens]", + "lm_eval[sentencepiece]", + "lm_eval[sparseml]", + "lm_eval[sparsify]", + "lm_eval[testing]", + "lm_eval[vllm]", + "lm_eval[wandb]", + "lm_eval[zeno]", +] + +[tool.pymarkdown] +plugins.md013.enabled = false # line-length +plugins.md024.allow_different_nesting = true # no-duplicate-headers +plugins.md025.enabled = false # single-header +plugins.md028.enabled = false # no-blanks-blockquote +plugins.md029.allow_extended_start_values = true # ol-prefix +plugins.md034.enabled = false # no-bare-urls + +[tool.ruff.lint] +extend-select = ["I"] + +[tool.ruff.lint.isort] +lines-after-imports = 2 +known-first-party = ["lm_eval"] + +[tool.ruff.lint.extend-per-file-ignores] +"__init__.py" = ["F401","F402","F403"] +"utils.py" = ["F401"] diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/requirements.txt b/Prism/Dream/Dream_Baseline/eval_instruct/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..d6e1198b1ab1f5a7f19c6f1fc2ba7338438cf718 --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/requirements.txt @@ -0,0 +1 @@ +-e . diff --git a/Prism/Dream/Dream_Baseline/eval_instruct/setup.py b/Prism/Dream/Dream_Baseline/eval_instruct/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..b5d8fabb8630f2b4c1e00465e4d22d60e8aa00c3 --- /dev/null +++ b/Prism/Dream/Dream_Baseline/eval_instruct/setup.py @@ -0,0 +1,5 @@ +import setuptools + + +# This is to make sure that the package supports editable installs +setuptools.setup() diff --git a/Prism/Dream/Dream_Baseline/metrics/gsmk8_eval.py b/Prism/Dream/Dream_Baseline/metrics/gsmk8_eval.py new file mode 100644 index 0000000000000000000000000000000000000000..28ed927a5d38cb2f9a6e6c1501fb1e79821da959 --- /dev/null +++ b/Prism/Dream/Dream_Baseline/metrics/gsmk8_eval.py @@ -0,0 +1,188 @@ +import json +import re +import os +import glob +import math +import argparse +from collections import Counter + + +RES_PATH = "" + + +def last_boxed_only_string(string): + if not string: return None + idx = string.rfind("\\boxed") + if "\\boxed " in string: + return "\\boxed " + string.split("\\boxed ")[-1].split("$")[0] + if idx < 0: + idx = string.rfind("\\fbox") + if idx < 0: return None + i = idx + right_brace_idx = None + num_left_braces_open = 0 + while i < len(string): + if string[i] == "{": + num_left_braces_open += 1 + if string[i] == "}": + num_left_braces_open -= 1 + if num_left_braces_open == 0: + right_brace_idx = i + break + i += 1 + return string[idx : right_brace_idx + 1] if right_brace_idx else None + +def remove_boxed(s): + if not s: return None + if "\\boxed " in s: return s[len("\\boxed ") :] + if "\\boxed{" in s and s.endswith("}"): return s[len("\\boxed{") : -1] + return s + +def strip_string(string): + if string is None: return "" + string = str(string).strip() + while re.search(r"(\d),(\d{3})", string): + string = re.sub(r"(\d),(\d{3})", r"\1\2", string) + string = string.replace("\n", "").replace("\\!", "") + string = string.replace("tfrac", "frac").replace("dfrac", "frac") + string = string.replace("\\left", "").replace("\\right", "") + string = string.replace("^{\\circ}", "").replace("^\\circ", "") + string = string.replace("\\$", "").replace("\\%", "").replace("\%", "") + if "=" in string and len(string.split("=")[0]) <= 3: + string = string.split("=")[1].strip() + string = string.replace(" ", "") + return string + +def extract_answer_gsm8k(text): + if not text: return "" + boxed = last_boxed_only_string(text) + if boxed: + ans = remove_boxed(boxed) + if ans: return strip_string(ans) + + tag_match = re.search(r"(.*?)", text, re.DOTALL) + if tag_match: + return strip_string(tag_match.group(1)) + + nums = re.findall(r"-?\d+\.?\d*", text[-50:]) + if nums: + return strip_string(nums[-1]) + + return "" + +def extract_gold_gsm8k(target_str): + if "####" in target_str: + return strip_string(target_str.split("####")[-1]) + return strip_string(target_str) + +def is_equiv(pred, gold): + p = strip_string(pred) + g = strip_string(gold) + try: + return math.isclose(float(p), float(g), rel_tol=1e-4) + except: + return p == g + +def run_evaluation(target_path): + if os.path.isdir(target_path): + jsonl_files = glob.glob(os.path.join(target_path, "*.jsonl")) + else: + jsonl_files = [target_path] + + for file_path in jsonl_files: + print(f">>> 正在评测: {file_path}") + detailed_results = [] + correct_count = 0 + total_count = 0 + nfe_list = [] + svf_list = [] + + with open(file_path, 'r', encoding='utf-8') as f: + for line in f: + if not line.strip(): continue + item = json.loads(line) + doc = item.get("doc", {}) + + ground_truth = extract_gold_gsm8k(str(item.get("target", ""))) + nfe_list.append(item.get("nfe", 0)) + svf_list.append(item.get("svf_calls", 0)) + + ans_stats = {} + + trajectories = item.get("all_trajectories", []) + if not trajectories: + resps = item.get("resps", []) + for r in resps: + text = r[0] if isinstance(r, list) else r + trajectories.append({"resp": text, "score": 0.0}) + + for traj in trajectories: + raw_text = traj.get("resp", "") + score = traj.get("score", -float('inf')) + extracted = extract_answer_gsm8k(raw_text) + + if not extracted: continue + + norm = strip_string(extracted) + if norm not in ans_stats: + ans_stats[norm] = {"count": 0, "max_score": -float('inf'), "original": extracted} + + ans_stats[norm]["count"] += 1 + if score > ans_stats[norm]["max_score"]: + ans_stats[norm]["max_score"] = score + ans_stats[norm]["original"] = extracted + + if not ans_stats: + best_pred = "" + else: + sorted_norms = sorted( + ans_stats.keys(), + key=lambda x: (ans_stats[x]["count"], ans_stats[x]["max_score"]), + reverse=True + ) + best_norm = sorted_norms[0] + best_pred = ans_stats[best_norm]["original"] + + ans_correct = is_equiv(best_pred, ground_truth) + if ans_correct: + correct_count += 1 + total_count += 1 + + detailed_results.append({ + "question": doc.get("question", "N/A"), + "final_voted_answer": best_pred, + "ground_truth": ground_truth, + "is_correct": ans_correct, + "nfe": item.get("nfe", 0), + "svf_calls": item.get("svf_calls", 0) + }) + + accuracy = (correct_count / total_count * 100) if total_count > 0 else 0 + + avg_nfe = int(round(sum(nfe_list) / len(nfe_list))) if nfe_list else 0 + avg_svf = int(round(sum(svf_list) / len(svf_list))) if svf_list else 0 + + print(f"Accuracy: {accuracy:.2f}% | NFE: {avg_nfe} | SVF: {avg_svf} ---") + + output_name = f"eval_voted_{os.path.basename(file_path).replace('.jsonl', '.json')}" + output_path = os.path.join(os.path.dirname(file_path), output_name) + + final_report = { + "summary": { + "accuracy": f"{accuracy:.2f}%", + "correct": correct_count, + "total": total_count, + "nfe": avg_nfe, + "svf_calls": avg_svf + }, + "details": detailed_results + } + + with open(output_path, 'w', encoding='utf-8') as out_f: + json.dump(final_report, out_f, ensure_ascii=False, indent=4) + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("-r", "--res_path", type=str, default=RES_PATH) + args = parser.parse_args() + run_evaluation(args.res_path) \ No newline at end of file diff --git a/Prism/Dream/Dream_Baseline/metrics/humaneval_eval.py b/Prism/Dream/Dream_Baseline/metrics/humaneval_eval.py new file mode 100644 index 0000000000000000000000000000000000000000..c68643e27789e2c99183c915c0bdf05daf41e1bf --- /dev/null +++ b/Prism/Dream/Dream_Baseline/metrics/humaneval_eval.py @@ -0,0 +1,234 @@ +import os +import sys +import json +import ast +import re +import glob +import argparse +import textwrap +import evaluate as hf_evaluate +from collections import Counter + +os.environ["HF_ALLOW_CODE_EVAL"] = "1" + +RES_PATH = "" + +def strict_dedent(text: str) -> str: + lines = text.split('\n') + while lines and not lines[0].strip(): lines.pop(0) + while lines and not lines[-1].strip(): lines.pop() + + if not lines: + return "" + + min_indent = None + for line in lines: + if line.strip(): + indent = len(line) - len(line.lstrip()) + if min_indent is None or indent < min_indent: + min_indent = indent + + if min_indent is None: + min_indent = 0 + + dedented_lines = [] + for line in lines: + if line.strip(): + if len(line) >= min_indent: + dedented_lines.append(line[min_indent:]) + else: + dedented_lines.append(line.lstrip()) + else: + dedented_lines.append("") + + return "\n".join(dedented_lines) + +def extract_python_code(text: str) -> str: + if not text: + return "" + + text = text.replace("<|role_end|>", "").replace("<|endoftext|>", "").replace("<|notification_end|>", "") + + tag_match = re.search(r"(.*?)", text, re.DOTALL) + if tag_match: + text = tag_match.group(1) + + code_block_pattern = re.compile(r"```(?:python)?\n?(.*?)```", re.DOTALL) + match = code_block_pattern.search(text) + + if match: + content = match.group(1) + else: + if "```" in text: + content = text.split("```")[0] + else: + lines = text.split('\n') + cleaned_lines = [] + stop_words = ["Explanation:", "Example:", "Test Case:", "Output:", "Here are the tests:"] + for line in lines: + if any(sw in line for sw in stop_words): + break + cleaned_lines.append(line) + content = "\n".join(cleaned_lines) + + return strict_dedent(content) + +def normalize_code(code: str) -> str: + try: + tree = ast.parse(code) + for node in ast.walk(tree): + if isinstance(node, (ast.FunctionDef, ast.ClassDef, ast.Module)): + if (node.body and isinstance(node.body[0], ast.Expr) and + isinstance(node.body[0].value, ast.Constant) and isinstance(node.body[0].value.value, str)): + node.body.pop(0) + return ast.unparse(tree).strip() + except: + return re.sub(r"\s+", "", code) + +def sanitize(prompt: str, completion: str, entry_point: str) -> str: + if f"def {entry_point}" in completion: + imports = [line for line in prompt.split("\n") if line.startswith("import ") or line.startswith("from ")] + return "\n".join(imports) + "\n" + completion + + clean_body = strict_dedent(completion) + if not clean_body: + return prompt + + indented_body = "\n".join([" " + line if line.strip() else "" for line in clean_body.split('\n')]) + return prompt.strip() + "\n" + indented_body + +def perform_majority_voting(trajectories, prompt, entry_point): + candidate_stats = {} + + for item in trajectories: + if isinstance(item, dict): + raw_text = item.get("resp", "") + score = item.get("score", 0.0) + else: + raw_text = str(item[0] if isinstance(item, list) else item) + score = 0.0 + + extracted_code = extract_python_code(raw_text) + full_code = sanitize(prompt, extracted_code, entry_point) + + is_valid = False + try: + ast.parse(full_code) + is_valid = True + except: + is_valid = False + + norm_key = normalize_code(full_code) + if not norm_key: continue + + if norm_key not in candidate_stats: + candidate_stats[norm_key] = { + "count": 0, + "max_score": -float("inf"), + "code": full_code, + "is_valid": is_valid + } + + candidate_stats[norm_key]["count"] += 1 + candidate_stats[norm_key]["max_score"] = max(candidate_stats[norm_key]["max_score"], score) + + if not candidate_stats: + return prompt + + sorted_candidates = sorted( + candidate_stats.values(), + key=lambda x: (x["is_valid"], x["count"], x["max_score"]), + reverse=True + ) + + return sorted_candidates[0]["code"] + +def run_evaluation(target_path): + if os.path.isdir(target_path): + jsonl_files = glob.glob(os.path.join(target_path, "*.jsonl")) + else: + jsonl_files = [target_path] + + try: + code_eval = hf_evaluate.load("code_eval") + except Exception as e: + print(f"Error loading code_eval: {e}") + return + + for file_path in jsonl_files: + print(f">>> 正在评测文件: {file_path}") + + all_voted_predictions = [] + all_references = [] + detailed_logs = [] + + nfe_sum = 0 + svf_sum = 0 + valid_samples = 0 + + with open(file_path, 'r', encoding='utf-8') as f: + for line in f: + if not line.strip(): continue + try: + data = json.loads(line) + except: + continue + + doc = data.get("doc", {}) + task_id = doc.get("task_id", f"Task_{valid_samples}") + prompt = doc.get("prompt", "") + entry_point = doc.get("entry_point", "solution") + test_code = doc.get("test", "") + f"\ncheck({entry_point})" + + nfe_sum += data.get("nfe", 0) + svf_sum += data.get("svf_calls", 0) + valid_samples += 1 + + trajectories = data.get("all_trajectories", data.get("resps", [])) + voted_code = perform_majority_voting(trajectories, prompt, entry_point) + + all_voted_predictions.append([voted_code]) + all_references.append(test_code) + + detailed_logs.append({ + "task_id": task_id, + "entry_point": entry_point, + "final_code": voted_code, + "nfe": data.get("nfe", 0), + "svf": data.get("svf_calls", 0), + }) + + if not all_voted_predictions: continue + + print(f"执行测试中...") + pass_at_k, exec_results = code_eval.compute( + references=all_references, + predictions=all_voted_predictions, + k=[1] + ) + + accuracy = pass_at_k.get("pass@1", 0.0) * 100 + avg_nfe = nfe_sum / valid_samples if valid_samples > 0 else 0 + avg_svf = svf_sum / valid_samples if valid_samples > 0 else 0 + + for i, log in enumerate(detailed_logs): + res = exec_results.get(i, []) + log["passed"] = res[0][1].get("passed", False) if res else False + log["exec_msg"] = res[0][1].get("result", "failed") if res else "failed" + + output_path = file_path.replace(".jsonl", "_voted_result.json") + final_report = { + "meta": {"file": file_path, "total_samples": valid_samples}, + "metrics": {"accuracy": f"{accuracy:.2f}%", "avg_nfe": avg_nfe, "avg_svf": avg_svf}, + "details": detailed_logs + } + + with open(output_path, 'w', encoding='utf-8') as out_f: + json.dump(final_report, out_f, ensure_ascii=False, indent=4) + print(f"Accuracy: {accuracy:.2f}% | SVF: {avg_svf:.1f}\n") + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("-r", "--res_path", type=str, default=RES_PATH) + args = parser.parse_args() + run_evaluation(args.res_path) \ No newline at end of file diff --git a/Prism/Dream/Dream_Baseline/metrics/math500_eval.py b/Prism/Dream/Dream_Baseline/metrics/math500_eval.py new file mode 100644 index 0000000000000000000000000000000000000000..9a091896593d42a14ffc5e05d1726168dca9b14b --- /dev/null +++ b/Prism/Dream/Dream_Baseline/metrics/math500_eval.py @@ -0,0 +1,205 @@ +import json +import re +import os +import math +import argparse +from collections import Counter + +RES_PATH = "" + +def extract_answer(text): + if not text: + return "", False + text = text.replace("<|role_end|>", "").replace("<|endoftext|>", "").strip() + + boxed_pattern = r"\\boxed\{((?:[^{}]|\{(?:[^{}]|\{[^{}]*\})*\})*)\}" + all_boxes = re.findall(boxed_pattern, text) + if all_boxes: + return all_boxes[-1], True + + tag_match = re.search(r"(.*?)", text, re.DOTALL) + if tag_match: + return tag_match.group(1).strip(), True + + marker = "the answer is" + if marker in text.lower(): + pos = text.lower().rfind(marker) + after_text = text[pos + len(marker):].strip() + after_text = re.sub(r"^[:\s]+", "", after_text) + return after_text.split('\n')[0].split('$')[0].strip(), True + + tail = text[-50:].strip() + nums = re.findall(r"(-?\d+[\./\d]*|\\sqrt\{\d+\}|\(-?\d+.*?\))", tail) + if nums: + return nums[-1], False + return "", False + +def normalize_math(string): + if not string: return "" + string = str(string).lower().strip() + + string = string.replace("", "").replace("", "").replace("", "") + string = string.replace("...", "").replace("cannot be determined", "") + + string = re.sub(r"([a-z]+|\\theta|\\alpha|\\pi)\s*=\s*", "", string) + string = re.sub(r"\\text\{([^}]*)\}", r"\1", string) + string = re.sub(r"\\(mathbf|mathrm|bold|unit|mbox|operatorname|mathrm)\{([^}]*)\}", r"\2", string) + string = re.sub(r"\\(d|t)?frac\{([^{}]*)\}\{([^{}]*)\}", r"\2/\3", string) + string = string.replace("\\!", "").replace("\\ ", "").replace("{", "").replace("}", "") + string = string.replace("\\left", "").replace("\\right", "") + string = string.replace("\\$", "").replace("$", "").replace("\\%", "").replace("%", "") + + units_pattern = r"(units?|cm\^2|cm|inches|inch|square|degrees?|radians?|miles?|per|hour|cents?)" + string = re.sub(units_pattern, "", string) + string = string.replace("^{\\circ}", "").replace("^\\circ", "").replace("°", "").replace("\\degree", "") + string = string.replace("\\pi", "pi") + string = re.sub(r"(\d),(\d{3})", r"\1\2", string) + string = string.rstrip(".:,; ").replace(" ", "") + + if "=" in string: + string = string.split("=")[-1] + + return string + +def is_equiv(pred, gold): + if not pred: return False + p, g = normalize_math(pred), normalize_math(gold) + if p == g: return True + + if "=" in pred: + if normalize_math(pred.split("=")[-1]) == g: + return True + + try: + def to_float(s): + if '/' in s and s.count('/') == 1: + parts = s.split('/') + return float(parts[0]) / float(parts[1]) + if '_' in s: s = s.split('_')[0] + return float(s) + return math.isclose(to_float(p), to_float(g), rel_tol=1e-4) + except: + p_fuzzy = re.sub(r"[^a-z0-9/,\-]", "", p) + g_fuzzy = re.sub(r"[^a-z0-9/,\-]", "", g) + return p_fuzzy == g_fuzzy if p_fuzzy else False + +def run_evaluation(target_path): + jsonl_files = [] + if os.path.isdir(target_path): + for root, dirs, files in os.walk(target_path): + for file in files: + if file.endswith(".jsonl") and not file.startswith("eval_voted_"): + jsonl_files.append(os.path.join(root, file)) + else: + jsonl_files = [target_path] + + for file_path in jsonl_files: + print(f">>> 正在评测: {file_path}") + detailed_results = [] + + voted_correct_count = 0 + total_count = 0 + + nfe_list = [] + svf_list = [] + + with open(file_path, 'r', encoding='utf-8') as f: + for line in f: + if not line.strip(): continue + try: + item = json.loads(line) + except: + continue + + doc = item.get("doc", {}) + ground_truth = str(item.get("target", doc.get("answer", ""))) + + current_nfe = item.get("nfe", 0) + nfe_list.append(current_nfe) + current_svf = item.get("svf_calls", 0) + svf_list.append(current_svf) + + ans_stats = {} + trajectories = item.get("all_trajectories", []) + + for traj in trajectories: + raw_text = traj.get("resp", "") + score = traj.get("score", 0) + + extracted, _ = extract_answer(raw_text) + if not extracted: continue + + norm = normalize_math(extracted) + if norm not in ans_stats: + ans_stats[norm] = { + "count": 0, + "max_score": -float('inf'), + "total_weight": 0.0, + "original": extracted + } + + ans_stats[norm]["count"] += 1 + if score > ans_stats[norm]["max_score"]: + ans_stats[norm]["max_score"] = score + + try: + weight = math.exp(score) + except OverflowError: + weight = float('inf') + ans_stats[norm]["total_weight"] += weight + + if not ans_stats: + best_pred = "" + else: + sorted_norms = sorted( + ans_stats.keys(), + key=lambda x: (ans_stats[x]["total_weight"], ans_stats[x]["max_score"], ans_stats[x]["count"]), + reverse=True + ) + best_norm = sorted_norms[0] + best_pred = ans_stats[best_norm]["original"] + + is_voted_correct = False + if best_pred and is_equiv(best_pred, ground_truth): + voted_correct_count += 1 + is_voted_correct = True + + total_count += 1 + + detailed_results.append({ + "question": doc.get("problem", "N/A"), + "final_voted_answer": best_pred, + "ground_truth": ground_truth, + "is_voted_correct": is_voted_correct, + "nfe": current_nfe, + "svf_calls": current_svf + }) + + accuracy = (voted_correct_count / total_count * 100) if total_count > 0 else 0 + + avg_nfe = sum(nfe_list) / len(nfe_list) if nfe_list else 0 + avg_svf = sum(svf_list) / len(svf_list) if svf_list else 0 + + print(f"--- Accuracy : {accuracy:.2f}% | NFE: {avg_nfe:.1f} | SVF: {avg_svf:.1f} ---") + + output_name = f"eval_voted_{os.path.basename(file_path).replace('.jsonl', '.json')}" + output_path = os.path.join(os.path.dirname(file_path), output_name) + + final_report = { + "summary": { + "Accuracy": f"{accuracy:.2f}%", + "correct_voted_count": voted_correct_count, + "total": total_count, + "avg_nfe": avg_nfe, + "avg_svf": avg_svf + }, + "details": detailed_results + } + with open(output_path, 'w', encoding='utf-8') as out_f: + json.dump(final_report, out_f, ensure_ascii=False, indent=4) + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("-r", "--res_path", type=str, default=RES_PATH) + args = parser.parse_args() + run_evaluation(args.res_path) \ No newline at end of file diff --git a/Prism/Dream/Dream_Baseline/metrics/mbpp_eval.py b/Prism/Dream/Dream_Baseline/metrics/mbpp_eval.py new file mode 100644 index 0000000000000000000000000000000000000000..02122b8f5c3a40a1a46675fa8328e60b0b11a8f5 --- /dev/null +++ b/Prism/Dream/Dream_Baseline/metrics/mbpp_eval.py @@ -0,0 +1,281 @@ +import os +import sys +import json +import ast +import re +import glob +import argparse +import textwrap +import evaluate as hf_evaluate +from collections import Counter + +os.environ["HF_ALLOW_CODE_EVAL"] = "1" +os.environ["TOKENIZERS_PARALLELISM"] = "false" + +RES_PATH = "" + +def strict_dedent(text: str) -> str: + lines = text.split('\n') + while lines and not lines[0].strip(): lines.pop(0) + while lines and not lines[-1].strip(): lines.pop() + + if not lines: + return "" + + min_indent = None + for line in lines: + if line.strip(): + indent = len(line) - len(line.lstrip()) + if min_indent is None or indent < min_indent: + min_indent = indent + + if min_indent is None: + min_indent = 0 + + dedented_lines = [] + for line in lines: + if line.strip(): + if len(line) >= min_indent: + dedented_lines.append(line[min_indent:]) + else: + dedented_lines.append(line.lstrip()) + else: + dedented_lines.append("") + + return "\n".join(dedented_lines) + +def extract_python_code(text: str) -> str: + if not text: + return "" + + text = text.replace("<|role_end|>", "").replace("<|endoftext|>", "").replace("<|notification_end|>", "") + text = text.replace("[DONE]", "") + + tag_match = re.search(r"(.*?)", text, re.DOTALL) + if tag_match: + text = tag_match.group(1) + + code_block_pattern = re.compile(r"```(?:python)?\n?(.*?)```", re.DOTALL) + match = code_block_pattern.search(text) + + if match: + content = match.group(1) + else: + if "```" in text: + content = text.split("```")[0] + else: + lines = text.split('\n') + start_idx = 0 + stop_words = ["Here is", "Explanation", "Example", "Note", "python", "The code"] + + for i, line in enumerate(lines): + stripped = line.strip() + if stripped.startswith(("def ", "import ", "from ", "class ")): + start_idx = i + break + if any(sw in line for sw in stop_words) and not stripped.endswith(":"): + continue + + content = "\n".join(lines[start_idx:]) + + return strict_dedent(content) + +def normalize_code(code: str) -> str: + try: + tree = ast.parse(code) + for node in ast.walk(tree): + if isinstance(node, (ast.FunctionDef, ast.ClassDef, ast.Module)): + if (node.body and isinstance(node.body[0], ast.Expr) and + isinstance(node.body[0].value, ast.Constant) and isinstance(node.body[0].value.value, str)): + node.body.pop(0) + return ast.unparse(tree).strip() + except: + return re.sub(r"\s+", "", code) + +def perform_majority_voting(trajectories): + candidate_stats = {} + + for item in trajectories: + if isinstance(item, dict): + raw_text = item.get("resp", "") + score = item.get("score", 0.0) + elif isinstance(item, (list, tuple)): + raw_text = item[0] + score = 0.0 + else: + raw_text = str(item) + score = 0.0 + + extracted_code = extract_python_code(raw_text) + + if not extracted_code.strip(): + continue + + is_valid = False + try: + ast.parse(extracted_code) + is_valid = True + except: + is_valid = False + + norm_key = normalize_code(extracted_code) + if not norm_key: continue + + if norm_key not in candidate_stats: + candidate_stats[norm_key] = { + "count": 0, + "max_score": -float("inf"), + "code": extracted_code, + "is_valid": is_valid + } + + candidate_stats[norm_key]["count"] += 1 + candidate_stats[norm_key]["max_score"] = max(candidate_stats[norm_key]["max_score"], score) + + if not candidate_stats: + return "" + + sorted_candidates = sorted( + candidate_stats.values(), + key=lambda x: (x["is_valid"], x["count"], x["max_score"]), + reverse=True + ) + + return sorted_candidates[0]["code"] + +def run_evaluation(target_path): + if os.path.isdir(target_path): + jsonl_files = glob.glob(os.path.join(target_path, "*.jsonl")) + else: + jsonl_files = [target_path] + + try: + code_eval = hf_evaluate.load("code_eval") + except Exception as e: + print(f"Error loading code_eval: {e}") + return + + for file_path in jsonl_files: + print(f"\n>>> 正在评测 MBPP 文件: {file_path}") + + all_voted_predictions = [] + all_references = [] + detailed_logs = [] + + nfe_total = 0 + svf_total = 0 + count_valid_samples = 0 + + with open(file_path, 'r', encoding='utf-8') as f: + lines = f.readlines() + + for idx, line in enumerate(lines): + if not line.strip(): continue + try: + data = json.loads(line) + except json.JSONDecodeError: + continue + + doc = data.get("doc", {}) + task_id = doc.get("task_id", f"MBPP_{idx}") + + test_list = doc.get("test_list", []) + test_setup = doc.get("test_setup_code", "") + challenge_tests = doc.get("challenge_test_list", []) + + full_test_code = "" + if test_setup: + full_test_code += test_setup + "\n" + if test_list: + full_test_code += "\n".join(test_list) + "\n" + if challenge_tests: + full_test_code += "\n".join(challenge_tests) + + current_nfe = data.get("nfe", 0) + current_svf = data.get("svf_calls", 0) + + nfe_total += current_nfe + svf_total += current_svf + count_valid_samples += 1 + + trajectories = data.get("all_trajectories", []) + if not trajectories: + resps = data.get("resps", []) + trajectories = [{"resp": r} for r in resps] + + voted_code = perform_majority_voting(trajectories) + + if not voted_code: + voted_code = "def placeholder(): pass" + + all_voted_predictions.append([voted_code]) + all_references.append(full_test_code) + + detailed_logs.append({ + "task_id": task_id, + "final_code": voted_code, + "reference": full_test_code, + "nfe": current_nfe, + "svf": current_svf, + "traj_count": len(trajectories) + }) + + if not all_voted_predictions: + print("未找到有效数据。") + continue + + print(f"正在执行代码测试 (共 {len(all_voted_predictions)} 题)...") + + pass_at_k, exec_results = code_eval.compute( + references=all_references, + predictions=all_voted_predictions, + k=[1], + num_workers=4 + ) + + accuracy = pass_at_k.get("pass@1", 0.0) * 100 + avg_nfe = nfe_total / count_valid_samples if count_valid_samples > 0 else 0 + avg_svf = svf_total / count_valid_samples if count_valid_samples > 0 else 0 + print(f"Accuracy: {accuracy:.2f}% | NFE: {avg_nfe:.1f} | SVF: {avg_svf:.1f}") + + for i, log in enumerate(detailed_logs): + res = exec_results.get(i, []) + if res and len(res) > 0: + is_passed = res[0][1].get("passed", False) + eval_result_str = res[0][1].get("result", "passed") if not is_passed else "passed" + else: + is_passed = False + eval_result_str = "Execution Failed" + + log["passed"] = is_passed + log["exec_msg"] = eval_result_str + + output_name = f"eval_mbpp_{os.path.basename(file_path).replace('.jsonl', '.json')}" + output_path = os.path.join(os.path.dirname(file_path), output_name) + + final_report = { + "meta": { + "file": file_path, + "total_samples": count_valid_samples + }, + "metrics": { + "accuracy": f"{accuracy:.2f}%", + "avg_nfe": avg_nfe, + "avg_svf": avg_svf + }, + "details": detailed_logs + } + + with open(output_path, 'w', encoding='utf-8') as out_f: + json.dump(final_report, out_f, ensure_ascii=False, indent=4) + print(f"结果已保存至: {output_path}\n") + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="MBPP Metrics Evaluation Script") + parser.add_argument("-r", "--res_path", type=str, default=RES_PATH, help="Path to jsonl result file or directory") + args = parser.parse_args() + + if os.path.exists(args.res_path): + run_evaluation(args.res_path) + else: + print(f"Path not found: {args.res_path}") \ No newline at end of file diff --git a/Prism/Dream/Dream_Baseline/scripts/run_gsm8k.sh b/Prism/Dream/Dream_Baseline/scripts/run_gsm8k.sh new file mode 100644 index 0000000000000000000000000000000000000000..871a13905b8ce4a80ddbdfbc26cb763404d0d915 --- /dev/null +++ b/Prism/Dream/Dream_Baseline/scripts/run_gsm8k.sh @@ -0,0 +1,31 @@ +#!/bin/bash +set -e +set -x + +PROJECT_ROOT="" +MODEL_PATH="" +BASE_OUTPUT_PATH="${PROJECT_ROOT}/outputs/baseline_dream_gsm8k" + +cd ${PROJECT_ROOT} +export CUDA_VISIBLE_DEVICES=0 +export HF_ENDPOINT=https://hf-mirror.com +export HF_ALLOW_CODE_EVAL=1 +export PYTHONPATH=. + + +TASK="gsm8k" +LENGTH=256 +STEPS=256 +PORT=12334 +NAME="baseline_n1" + +mkdir -p "${BASE_OUTPUT_PATH}/${NAME}" + +accelerate launch --main_process_port ${PORT} -m lm_eval\ + --model diffllm \ + --tasks ${TASK} \ + --batch_size 1 \ + --model_args "pretrained=${MODEL_PATH},trust_remote_code=True,dtype=bfloat16,max_new_tokens=${LENGTH},diffusion_steps=${STEPS}" \ + --gen_kwargs "use_hts=True,initial_N=1,final_K=1,hts_mode=False,task_type=math,temperature=0.7,realtime_output=${BASE_OUTPUT_PATH}/${NAME}/baseline.jsonl" \ + --num_fewshot 0 \ + --output_path "${BASE_OUTPUT_PATH}/${NAME}" \ No newline at end of file diff --git a/Prism/Dream/Dream_Baseline/scripts/run_humaneval.sh b/Prism/Dream/Dream_Baseline/scripts/run_humaneval.sh new file mode 100644 index 0000000000000000000000000000000000000000..b98108a6402f8cbf4158aefeeed725be080b965e --- /dev/null +++ b/Prism/Dream/Dream_Baseline/scripts/run_humaneval.sh @@ -0,0 +1,31 @@ +#!/bin/bash +set -e +set -x + +PROJECT_ROOT="" +MODEL_PATH="" +BASE_OUTPUT_PATH="${PROJECT_ROOT}/outputs/baseline_dream_humaneval" + +cd ${PROJECT_ROOT} +export CUDA_VISIBLE_DEVICES=0,1,2,3 +export HF_ENDPOINT=https://hf-mirror.com +export HF_ALLOW_CODE_EVAL=1 +export PYTHONPATH=. + +TASK="humaneval_instruct" +LENGTH=512 +STEPS=512 +PORT=12334 +NAME="baseline_n1" + +mkdir -p "${BASE_OUTPUT_PATH}/${NAME}" + +accelerate launch --main_process_port ${PORT} -m lm_eval\ + --model diffllm \ + --tasks ${TASK} \ + --batch_size 1 \ + --model_args "pretrained=${MODEL_PATH},trust_remote_code=True,dtype=bfloat16,max_new_tokens=${LENGTH},diffusion_steps=${STEPS}" \ + --gen_kwargs "use_hts=True,initial_N=1,final_K=1,hts_mode=False,task_type=code,temperature=0.7,realtime_output=${BASE_OUTPUT_PATH}/${NAME}/baseline.jsonl" \ + --num_fewshot 0 \ + --confirm_run_unsafe_code \ + --output_path "${BASE_OUTPUT_PATH}/${NAME}" \ No newline at end of file diff --git a/Prism/Dream/Dream_Baseline/scripts/run_math500.sh b/Prism/Dream/Dream_Baseline/scripts/run_math500.sh new file mode 100644 index 0000000000000000000000000000000000000000..8b3d002602efdef72d0e12850de963e347f2e8e7 --- /dev/null +++ b/Prism/Dream/Dream_Baseline/scripts/run_math500.sh @@ -0,0 +1,31 @@ +#!/bin/bash +set -e +set -x + +PROJECT_ROOT="" +MODEL_PATH="" +BASE_OUTPUT_PATH="${PROJECT_ROOT}/outputs/baseline_dream_math500" + +cd ${PROJECT_ROOT} +export CUDA_VISIBLE_DEVICES=0,1 +export HF_ENDPOINT=https://hf-mirror.com +export HF_ALLOW_CODE_EVAL=1 +export PYTHONPATH=. + + +TASK="math500" +LENGTH=256 +STEPS=256 +PORT=12334 +NAME="baseline_n1" + +mkdir -p "${BASE_OUTPUT_PATH}/${NAME}" + +accelerate launch --main_process_port ${PORT} -m lm_eval\ + --model diffllm \ + --tasks ${TASK} \ + --batch_size 1 \ + --model_args "pretrained=${MODEL_PATH},trust_remote_code=True,dtype=bfloat16,max_new_tokens=${LENGTH},diffusion_steps=${STEPS}" \ + --gen_kwargs "use_hts=True,initial_N=1,final_K=1,hts_mode=False,task_type=math,temperature=0.7,realtime_output=${BASE_OUTPUT_PATH}/${NAME}/baseline.jsonl" \ + --num_fewshot 0 \ + --output_path "${BASE_OUTPUT_PATH}/${NAME}" \ No newline at end of file diff --git a/Prism/Dream/Dream_Baseline/scripts/run_mbpp.sh b/Prism/Dream/Dream_Baseline/scripts/run_mbpp.sh new file mode 100644 index 0000000000000000000000000000000000000000..3bf282d012efbf6a40455a25a64413a977cb58b7 --- /dev/null +++ b/Prism/Dream/Dream_Baseline/scripts/run_mbpp.sh @@ -0,0 +1,31 @@ +#!/bin/bash +set -e +set -x + +PROJECT_ROOT="" +MODEL_PATH="" +BASE_OUTPUT_PATH="${PROJECT_ROOT}/outputs/baseline_dream_mbpp" + +cd ${PROJECT_ROOT} +export CUDA_VISIBLE_DEVICES=0,1,2,3 +export HF_ENDPOINT=https://hf-mirror.com +export HF_ALLOW_CODE_EVAL=1 +export PYTHONPATH=. + +TASK="mbpp" +LENGTH=512 +STEPS=512 +PORT=12334 +NAME="baseline_n1" + +mkdir -p "${BASE_OUTPUT_PATH}/${NAME}" + +accelerate launch --main_process_port ${PORT} -m lm_eval\ + --model diffllm \ + --tasks ${TASK} \ + --batch_size 1 \ + --model_args "pretrained=${MODEL_PATH},trust_remote_code=True,dtype=bfloat16,max_new_tokens=${LENGTH},diffusion_steps=${STEPS}" \ + --gen_kwargs "use_hts=True,initial_N=1,final_K=1,hts_mode=False,task_type=code,temperature=0.7,realtime_output=${BASE_OUTPUT_PATH}/${NAME}/baseline.jsonl" \ + --num_fewshot 0 \ + --confirm_run_unsafe_code \ + --output_path "${BASE_OUTPUT_PATH}/${NAME}" \ No newline at end of file diff --git a/Prism/Dream/Dream_Baseline/src/__init__.py b/Prism/Dream/Dream_Baseline/src/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/api/group.py b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/api/group.py new file mode 100644 index 0000000000000000000000000000000000000000..0c60739bbd26c79ecab91f54240798b2ae9e3313 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/api/group.py @@ -0,0 +1,115 @@ +import abc +from dataclasses import asdict, dataclass +from inspect import getsource +from typing import Any, Callable, List, Optional, Union + + +@dataclass +class AggMetricConfig(dict): + metric: Optional[str] = None + aggregation: Optional[str] = "mean" + weight_by_size: Optional[str] = False + # list of filter names which should be incorporated into the aggregated metric. + filter_list: Optional[Union[str, list]] = "none" + + def __post_init__(self): + if self.aggregation != "mean" and not callable(self.aggregation): + raise ValueError( + f"Currently, 'mean' is the only pre-defined aggregation across groups' subtasks. Got '{self.aggregation}'." + ) + + if isinstance(self.filter_list, str): + self.filter_list = [self.filter_list] + + +@dataclass +class GroupConfig(dict): + group: Optional[str] = None + group_alias: Optional[str] = None + task: Optional[Union[str, list]] = None + aggregate_metric_list: Optional[ + Union[List[AggMetricConfig], AggMetricConfig, dict] + ] = None + metadata: Optional[dict] = ( + None # by default, not used in the code. allows for users to pass arbitrary info to tasks + ) + + def __getitem__(self, item): + return getattr(self, item) + + def __setitem__(self, item, value): + return setattr(self, item, value) + + def __post_init__(self): + if self.aggregate_metric_list is not None: + if isinstance(self.aggregate_metric_list, dict): + self.aggregate_metric_list = [self.aggregate_metric_list] + + self.aggregate_metric_list = [ + AggMetricConfig(**item) if isinstance(item, dict) else item + for item in self.aggregate_metric_list + ] + + def to_dict(self, keep_callable: bool = False) -> dict: + """dumps the current config as a dictionary object, as a printable format. + null fields will not be printed. + Used for dumping results alongside full task configuration + + :return: dict + A printable dictionary version of the TaskConfig object. + + # TODO: should any default value in the TaskConfig not be printed? + """ + cfg_dict = asdict(self) + # remove values that are `None` + for k, v in list(cfg_dict.items()): + if callable(v): + cfg_dict[k] = self.serialize_function(v, keep_callable=keep_callable) + return cfg_dict + + def serialize_function( + self, value: Union[Callable, str], keep_callable=False + ) -> Union[Callable, str]: + """Serializes a given function or string. + + If 'keep_callable' is True, the original callable is returned. + Otherwise, attempts to return the source code of the callable using 'getsource'. + """ + if keep_callable: + return value + else: + try: + return getsource(value) + except (TypeError, OSError): + return str(value) + + +class ConfigurableGroup(abc.ABC): + def __init__( + self, + config: Optional[dict] = None, + ) -> None: + self._config = GroupConfig(**config) + + @property + def group(self): + return self._config.group + + @property + def group_alias(self): + return self._config.group_alias + + @property + def version(self): + return self._config.version + + @property + def config(self): + return self._config.to_dict() + + @property + def group_name(self) -> Any: + return self._config.group + + def __repr__(self): + return f"ConfigurableGroup(group={self.group},group_alias={self.group_alias})" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/loggers/__init__.py b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/loggers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..02b7a6834c6486fde35ef02d715e90be3fba223a --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/loggers/__init__.py @@ -0,0 +1,2 @@ +from .evaluation_tracker import EvaluationTracker +from .wandb_logger import WandbLogger diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/loggers/evaluation_tracker.py b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/loggers/evaluation_tracker.py new file mode 100644 index 0000000000000000000000000000000000000000..ef56965d2ad19162811767f50cb4372abe0096b3 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/loggers/evaluation_tracker.py @@ -0,0 +1,524 @@ +import json +import logging +import os +import re +import time +from collections import defaultdict +from dataclasses import asdict, dataclass +from datetime import datetime +from pathlib import Path + +from datasets import load_dataset +from datasets.utils.metadata import MetadataConfigs +from huggingface_hub import ( + DatasetCard, + DatasetCardData, + HfApi, + hf_hub_url, +) +from huggingface_hub.utils import build_hf_headers, get_session, hf_raise_for_status + +from lm_eval.utils import ( + get_file_datetime, + get_file_task_name, + get_results_filenames, + get_sample_results_filenames, + handle_non_serializable, + hash_string, + sanitize_list, + sanitize_model_name, + sanitize_task_name, +) + + +eval_logger = logging.getLogger(__name__) + + +@dataclass(init=False) +class GeneralConfigTracker: + """ + Tracker for the evaluation parameters. + + Attributes: + model_source (str): Source of the model (e.g. Hugging Face, GGUF, etc.) + model_name (str): Name of the model. + model_name_sanitized (str): Sanitized model name for directory creation. + start_time (float): Start time of the experiment. Logged at class init. + end_time (float): Start time of the experiment. Logged when calling [`GeneralConfigTracker.log_end_time`] + total_evaluation_time_seconds (str): Inferred total evaluation time in seconds (from the start and end times). + """ + + model_source: str = None + model_name: str = None + model_name_sanitized: str = None + system_instruction: str = None + system_instruction_sha: str = None + fewshot_as_multiturn: bool = None + chat_template: str = None + chat_template_sha: str = None + start_time: float = None + end_time: float = None + total_evaluation_time_seconds: str = None + + def __init__(self) -> None: + """Starts the evaluation timer.""" + self.start_time = time.perf_counter() + + @staticmethod + def _get_model_name(model_args: str) -> str: + """Extracts the model name from the model arguments.""" + + def extract_model_name(model_args: str, key: str) -> str: + """Extracts the model name from the model arguments using a key.""" + args_after_key = model_args.split(key)[1] + return args_after_key.split(",")[0] + + # order does matter, e.g. peft and delta are provided together with pretrained + prefixes = ["peft=", "delta=", "pretrained=", "model=", "path=", "engine="] + for prefix in prefixes: + if prefix in model_args: + return extract_model_name(model_args, prefix) + return "" + + def log_experiment_args( + self, + model_source: str, + model_args: str, + system_instruction: str, + chat_template: str, + fewshot_as_multiturn: bool, + ) -> None: + """Logs model parameters and job ID.""" + self.model_source = model_source + self.model_name = GeneralConfigTracker._get_model_name(model_args) + self.model_name_sanitized = sanitize_model_name(self.model_name) + self.system_instruction = system_instruction + self.system_instruction_sha = ( + hash_string(system_instruction) if system_instruction else None + ) + self.chat_template = chat_template + self.chat_template_sha = hash_string(chat_template) if chat_template else None + self.fewshot_as_multiturn = fewshot_as_multiturn + + def log_end_time(self) -> None: + """Logs the end time of the evaluation and calculates the total evaluation time.""" + self.end_time = time.perf_counter() + self.total_evaluation_time_seconds = str(self.end_time - self.start_time) + + +class EvaluationTracker: + """ + Keeps track and saves relevant information of the evaluation process. + Compiles the data from trackers and writes it to files, which can be published to the Hugging Face hub if requested. + """ + + def __init__( + self, + output_path: str = None, + hub_results_org: str = "", + hub_repo_name: str = "", + details_repo_name: str = "", + results_repo_name: str = "", + push_results_to_hub: bool = False, + push_samples_to_hub: bool = False, + public_repo: bool = False, + token: str = "", + leaderboard_url: str = "", + point_of_contact: str = "", + gated: bool = False, + ) -> None: + """ + Creates all the necessary loggers for evaluation tracking. + + Args: + output_path (str): Path to save the results. If not provided, the results won't be saved. + hub_results_org (str): The Hugging Face organization to push the results to. If not provided, the results will be pushed to the owner of the Hugging Face token. + hub_repo_name (str): The name of the Hugging Face repository to push the results to. If not provided, the results will be pushed to `lm-eval-results`. + details_repo_name (str): The name of the Hugging Face repository to push the details to. If not provided, the results will be pushed to `lm-eval-results`. + result_repo_name (str): The name of the Hugging Face repository to push the results to. If not provided, the results will not be pushed and will be found in the details_hub_repo. + push_results_to_hub (bool): Whether to push the results to the Hugging Face hub. + push_samples_to_hub (bool): Whether to push the samples to the Hugging Face hub. + public_repo (bool): Whether to push the results to a public or private repository. + token (str): Token to use when pushing to the Hugging Face hub. This token should have write access to `hub_results_org`. + leaderboard_url (str): URL to the leaderboard on the Hugging Face hub on the dataset card. + point_of_contact (str): Contact information on the Hugging Face hub dataset card. + gated (bool): Whether to gate the repository. + """ + self.general_config_tracker = GeneralConfigTracker() + + self.output_path = output_path + self.push_results_to_hub = push_results_to_hub + self.push_samples_to_hub = push_samples_to_hub + self.public_repo = public_repo + self.leaderboard_url = leaderboard_url + self.point_of_contact = point_of_contact + self.api = HfApi(token=token) if token else None + self.gated_repo = gated + + if not self.api and (push_results_to_hub or push_samples_to_hub): + raise ValueError( + "Hugging Face token is not defined, but 'push_results_to_hub' or 'push_samples_to_hub' is set to True. " + "Please provide a valid Hugging Face token by setting the HF_TOKEN environment variable." + ) + + if ( + self.api + and hub_results_org == "" + and (push_results_to_hub or push_samples_to_hub) + ): + hub_results_org = self.api.whoami()["name"] + eval_logger.warning( + f"hub_results_org was not specified. Results will be pushed to '{hub_results_org}'." + ) + + if hub_repo_name == "": + details_repo_name = ( + details_repo_name if details_repo_name != "" else "lm-eval-results" + ) + results_repo_name = ( + results_repo_name if results_repo_name != "" else details_repo_name + ) + else: + details_repo_name = hub_repo_name + results_repo_name = hub_repo_name + eval_logger.warning( + "hub_repo_name was specified. Both details and results will be pushed to the same repository. Using hub_repo_name is no longer recommended, details_repo_name and results_repo_name should be used instead." + ) + + self.details_repo = f"{hub_results_org}/{details_repo_name}" + self.details_repo_private = f"{hub_results_org}/{details_repo_name}-private" + self.results_repo = f"{hub_results_org}/{results_repo_name}" + self.results_repo_private = f"{hub_results_org}/{results_repo_name}-private" + + def save_results_aggregated( + self, + results: dict, + samples: dict, + ) -> None: + """ + Saves the aggregated results and samples to the output path and pushes them to the Hugging Face hub if requested. + + Args: + results (dict): The aggregated results to save. + samples (dict): The samples results to save. + """ + self.general_config_tracker.log_end_time() + + if self.output_path: + try: + eval_logger.info("Saving results aggregated") + + # calculate cumulative hash for each task - only if samples are provided + task_hashes = {} + if samples: + for task_name, task_samples in samples.items(): + sample_hashes = [ + s["doc_hash"] + s["prompt_hash"] + s["target_hash"] + for s in task_samples + ] + task_hashes[task_name] = hash_string("".join(sample_hashes)) + + # update initial results dict + results.update({"task_hashes": task_hashes}) + results.update(asdict(self.general_config_tracker)) + dumped = json.dumps( + results, + indent=2, + default=handle_non_serializable, + ensure_ascii=False, + ) + + path = Path(self.output_path if self.output_path else Path.cwd()) + path = path.joinpath(self.general_config_tracker.model_name_sanitized) + path.mkdir(parents=True, exist_ok=True) + + self.date_id = datetime.now().isoformat().replace(":", "-") + file_results_aggregated = path.joinpath(f"results_{self.date_id}.json") + file_results_aggregated.open("w", encoding="utf-8").write(dumped) + + if self.api and self.push_results_to_hub: + repo_id = ( + self.results_repo + if self.public_repo + else self.results_repo_private + ) + self.api.create_repo( + repo_id=repo_id, + repo_type="dataset", + private=not self.public_repo, + exist_ok=True, + ) + self.api.upload_file( + repo_id=repo_id, + path_or_fileobj=str( + path.joinpath(f"results_{self.date_id}.json") + ), + path_in_repo=os.path.join( + self.general_config_tracker.model_name, + f"results_{self.date_id}.json", + ), + repo_type="dataset", + commit_message=f"Adding aggregated results for {self.general_config_tracker.model_name}", + ) + eval_logger.info( + "Successfully pushed aggregated results to the Hugging Face Hub. " + f"You can find them at: {repo_id}" + ) + + except Exception as e: + eval_logger.warning("Could not save results aggregated") + eval_logger.info(repr(e)) + else: + eval_logger.info( + "Output path not provided, skipping saving results aggregated" + ) + + def save_results_samples( + self, + task_name: str, + samples: dict, + ) -> None: + """ + Saves the samples results to the output path and pushes them to the Hugging Face hub if requested. + + Args: + task_name (str): The task name to save the samples for. + samples (dict): The samples results to save. + """ + if self.output_path: + try: + eval_logger.info(f"Saving per-sample results for: {task_name}") + + path = Path(self.output_path if self.output_path else Path.cwd()) + path = path.joinpath(self.general_config_tracker.model_name_sanitized) + path.mkdir(parents=True, exist_ok=True) + + file_results_samples = path.joinpath( + f"samples_{task_name}_{self.date_id}.jsonl" + ) + + for sample in samples: + # we first need to sanitize arguments and resps + # otherwise we won't be able to load the dataset + # using the datasets library + arguments = {} + for i, arg in enumerate(sample["arguments"]): + arguments[f"gen_args_{i}"] = {} + for j, tmp in enumerate(arg): + arguments[f"gen_args_{i}"][f"arg_{j}"] = tmp + + sample["resps"] = sanitize_list(sample["resps"]) + sample["filtered_resps"] = sanitize_list(sample["filtered_resps"]) + sample["arguments"] = arguments + sample["target"] = str(sample["target"]) + + sample_dump = ( + json.dumps( + sample, + default=handle_non_serializable, + ensure_ascii=False, + ) + + "\n" + ) + + with open(file_results_samples, "a", encoding="utf-8") as f: + f.write(sample_dump) + + if self.api and self.push_samples_to_hub: + repo_id = ( + self.details_repo + if self.public_repo + else self.details_repo_private + ) + self.api.create_repo( + repo_id=repo_id, + repo_type="dataset", + private=not self.public_repo, + exist_ok=True, + ) + try: + if self.gated_repo: + headers = build_hf_headers() + r = get_session().put( + url=f"https://huggingface.co/api/datasets/{repo_id}/settings", + headers=headers, + json={"gated": "auto"}, + ) + hf_raise_for_status(r) + except Exception as e: + eval_logger.warning("Could not gate the repository") + eval_logger.info(repr(e)) + self.api.upload_folder( + repo_id=repo_id, + folder_path=str(path), + path_in_repo=self.general_config_tracker.model_name_sanitized, + repo_type="dataset", + commit_message=f"Adding samples results for {task_name} to {self.general_config_tracker.model_name}", + ) + eval_logger.info( + f"Successfully pushed sample results for task: {task_name} to the Hugging Face Hub. " + f"You can find them at: {repo_id}" + ) + + except Exception as e: + eval_logger.warning("Could not save sample results") + eval_logger.info(repr(e)) + else: + eval_logger.info("Output path not provided, skipping saving sample results") + + def recreate_metadata_card(self) -> None: + """ + Creates a metadata card for the evaluation results dataset and pushes it to the Hugging Face hub. + """ + + eval_logger.info("Recreating metadata card") + repo_id = self.details_repo if self.public_repo else self.details_repo_private + + files_in_repo = self.api.list_repo_files(repo_id=repo_id, repo_type="dataset") + results_files = get_results_filenames(files_in_repo) + sample_files = get_sample_results_filenames(files_in_repo) + + # Build a dictionary to store the latest evaluation datetime for: + # - Each tested model and its aggregated results + # - Each task and sample results, if existing + # i.e. { + # "org__model_name__gsm8k": "2021-09-01T12:00:00", + # "org__model_name__ifeval": "2021-09-01T12:00:00", + # "org__model_name__results": "2021-09-01T12:00:00" + # } + latest_task_results_datetime = defaultdict(lambda: datetime.min.isoformat()) + + for file_path in sample_files: + file_path = Path(file_path) + filename = file_path.name + model_name = file_path.parent + task_name = get_file_task_name(filename) + results_datetime = get_file_datetime(filename) + task_name_sanitized = sanitize_task_name(task_name) + # Results and sample results for the same model and task will have the same datetime + samples_key = f"{model_name}__{task_name_sanitized}" + results_key = f"{model_name}__results" + latest_datetime = max( + latest_task_results_datetime[samples_key], + results_datetime, + ) + latest_task_results_datetime[samples_key] = latest_datetime + latest_task_results_datetime[results_key] = max( + latest_task_results_datetime[results_key], + latest_datetime, + ) + + # Create metadata card + card_metadata = MetadataConfigs() + + # Add the latest aggregated results to the metadata card for easy access + for file_path in results_files: + file_path = Path(file_path) + results_filename = file_path.name + model_name = file_path.parent + eval_date = get_file_datetime(results_filename) + eval_date_sanitized = re.sub(r"[^\w\.]", "_", eval_date) + results_filename = Path("**") / Path(results_filename).name + config_name = f"{model_name}__results" + sanitized_last_eval_date_results = re.sub( + r"[^\w\.]", "_", latest_task_results_datetime[config_name] + ) + + if eval_date_sanitized == sanitized_last_eval_date_results: + # Ensure that all results files are listed in the metadata card + current_results = card_metadata.get(config_name, {"data_files": []}) + current_results["data_files"].append( + {"split": eval_date_sanitized, "path": [str(results_filename)]} + ) + card_metadata[config_name] = current_results + # If the results file is the newest, update the "latest" field in the metadata card + card_metadata[config_name]["data_files"].append( + {"split": "latest", "path": [str(results_filename)]} + ) + + # Add the tasks details configs + for file_path in sample_files: + file_path = Path(file_path) + filename = file_path.name + model_name = file_path.parent + task_name = get_file_task_name(filename) + eval_date = get_file_datetime(filename) + task_name_sanitized = sanitize_task_name(task_name) + eval_date_sanitized = re.sub(r"[^\w\.]", "_", eval_date) + results_filename = Path("**") / Path(filename).name + config_name = f"{model_name}__{task_name_sanitized}" + sanitized_last_eval_date_results = re.sub( + r"[^\w\.]", "_", latest_task_results_datetime[config_name] + ) + if eval_date_sanitized == sanitized_last_eval_date_results: + # Ensure that all sample results files are listed in the metadata card + current_details_for_task = card_metadata.get( + config_name, {"data_files": []} + ) + current_details_for_task["data_files"].append( + {"split": eval_date_sanitized, "path": [str(results_filename)]} + ) + card_metadata[config_name] = current_details_for_task + # If the samples results file is the newest, update the "latest" field in the metadata card + card_metadata[config_name]["data_files"].append( + {"split": "latest", "path": [str(results_filename)]} + ) + + # Get latest results and extract info to update metadata card examples + latest_datetime = max(latest_task_results_datetime.values()) + latest_model_name = max( + latest_task_results_datetime, key=lambda k: latest_task_results_datetime[k] + ) + last_results_file = [ + f for f in results_files if latest_datetime.replace(":", "-") in f + ][0] + last_results_file_path = hf_hub_url( + repo_id=repo_id, filename=last_results_file, repo_type="dataset" + ) + latest_results_file = load_dataset( + "json", data_files=last_results_file_path, split="train" + ) + results_dict = latest_results_file["results"][0] + new_dictionary = {"all": results_dict} + new_dictionary.update(results_dict) + results_string = json.dumps(new_dictionary, indent=4) + + dataset_summary = ( + "Dataset automatically created during the evaluation run of model " + ) + if self.general_config_tracker.model_source == "hf": + dataset_summary += f"[{self.general_config_tracker.model_name}](https://huggingface.co/{self.general_config_tracker.model_name})\n" + else: + dataset_summary += f"{self.general_config_tracker.model_name}\n" + dataset_summary += ( + f"The dataset is composed of {len(card_metadata) - 1} configuration(s), each one corresponding to one of the evaluated task.\n\n" + f"The dataset has been created from {len(results_files)} run(s). Each run can be found as a specific split in each " + 'configuration, the split being named using the timestamp of the run.The "train" split is always pointing to the latest results.\n\n' + 'An additional configuration "results" store all the aggregated results of the run.\n\n' + "To load the details from a run, you can for instance do the following:\n" + ) + if self.general_config_tracker.model_source == "hf": + dataset_summary += ( + "```python\nfrom datasets import load_dataset\n" + f'data = load_dataset(\n\t"{repo_id}",\n\tname="{latest_model_name}",\n\tsplit="latest"\n)\n```\n\n' + ) + dataset_summary += ( + "## Latest results\n\n" + f"These are the [latest results from run {latest_datetime}]({last_results_file_path.replace('/resolve/', '/blob/')}) " + "(note that there might be results for other tasks in the repos if successive evals didn't cover the same tasks. " + 'You find each in the results and the "latest" split for each eval):\n\n' + f"```python\n{results_string}\n```" + ) + card_data = DatasetCardData( + dataset_summary=dataset_summary, + repo_url=f"https://huggingface.co/{self.general_config_tracker.model_name}", + pretty_name=f"Evaluation run of {self.general_config_tracker.model_name}", + leaderboard_url=self.leaderboard_url, + point_of_contact=self.point_of_contact, + ) + card_metadata.to_dataset_card_data(card_data) + card = DatasetCard.from_template( + card_data, + pretty_name=card_data.pretty_name, + ) + card.push_to_hub(repo_id, repo_type="dataset") diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/loggers/utils.py b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/loggers/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c2640953603635d2d96dd85bde9ef14b0175cac2 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/loggers/utils.py @@ -0,0 +1,149 @@ +import logging +import os +import re +import subprocess +from importlib.metadata import version +from pathlib import Path +from typing import Any, Dict, Optional, Tuple, Union + +import numpy as np +from torch.utils.collect_env import get_pretty_env_info +from transformers import __version__ as trans_version + + +logger = logging.getLogger(__name__) + + +def remove_none_pattern(input_string: str) -> Tuple[str, bool]: + """Remove the ',none' substring from the input_string if it exists at the end. + + Args: + input_string (str): The input string from which to remove the ',none' substring. + + Returns: + Tuple[str, bool]: A tuple containing the modified input_string with the ',none' substring removed + and a boolean indicating whether the modification was made (True) or not (False). + """ + # Define the pattern to match ',none' at the end of the string + pattern = re.compile(r",none$") + + # Use sub() to replace ',none' with an empty string + result = re.sub(pattern, "", input_string) + + # check if the input_string changed + removed = result != input_string + + return result, removed + + +def _handle_non_serializable(o: Any) -> Union[int, str, list]: + """Handle non-serializable objects by converting them to serializable types. + + Args: + o (Any): The object to be handled. + + Returns: + Union[int, str, list]: The converted object. If the object is of type np.int64 or np.int32, + it will be converted to int. If the object is of type set, it will be converted + to a list. Otherwise, it will be converted to str. + """ + if isinstance(o, np.int64) or isinstance(o, np.int32): + return int(o) + elif isinstance(o, set): + return list(o) + else: + return str(o) + + +def get_commit_from_path(repo_path: Union[Path, str]) -> Optional[str]: + try: + git_folder = Path(repo_path, ".git") + if git_folder.is_file(): + git_folder = Path( + git_folder.parent, + git_folder.read_text(encoding="utf-8").split("\n")[0].split(" ")[-1], + ) + if Path(git_folder, "HEAD").exists(): + head_name = ( + Path(git_folder, "HEAD") + .read_text(encoding="utf-8") + .split("\n")[0] + .split(" ")[-1] + ) + head_ref = Path(git_folder, head_name) + git_hash = head_ref.read_text(encoding="utf-8").replace("\n", "") + else: + git_hash = None + except Exception as err: + logger.debug( + f"Failed to retrieve a Git commit hash from path: {str(repo_path)}. Error: {err}" + ) + return None + return git_hash + + +def get_git_commit_hash(): + """ + Gets the git commit hash of your current repo (if it exists). + Source: https://github.com/EleutherAI/gpt-neox/blob/b608043be541602170bfcfb8ec9bf85e8a0799e0/megatron/neox_arguments/neox_args.py#L42 + """ + try: + git_hash = subprocess.check_output(["git", "describe", "--always"]).strip() + git_hash = git_hash.decode() + except (subprocess.CalledProcessError, FileNotFoundError): + # FileNotFoundError occurs when git not installed on system + git_hash = get_commit_from_path(os.getcwd()) # git hash of repo if exists + return git_hash + + +def add_env_info(storage: Dict[str, Any]): + try: + pretty_env_info = get_pretty_env_info() + except Exception as err: + pretty_env_info = str(err) + try: + lm_eval_version = version("lm_eval") + except Exception as err: + lm_eval_version = str(err) + transformers_version = trans_version + upper_dir_commit = get_commit_from_path( + Path(os.getcwd(), "..") + ) # git hash of upper repo if exists + added_info = { + "pretty_env_info": pretty_env_info, + "transformers_version": transformers_version, + "lm_eval_version": lm_eval_version, + "upper_git_hash": upper_dir_commit, # in case this repo is submodule + } + storage.update(added_info) + + +def add_tokenizer_info(storage: Dict[str, Any], lm): + if getattr(lm, "tokenizer", False): + try: + tokenizer_info = { + "tokenizer_pad_token": [ + lm.tokenizer.pad_token, + str(lm.tokenizer.pad_token_id), + ], + "tokenizer_eos_token": [ + lm.tokenizer.eos_token, + str(lm.tokenizer.eos_token_id), + ], + "tokenizer_bos_token": [ + lm.tokenizer.bos_token, + str(lm.tokenizer.bos_token_id), + ], + "eot_token_id": getattr(lm, "eot_token_id", None), + "max_length": getattr(lm, "max_length", None), + } + storage.update(tokenizer_info) + except Exception as err: + logger.debug( + f"Logging detailed tokenizer info failed with {err}, skipping..." + ) + # seems gguf and textsynth do not have tokenizer + else: + logger.debug( + "LM does not have a 'tokenizer' attribute, not logging tokenizer metadata to results." + ) diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/loggers/wandb_logger.py b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/loggers/wandb_logger.py new file mode 100644 index 0000000000000000000000000000000000000000..8795f5d8343143ac3badcde98e536c003a1a3fb8 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/loggers/wandb_logger.py @@ -0,0 +1,358 @@ +import copy +import json +import logging +from typing import Any, Dict, List, Literal, Tuple + +import numpy as np +import pandas as pd +from packaging.version import Version + +from lm_eval.loggers.utils import _handle_non_serializable, remove_none_pattern + + +logger = logging.getLogger(__name__) + + +def get_wandb_printer() -> Literal["Printer"]: + """Returns a wandb printer instance for pretty stdout.""" + from wandb.sdk.lib.printer import new_printer + + printer = new_printer() + return printer + + +class WandbLogger: + def __init__(self, init_args=None, config_args=None) -> None: + """Attaches to wandb logger if already initialized. Otherwise, passes init_args to wandb.init() and config_args to wandb.config.update() + + Args: + init_args Optional[Dict]: Arguments for init configuration. + config_args Optional[Dict]: Arguments for config + + Parse and log the results returned from evaluator.simple_evaluate() with: + wandb_logger.post_init(results) + wandb_logger.log_eval_result() + wandb_logger.log_eval_samples(results["samples"]) + """ + try: + import wandb + + assert Version(wandb.__version__) >= Version("0.13.6") + if Version(wandb.__version__) < Version("0.13.6"): + wandb.require("report-editing:v0") + except Exception as e: + logger.warning( + "To use the wandb reporting functionality please install wandb>=0.13.6.\n" + "To install the latest version of wandb run `pip install wandb --upgrade`\n" + f"{e}" + ) + + self.wandb_args: Dict[str, Any] = init_args or {} + self.wandb_config_args: Dict[str, Any] = config_args or {} + + # pop the step key from the args to save for all logging calls + self.step = self.wandb_args.pop("step", None) + + # initialize a W&B run + if wandb.run is None: + self.run = wandb.init(**self.wandb_args) + if self.wandb_config_args: + self.run.config.update(self.wandb_config_args) + else: + self.run = wandb.run + + self.printer = get_wandb_printer() + + def post_init(self, results: Dict[str, Any]) -> None: + self.results: Dict[str, Any] = copy.deepcopy(results) + self.task_names: List[str] = list(results.get("results", {}).keys()) + self.group_names: List[str] = list(results.get("groups", {}).keys()) + + def _get_config(self) -> Dict[str, Any]: + """Get configuration parameters.""" + self.task_configs = self.results.get("configs", {}) + cli_configs = self.results.get("config", {}) + configs = { + "task_configs": self.task_configs, + "cli_configs": cli_configs, + } + + return configs + + def _sanitize_results_dict(self) -> Tuple[Dict[str, str], Dict[str, Any]]: + """Sanitize the results dictionary.""" + _results = copy.deepcopy(self.results.get("results", dict())) + + # Remove None from the metric string name + tmp_results = copy.deepcopy(_results) + for task_name in self.task_names: + task_result = tmp_results.get(task_name, dict()) + for metric_name, metric_value in task_result.items(): + _metric_name, removed = remove_none_pattern(metric_name) + if removed: + _results[task_name][_metric_name] = metric_value + _results[task_name].pop(metric_name) + + # remove string valued keys from the results dict + wandb_summary = {} + for task in self.task_names: + task_result = _results.get(task, dict()) + for metric_name, metric_value in task_result.items(): + if isinstance(metric_value, str): + wandb_summary[f"{task}/{metric_name}"] = metric_value + + for summary_metric, summary_value in wandb_summary.items(): + _task, _summary_metric = summary_metric.split("/") + _results[_task].pop(_summary_metric) + + tmp_results = copy.deepcopy(_results) + for task_name, task_results in tmp_results.items(): + for metric_name, metric_value in task_results.items(): + _results[f"{task_name}/{metric_name}"] = metric_value + _results[task_name].pop(metric_name) + for task in self.task_names: + _results.pop(task) + + return wandb_summary, _results + + def _log_results_as_table(self) -> None: + """Generate and log evaluation results as a table to W&B.""" + columns = [ + "Version", + "Filter", + "num_fewshot", + "Metric", + "Value", + "Stderr", + ] + + def make_table(columns: List[str], key: str = "results"): + import wandb + + table = wandb.Table(columns=columns) + results = copy.deepcopy(self.results) + + for k, dic in results.get(key).items(): + if k in self.group_names and not key == "groups": + continue + version = results.get("versions").get(k) + if version == "N/A": + version = None + n = results.get("n-shot").get(k) + + for (mf), v in dic.items(): + m, _, f = mf.partition(",") + if m.endswith("_stderr"): + continue + if m == "alias": + continue + + if m + "_stderr" + "," + f in dic: + se = dic[m + "_stderr" + "," + f] + if se != "N/A": + se = "%.4f" % se + table.add_data(*[k, version, f, n, m, str(v), str(se)]) + else: + table.add_data(*[k, version, f, n, m, str(v), ""]) + + return table + + # log the complete eval result to W&B Table + table = make_table(["Tasks"] + columns, "results") + self.run.log({"evaluation/eval_results": table}, step=self.step) + + if "groups" in self.results.keys(): + table = make_table(["Groups"] + columns, "groups") + self.run.log({"evaluation/group_eval_results": table}, step=self.step) + + def _log_results_as_artifact(self) -> None: + """Log results as JSON artifact to W&B.""" + import wandb + + dumped = json.dumps( + self.results, indent=2, default=_handle_non_serializable, ensure_ascii=False + ) + artifact = wandb.Artifact("results", type="eval_results") + with artifact.new_file("results.json", mode="w", encoding="utf-8") as f: + f.write(dumped) + self.run.log_artifact(artifact) + + def log_eval_result(self) -> None: + """Log evaluation results to W&B.""" + # Log configs to wandb + configs = self._get_config() + self.run.config.update(configs, allow_val_change=self.step is not None) + + wandb_summary, self.wandb_results = self._sanitize_results_dict() + # update wandb.run.summary with items that were removed + self.run.summary.update(wandb_summary) + # Log the evaluation metrics to wandb + self.run.log(self.wandb_results, step=self.step) + # Log the evaluation metrics as W&B Table + self._log_results_as_table() + # Log the results dict as json to W&B Artifacts + self._log_results_as_artifact() + + def _generate_dataset( + self, data: List[Dict[str, Any]], config: Dict[str, Any] + ) -> pd.DataFrame: + """Generate a dataset from evaluation data. + + Args: + data (List[Dict[str, Any]]): The data to generate a dataset for. + config (Dict[str, Any]): The configuration of the task. + + Returns: + pd.DataFrame: A dataframe that is ready to be uploaded to W&B. + """ + ids = [x["doc_id"] for x in data] + labels = [x["target"] for x in data] + instance = [""] * len(ids) + resps = [""] * len(ids) + filtered_resps = [""] * len(ids) + model_outputs = {} + + metrics_list = config["metric_list"] + metrics = {} + for metric in metrics_list: + metric = metric.get("metric") + if metric in ["word_perplexity", "byte_perplexity", "bits_per_byte"]: + metrics[f"{metric}_loglikelihood"] = [x[metric][0] for x in data] + if metric in ["byte_perplexity", "bits_per_byte"]: + metrics[f"{metric}_bytes"] = [x[metric][1] for x in data] + else: + metrics[f"{metric}_words"] = [x[metric][1] for x in data] + else: + metrics[metric] = [x[metric] for x in data] + + if config["output_type"] == "loglikelihood": + instance = [x["arguments"][0][0] for x in data] + labels = [x["arguments"][0][1] for x in data] + resps = [ + f"log probability of continuation is {x['resps'][0][0][0]} " + + "\n\n" + + "continuation will {} generated with greedy sampling".format( + "not be" if not x["resps"][0][0][1] else "be" + ) + for x in data + ] + filtered_resps = [ + f"log probability of continuation is {x['filtered_resps'][0][0]} " + + "\n\n" + + "continuation will {} generated with greedy sampling".format( + "not be" if not x["filtered_resps"][0][1] else "be" + ) + for x in data + ] + elif config["output_type"] == "multiple_choice": + instance = [x["arguments"][0][0] for x in data] + choices = [ + "\n".join([f"{idx}. {y[1]}" for idx, y in enumerate(x["arguments"])]) + for x in data + ] + resps = [np.argmax([n[0][0] for n in x["resps"]]) for x in data] + filtered_resps = [ + np.argmax([n[0] for n in x["filtered_resps"]]) for x in data + ] + elif config["output_type"] == "loglikelihood_rolling": + instance = [x["arguments"][0][0] for x in data] + resps = [x["resps"][0][0] for x in data] + filtered_resps = [x["filtered_resps"][0] for x in data] + elif config["output_type"] == "generate_until": + instance = [x["arguments"][0][0] for x in data] + resps = [x["resps"][0][0] for x in data] + filtered_resps = [x["filtered_resps"][0] for x in data] + + model_outputs["raw_predictions"] = resps + model_outputs["filtered_predictions"] = filtered_resps + + df_data = { + "id": ids, + "data": instance, + } + if config["output_type"] == "multiple_choice": + df_data["choices"] = choices + + tmp_data = { + "input_len": [len(x) for x in instance], + "labels": labels, + "output_type": config["output_type"], + } + df_data.update(tmp_data) + df_data.update(model_outputs) + df_data.update(metrics) + + return pd.DataFrame(df_data) + + def _log_samples_as_artifact( + self, data: List[Dict[str, Any]], task_name: str + ) -> None: + import wandb + + # log the samples as an artifact + dumped = json.dumps( + data, + indent=2, + default=_handle_non_serializable, + ensure_ascii=False, + ) + artifact = wandb.Artifact(f"{task_name}", type="samples_by_task") + with artifact.new_file( + f"{task_name}_eval_samples.json", mode="w", encoding="utf-8" + ) as f: + f.write(dumped) + self.run.log_artifact(artifact) + # artifact.wait() + + def log_eval_samples(self, samples: Dict[str, List[Dict[str, Any]]]) -> None: + """Log evaluation samples to W&B. + + Args: + samples (Dict[str, List[Dict[str, Any]]]): Evaluation samples for each task. + """ + task_names: List[str] = [ + x for x in self.task_names if x not in self.group_names + ] + + ungrouped_tasks = [] + tasks_by_groups = {} + + for task_name in task_names: + group_names = self.task_configs[task_name].get("group", None) + if group_names: + if isinstance(group_names, str): + group_names = [group_names] + + for group_name in group_names: + if not tasks_by_groups.get(group_name): + tasks_by_groups[group_name] = [task_name] + else: + tasks_by_groups[group_name].append(task_name) + else: + ungrouped_tasks.append(task_name) + + for task_name in ungrouped_tasks: + eval_preds = samples[task_name] + + # log the samples as a W&B Table + df = self._generate_dataset(eval_preds, self.task_configs.get(task_name)) + self.run.log({f"{task_name}_eval_results": df}, step=self.step) + + # log the samples as a json file as W&B Artifact + self._log_samples_as_artifact(eval_preds, task_name) + + for group, grouped_tasks in tasks_by_groups.items(): + grouped_df = pd.DataFrame() + for task_name in grouped_tasks: + eval_preds = samples[task_name] + df = self._generate_dataset( + eval_preds, self.task_configs.get(task_name) + ) + df["group"] = group + df["task"] = task_name + grouped_df = pd.concat([grouped_df, df], ignore_index=True) + + # log the samples as a json file as W&B Artifact + self._log_samples_as_artifact(eval_preds, task_name) + + self.run.log({f"{group}_eval_results": grouped_df}, step=self.step) diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/prompts/__init__.py b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/prompts/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e1ad14105c1b2cd6f9985a9cba796face12b9bca --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/prompts/__init__.py @@ -0,0 +1,128 @@ +import ast +import logging +import os +from typing import Dict + +from lm_eval import utils + + +eval_logger = logging.getLogger(__name__) + +# Prompt library. +# Stores prompts in a dictionary indexed by 2 levels: +# prompt category name, and prompt name. +# This allows us to access prompts +PROMPT_REGISTRY: Dict[str, Dict[str, str]] = { + "qa-basic": { + "question-newline-answer": "Question: {{question}}\nAnswer:", + "q-newline-a": "Q: {{question}}\nA:", + }, +} + + +def get_prompt(prompt_id: str, dataset_name: str = None, subset_name: str = None): + # unpack prompt name + category_name, prompt_name = prompt_id.split(":") + if subset_name is None: + dataset_full_name = dataset_name + else: + dataset_full_name = f"{dataset_name}-{subset_name}" + eval_logger.info(f"Loading prompt from {category_name} for {dataset_full_name}") + if category_name == "promptsource": + try: + from promptsource.templates import DatasetTemplates + except ModuleNotFoundError as exception: + raise type(exception)( + "Tried to load a Promptsource template, but promptsource is not installed ", + "please install promptsource via pip install lm-eval[promptsource] or pip install -e .[promptsource]", + ) + try: + if subset_name is None: + prompts = DatasetTemplates(dataset_name=dataset_name) + else: + prompts = DatasetTemplates( + dataset_name=dataset_name, subset_name=subset_name + ) + except Exception: + raise ValueError(f"{dataset_name} and {subset_name} not found") + if prompt_name in prompts.all_template_names: + return prompts[prompt_name] + else: + raise ValueError( + f"{prompt_name} not in prompt list {prompts.all_template_names}" + ) + elif ".yaml" in category_name: + import yaml + + with open(category_name, "rb") as file: + prompt_yaml_file = yaml.full_load(file) + + prompt_string = prompt_yaml_file["prompts"][prompt_name] + return PromptString(prompt_string) + else: + try: + return PROMPT_REGISTRY[category_name][prompt_name] + except Exception: + raise ValueError( + f"expected only a single `:` as separator between \ + prompt category and name, but got `{prompt_id}` instead" + ) + + +def load_prompt_list( + use_prompt: str, dataset_name=None, subset_name=None, yaml_path=None, **kwargs +): + category_name, prompt_name = use_prompt.split(":") + + if category_name == "promptsource": + from promptsource.templates import DatasetTemplates + + if subset_name is None: + prompts = DatasetTemplates(dataset_name=dataset_name) + else: + prompts = DatasetTemplates( + dataset_name=dataset_name, subset_name=subset_name + ) + + prompt_list = utils.pattern_match(prompt_name, prompts.all_template_names) + + elif ".yaml" in category_name: + import yaml + + if yaml_path is not None: + category_name = os.path.realpath(os.path.join(yaml_path, category_name)) + + with open(category_name, "rb") as file: + prompt_yaml_file = yaml.full_load(file) + + prompt_list = utils.pattern_match( + prompt_name, prompt_yaml_file["prompts"].keys() + ) + + # category_name, *prompt_name = use_prompt.split(":") + # TODO allow to multiple prompt naming + # if len(prompt_name) > 1: + # prompt_list = [] + # for prompt in prompt_name: + # prompt_list.append(utils.pattern_match(prompt_name, prompts.all_template_names)) + # else: + # prompt_list = utils.pattern_match(prompt_name, prompts.all_template_names) + return [":".join([category_name, prompt]) for prompt in prompt_list] + + +class PromptString: + def __init__(self, prompt_string): + self.prompt_string = prompt_string + + def apply(self, doc): + doc_to_text = self.prompt_string["doc_to_text"] + doc_to_target = self.prompt_string["doc_to_target"] + + # TODO need a way to process doc_to_choice + if "doc_to_choice" in self.prompt_string: + raise NotImplementedError("Not yet implemented to accept doc_to_choice") + + text_string = utils.apply_template(doc_to_text, doc) + target_string = utils.apply_template(doc_to_target, doc) + + return [text_string, target_string] diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/README.md b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/README.md new file mode 100644 index 0000000000000000000000000000000000000000..dd6403428d6492332077b195853fbfca574634ef --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/README.md @@ -0,0 +1,165 @@ + +# Tasks + + A list of supported tasks and task groupings can be viewed with `lm-eval --tasks list`. + + For more information, including a full list of task names and their precise meanings or sources, follow the links provided to the individual README.md files for each subfolder. + +| Task Family | Description | Language(s) | +|--------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------| +| [aclue](aclue/README.md) | Tasks focusing on ancient Chinese language understanding and cultural aspects. | Ancient Chinese | +| [aexams](aexams/README.md) | Tasks in Arabic related to various academic exams covering a range of subjects. | Arabic | +| [agieval](agieval/README.md) | Tasks involving historical data or questions related to history and historical texts. | English, Chinese | +| [anli](anli/README.md) | Adversarial natural language inference tasks designed to test model robustness. | English | +| [arabic_leaderboard_complete](arabic_leaderboard_complete/README.md) | A full version of the tasks in the Open Arabic LLM Leaderboard, focusing on the evaluation of models that reflect the characteristics of Arabic language understanding and comprehension, culture, and heritage. Note that some of these tasks are machine-translated. | Arabic (Some MT) | +| [arabic_leaderboard_light](arabic_leaderboard_light/README.md) | A light version of the tasks in the Open Arabic LLM Leaderboard (i.e., 10% samples of the test set in the original benchmarks), focusing on the evaluation of models that reflect the characteristics of Arabic language understanding and comprehension, culture, and heritage. Note that some of these tasks are machine-translated. | Arabic (Some MT) | +| [arabicmmlu](arabicmmlu/README.md) | Localized Arabic version of MMLU with multiple-choice questions from 40 subjects. | Arabic | +| [AraDICE](aradice/README.md) | A collection of multiple tasks carefully designed to evaluate dialectal and cultural capabilities in large language models (LLMs). | Arabic | +| [arc](arc/README.md) | Tasks involving complex reasoning over a diverse set of questions. | English | +| [arithmetic](arithmetic/README.md) | Tasks involving numerical computations and arithmetic reasoning. | English | +| [asdiv](asdiv/README.md) | Tasks involving arithmetic and mathematical reasoning challenges. | English | +| [babi](babi/README.md) | Tasks designed as question and answering challenges based on simulated stories. | English | +| [basque_bench](basque_bench/README.md) | Collection of tasks in Basque encompassing various evaluation areas. | Basque | +| [basqueglue](basqueglue/README.md) | Tasks designed to evaluate language understanding in Basque language. | Basque | +| [bbh](bbh/README.md) | Tasks focused on deep semantic understanding through hypothesization and reasoning. | English, German | +| [bbq](bbq/README.md) | A question-answering benchmark designed to measure social biases in language models across various demographic categories and contexts. | English | +| [belebele](belebele/README.md) | Language understanding tasks in a variety of languages and scripts. | Multiple (122 languages) | +| benchmarks | General benchmarking tasks that test a wide range of language understanding capabilities. | | +| [bertaqa](bertaqa/README.md) | Local Basque cultural trivia QA tests in English and Basque languages. | English, Basque, Basque (MT) | +| [bigbench](bigbench/README.md) | Broad tasks from the BIG-bench benchmark designed to push the boundaries of large models. | Multiple | +| [blimp](blimp/README.md) | Tasks testing grammatical phenomena to evaluate language model's linguistic capabilities. | English | +| [careqa](careqa/README.md) | Multiple choice and open-ended medical question answering based on the Spanish Specialised Healthcare Training (MIR) exams. | English, Spanish | +| [catalan_bench](catalan_bench/README.md) | Collection of tasks in Catalan encompassing various evaluation areas. | Catalan | +| [ceval](ceval/README.md) | Tasks that evaluate language understanding and reasoning in an educational context. | Chinese | +| [cmmlu](cmmlu/README.md) | Multi-subject multiple choice question tasks for comprehensive academic assessment. | Chinese | +| code_x_glue | Tasks that involve understanding and generating code across multiple programming languages. | Go, Java, JS, PHP, Python, Ruby | +| [commonsense_qa](commonsense_qa/README.md) | CommonsenseQA, a multiple-choice QA dataset for measuring commonsense knowledge. | English | +| [copal_id](copal_id/README.md) United States | Indonesian causal commonsense reasoning dataset that captures local nuances. | Indonesian | +| [coqa](coqa/README.md) | Conversational question answering tasks to test dialog understanding. | English | +| [crows_pairs](crows_pairs/README.md) | Tasks designed to test model biases in various sociodemographic groups. | English, French | +| csatqa | Tasks related to SAT and other standardized testing questions for academic assessment. | Korean | +| [drop](drop/README.md) | Tasks requiring numerical reasoning, reading comprehension, and question answering. | English | +| [eq_bench](eq_bench/README.md) | Tasks focused on equality and ethics in question answering and decision-making. | English | +| [eus_exams](eus_exams/README.md) | Tasks based on various professional and academic exams in the Basque language. | Basque | +| [eus_proficiency](eus_proficiency/README.md) | Tasks designed to test proficiency in the Basque language across various topics. | Basque | +| [eus_reading](eus_reading/README.md) | Reading comprehension tasks specifically designed for the Basque language. | Basque | +| [eus_trivia](eus_trivia/README.md) | Trivia and knowledge testing tasks in the Basque language. | Basque | +| [evalita-LLM](evalita-LLM/README.md) | A native Italian benchmark with diverse tasks formats and multiple prompts. | Italian | +| [fda](fda/README.md) | Tasks for extracting key-value pairs from FDA documents to test information extraction. | English | +| [fld](fld/README.md) | Tasks involving free-form and directed dialogue understanding. | English | +| [french_bench](french_bench/README.md) | Set of tasks designed to assess language model performance in French. | French | +| [galician_bench](galician_bench/README.md) | Collection of tasks in Galician encompassing various evaluation areas. | Galician | +| [global_mmlu](global_mmlu/README.md) | Collection of culturally sensitive and culturally agnostic MMLU tasks in 15 languages with human translations or post-edits. | Multiple (15 languages) | +| [glue](glue/README.md) | General Language Understanding Evaluation benchmark to test broad language abilities. | English | +| [gpqa](gpqa/README.md) | Tasks designed for general public question answering and knowledge verification. | English | +| [gsm8k](gsm8k/README.md) | A benchmark of grade school math problems aimed at evaluating reasoning capabilities. | English | +| [groundcocoa](groundcocoa/README.md) | A benchmark evaluating the conditional and compositional reasoning of language models using a grounding task. | English | +| [haerae](haerae/README.md) | Tasks focused on assessing detailed factual and historical knowledge. | Korean | +| [headqa](headqa/README.md) | A high-level education-based question answering dataset to test specialized knowledge. | Spanish, English | +| [hellaswag](hellaswag/README.md) | Tasks to predict the ending of stories or scenarios, testing comprehension and creativity. | English | +| [hendrycks_ethics](hendrycks_ethics/README.md) | Tasks designed to evaluate the ethical reasoning capabilities of models. | English | +| [hendrycks_math](hendrycks_math/README.md) | Mathematical problem-solving tasks to test numerical reasoning and problem-solving. | English | +| [histoires_morales](histoires_morales/README.md) | A dataset of structured narratives that describe normative and norm-divergent actions taken by individuals to accomplish certain intentions in concrete situations. | French (Some MT) | +| [hrm8k](hrm8k/README.md) | A challenging bilingual math reasoning benchmark for Korean and English. | Korean (Some MT), English (Some MT) | +| [humaneval](humaneval/README.md) | Code generation task that measure functional correctness for synthesizing programs from docstrings. | Python | +| [ifeval](ifeval/README.md) | Interactive fiction evaluation tasks for narrative understanding and reasoning. | English | +| [inverse_scaling](inverse_scaling/README.md) | Multiple-choice tasks from the Inverse Scaling Prize, designed to find settings where larger language models perform worse. | English | +| [japanese_leaderboard](japanese_leaderboard/README.md) | Japanese language understanding tasks to benchmark model performance on various linguistic aspects. | Japanese | +| [kbl](kbl/README.md) | Korean Benchmark for Legal Language Understanding. | Korean | +| [kmmlu](kmmlu/README.md) | Knowledge-based multi-subject multiple choice questions for academic evaluation. | Korean | +| [kobest](kobest/README.md) | A collection of tasks designed to evaluate understanding in Korean language. | Korean | +| [kormedmcqa](kormedmcqa/README.md) | Medical question answering tasks in Korean to test specialized domain knowledge. | Korean | +| [lambada](lambada/README.md) | Tasks designed to predict the endings of text passages, testing language prediction skills. | English | +| [lambada_cloze](lambada_cloze/README.md) | Cloze-style LAMBADA dataset. | English | +| [lambada_multilingual](lambada_multilingual/README.md) | Multilingual LAMBADA dataset. This is a legacy version of the multilingual dataset, and users should instead use `lambada_multilingual_stablelm`. | German, English, Spanish, French, Italian | +| [lambada_multilingual_stablelm](lambada_multilingual_stablelm/README.md) | Multilingual LAMBADA dataset. Users should prefer evaluating on this version of the multilingual dataset instead of on `lambada_multilingual`. | German, English, Spanish, French, Italian, Dutch, Portuguese | +| [leaderboard](leaderboard/README.md) | Task group used by Hugging Face's [Open LLM Leaderboard v2](https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard). Those tasks are static and will not change through time | English | +| [lingoly](lingoly/README.md) | Challenging logical reasoning benchmark in low-resource languages with controls for memorization | English, Multilingual | +| [logiqa](logiqa/README.md) | Logical reasoning tasks requiring advanced inference and deduction. | English, Chinese | +| [logiqa2](logiqa2/README.md) | Large-scale logical reasoning dataset adapted from the Chinese Civil Service Examination. | English, Chinese | +| [mastermind](mastermind/README.md) | Reasoning benchmark based on the board game of Mastermind. | English | +| [mathqa](mathqa/README.md) | Question answering tasks involving mathematical reasoning and problem-solving. | English | +| [mbpp](mbpp/README.md) | A benchmark designed to measure the ability to synthesize short Python programs from natural language descriptions. | Python | +| [meddialog](meddialog/README.md) | Medical open-ended QA and Question Entailment stemming from the MedDialog dataset. | English | +| [medtext](medtext/README.md) | Medical open-ended QA from the MedText Clinical Notes dataset. | English | +| [mimic_repsum](mimic_repsum/README.md) | Medical report summarization from the MIMIC-III dataset. | English | +| [mc_taco](mc_taco/README.md) | Question-answer pairs that require temporal commonsense comprehension. | English | +| [med_concepts_qa](med_concepts_qa/README.md) | Benchmark for evaluating LLMs on their abilities to interpret medical codes and distinguish between medical concept. | English | +| [metabench](metabench/README.md) | Distilled versions of six popular benchmarks which are highly predictive of overall benchmark performance and of a single general ability latent trait. | English | +| [mediqa_qa2019](mediqa_qa2019/README.md) | Open-ended healthcare question answering benchmark from the MEDIQA 2019 challenge. | English | +| medmcqa | Medical multiple choice questions assessing detailed medical knowledge. | English | +| medqa | Multiple choice question answering based on the United States Medical License Exams. | | +| [meqsum](meqsum/README.md) | Healtcare Question Entailment benchmark from the MeqSum dataset. | | +| [mgsm](mgsm/README.md) | Benchmark of multilingual grade-school math problems. | Spanish, French, German, Russian, Chinese, Japanese, Thai, Swahili, Bengali, Telugu | +| [minerva_math](minerva_math/README.md) | Mathematics-focused tasks requiring numerical reasoning and problem-solving skills. | English | +| [mlqa](mlqa/README.md) | MultiLingual Question Answering benchmark dataset for evaluating cross-lingual question answering performance. | English, Arabic, German, Spanish, Hindi, Vietnamese, Simplified Chinese | +| [mmlu](mmlu/README.md) | Massive Multitask Language Understanding benchmark for broad domain language evaluation. Several variants are supported. | English | +| [mmlu_pro](mmlu_pro/README.md) | A refined set of MMLU, integrating more challenging, reasoning-focused questions and expanding the choice set from four to ten options. | English | +| [mmlu-pro-plus](mmlu-pro-plus/README.md) | A new test set for evaluating shortcut learning and higher-order reasoning of LLMs. | English | +| [mmlu_prox](mmlu_prox/README.md) | A multilingual benchmark that extends MMLU-Pro to multiple typologically diverse languages with human validation. | English, Japanese, Chinese, Korean, French, German, Spanish, Portuguese, Swahili, Thai, Arabic, Hindi, Bengali | +| [mmlusr](mmlusr/README.md) | Variation of MMLU designed to be more rigorous. | English | +| model_written_evals | Evaluation tasks auto-generated for evaluating a collection of AI Safety concerns. | | +| [moral_stories](moral_stories/README.md) | A crowd-sourced dataset of structured narratives that describe normative and norm-divergent actions taken by individuals to accomplish certain intentions in concrete situations. | English +| [mts_dialog](mts_dialog/README.md) | Open-ended healthcare QA from the MTS-Dialog dataset. | English | +| [mutual](mutual/README.md) | A retrieval-based dataset for multi-turn dialogue reasoning. | English | +| [nq_open](nq_open/README.md) | Open domain question answering tasks based on the Natural Questions dataset. | English | +| [okapi/arc_multilingual](okapi/arc_multilingual/README.md) | Tasks that involve reading comprehension and information retrieval challenges. | Multiple (31 languages) **Machine Translated.** | +| [okapi/hellaswag_multilingual](okapi/hellaswag_multilingual/README.md) | Tasks that involve reading comprehension and information retrieval challenges. | Multiple (30 languages) **Machine Translated.** | +| okapi/mmlu_multilingual | Tasks that involve reading comprehension and information retrieval challenges. | Multiple (34 languages) **Machine Translated.** | +| [okapi/truthfulqa_multilingual](okapi/truthfulqa_multilingual/README.md) | Tasks that involve reading comprehension and information retrieval challenges. | Multiple (31 languages) **Machine Translated.** | +| [olaph](olaph/README.md) | Open-ended medical factuality Question Answering from the OLAPH dataset. | English | +| [openbookqa](openbookqa/README.md) | Open-book question answering tasks that require external knowledge and reasoning. | English | +| [paloma](paloma/README.md) | Paloma is a comprehensive benchmark designed to evaluate open language models across a wide range of domains, ranging from niche artist communities to mental health forums on Reddit. | English | +| [paws-x](paws-x/README.md) | Paraphrase Adversaries from Word Scrambling, focusing on cross-lingual capabilities. | English, French, Spanish, German, Chinese, Japanese, Korean | +| [pile](pile/README.md) | Open source language modelling data set that consists of 22 smaller, high-quality datasets. | English | +| [pile_10k](pile_10k/README.md) | The first 10K elements of The Pile, useful for debugging models trained on it. | English | +| [piqa](piqa/README.md) | Physical Interaction Question Answering tasks to test physical commonsense reasoning. | English | +| [polemo2](polemo2/README.md) | Sentiment analysis and emotion detection tasks based on Polish language data. | Polish | +| [portuguese_bench](portuguese_bench/README.md) | Collection of tasks in European Portuguese encompassing various evaluation areas. | Portuguese | +| [prost](prost/README.md) | Tasks requiring understanding of professional standards and ethics in various domains. | English | +| [pubmedqa](pubmedqa/README.md) | Question answering tasks based on PubMed research articles for biomedical understanding. | English | +| [qa4mre](qa4mre/README.md) | Question Answering for Machine Reading Evaluation, assessing comprehension and reasoning. | English | +| [qasper](qasper/README.md) | Question Answering dataset based on academic papers, testing in-depth scientific knowledge. | English | +| [race](race/README.md) | Reading comprehension assessment tasks based on English exams in China. | English | +| realtoxicityprompts | Tasks to evaluate language models for generating text with potential toxicity. | | +| [ruler](ruler/README.md) | RULER is a benchmark for testing how well language models handle long pieces of text. Requires custom arg (see readme) | English | +| [sciq](sciq/README.md) | Science Question Answering tasks to assess understanding of scientific concepts. | English | +| [score](score/README.md) | Systematic consistency and robustness evaluation for LLMs on 3 datasets(MMLU-Pro, Agi Eval and MATH) | English | +| [scrolls](scrolls/README.md) | Tasks that involve long-form reading comprehension across various domains. | English | +| [simple_cooccurrence_bias](simple_cooccurrence_bias/README.md) | A metric that evaluates language models for biases based on stereotypical word associations and co-occurrences in text. | English | +| [siqa](siqa/README.md) | Social Interaction Question Answering to evaluate common sense and social reasoning. | English | +| [spanish_bench](spanish_bench/README.md) | Collection of tasks in Spanish encompassing various evaluation areas. | Spanish | +| [squad_completion](squad_completion/README.md) | A variant of the SQuAD question answering task designed for zero-shot evaluation of small LMs. | English | +| [squadv2](squadv2/README.md) | Stanford Question Answering Dataset version 2, a reading comprehension benchmark. | English | +| [storycloze](storycloze/README.md) | Tasks to predict story endings, focusing on narrative logic and coherence. | English | +| [super_glue](super_glue/README.md) | A suite of challenging tasks designed to test a range of language understanding skills. | English | +| [swag](swag/README.md) | Situations With Adversarial Generations, predicting the next event in videos. | English | +| [swde](swde/README.md) | Information extraction tasks from semi-structured web pages. | English | +| [tinyBenchmarks](tinyBenchmarks/README.md) | Evaluation of large language models with fewer examples using tiny versions of popular benchmarks. | English | +| [tmmluplus](tmmluplus/README.md) | An extended set of tasks under the TMMLU framework for broader academic assessments. | Traditional Chinese | +| [toxigen](toxigen/README.md) | Tasks designed to evaluate language models on their propensity to generate toxic content. | English | +| [translation](translation/README.md) | Tasks focused on evaluating the language translation capabilities of models. | Arabic, English, Spanish, Basque, Hindi, Indonesian, Burmese, Russian, Swahili, Telugu, Chinese | +| [triviaqa](triviaqa/README.md) | A large-scale dataset for trivia question answering to test general knowledge. | English | +| [truthfulqa](truthfulqa/README.md) | A QA task aimed at evaluating the truthfulness and factual accuracy of model responses. | English | +| [turkishmmlu](turkishmmlu/README.md) | A multiple-choice QA test modeled after MMLU, written in Turkish based on Turkish high-school level exams. | Turkish | +| [unitxt](unitxt/README.md) | A number of tasks implemented using the unitxt library for flexible, shareable, and reusable data preparation and evaluation for generative AI. | English | +| [unscramble](unscramble/README.md) | Tasks involving the rearrangement of scrambled sentences to test syntactic understanding. | English | +| [webqs](webqs/README.md) | Web-based question answering tasks designed to evaluate internet search and retrieval. | English | +| [wikitext](wikitext/README.md) | Tasks based on text from Wikipedia articles to assess language modeling and generation. | English | +| [winogender](winogender/README.md) | A diagnostic dataset that tests for gender bias in coreference resolution by measuring how models associate pronouns with different occupations. | English | +| [winogrande](winogrande/README.md) | A large-scale dataset for coreference resolution, inspired by the Winograd Schema Challenge. | English | +| [wmdp](wmdp/README.md) | A benchmark with the objective of minimizing performance, based on potentially-sensitive multiple-choice knowledge questions. | English | +| [wmt2016](wmt2016/README.md) | Tasks from the WMT 2016 shared task, focusing on translation between multiple languages. | English, Czech, German, Finnish, Russian, Romanian, Turkish | +| [wsc273](wsc273/README.md) | The Winograd Schema Challenge, a test of commonsense reasoning and coreference resolution. | English | +| [xcopa](xcopa/README.md) | Cross-lingual Choice of Plausible Alternatives, testing reasoning in multiple languages. | Estonian, Haitian, Indonesian, Italian, Quechua, Swahili, Tamil, Thai, Turkish, Vietnamese, Chinese | +| [xnli](xnli/README.md) | Cross-Lingual Natural Language Inference to test understanding across different languages. | Arabic, Bulgarian, German, Greek, English, Spanish, French, Hindi, Russian, Swahili, Thai, Turkish, Urdu, Vietnamese, Chinese | +| [xnli_eu](xnli_eu/README.md) | Cross-lingual Natural Language Inference tasks in Basque. | Basque | +| [xquad](xquad/README.md) | Cross-lingual Question Answering Dataset in multiple languages. | Arabic, German, Greek, English, Spanish, Hindi, Romanian, Russian, Thai, Turkish, Vietnamese, Chinese | +| [xstorycloze](xstorycloze/README.md) | Cross-lingual narrative understanding tasks to predict story endings in multiple languages. | Russian, Simplified Chinese, Spanish, Arabic, Hindi, Indonesian, Telugu, Swahili, Basque, Burmese | +| [xwinograd](xwinograd/README.md) | Cross-lingual Winograd schema tasks for coreference resolution in multiple languages. | English, French, Japanese, Portuguese, Russian, Chinese | + +## Multilingual Tasks +| Task Family | Description | Modality | +|------------------------------|---------------------------------------------------------------------------------------------------------|-------------| +| [chartqa](chartqa/README.md) | A benchmark for question answering about charts that requires both visual and logical reasoning. | Image, Text | +| [mmmu](mmmu/README.md) | Evaluate multimodal models on massive multi-discipline tasks demanding college-level subject knowledge. | Image, Text | diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/__init__.py b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..602337a4355944a68953390ce21911f0f782e393 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/__init__.py @@ -0,0 +1,669 @@ +import collections +import inspect +import logging +import os +from functools import partial +from typing import Dict, List, Mapping, Optional, Union + +from lm_eval import utils +from lm_eval.api.group import ConfigurableGroup, GroupConfig +from lm_eval.api.task import ConfigurableTask, Task +from lm_eval.evaluator_utils import get_subtask_list + + +GROUP_ONLY_KEYS = list(GroupConfig().to_dict().keys()) + +eval_logger = logging.getLogger(__name__) + + +class TaskManager: + """TaskManager indexes all tasks from the default `lm_eval/tasks/` + and an optional directory if provided. + + """ + + def __init__( + self, + verbosity: Optional[str] = None, + include_path: Optional[Union[str, List]] = None, + include_defaults: bool = True, + metadata: Optional[dict] = None, + ) -> None: + if verbosity is not None: + utils.setup_logging(verbosity) + self.include_path = include_path + self.metadata = metadata + self._task_index = self.initialize_tasks( + include_path=include_path, include_defaults=include_defaults + ) + self._all_tasks = sorted(list(self._task_index.keys())) + + self._all_groups = sorted( + [x for x in self._all_tasks if self._task_index[x]["type"] == "group"] + ) + self._all_subtasks = sorted( + [ + x + for x in self._all_tasks + if self._task_index[x]["type"] in ["task", "python_task"] + ] + ) + self._all_tags = sorted( + [x for x in self._all_tasks if self._task_index[x]["type"] == "tag"] + ) + + self.task_group_map = collections.defaultdict(list) + + def initialize_tasks( + self, + include_path: Optional[Union[str, List]] = None, + include_defaults: bool = True, + ) -> dict[str, dict]: + """Creates a dictionary of tasks indexes. + + :param include_path: Union[str, List] = None + An additional path to be searched for tasks recursively. + Can provide more than one such path as a list. + :param include_defaults: bool = True + If set to false, default tasks (those in lm_eval/tasks/) are not indexed. + return + Dictionary of task names as key and task metadata + """ + if include_defaults: + all_paths = [os.path.dirname(os.path.abspath(__file__)) + "/"] + else: + all_paths = [] + if include_path is not None: + if isinstance(include_path, str): + include_path = [include_path] + all_paths.extend(include_path) + + task_index = {} + for task_dir in all_paths: + tasks = self._get_task_and_group(task_dir) + task_index = {**tasks, **task_index} + + return task_index + + @property + def all_tasks(self): + return self._all_tasks + + @property + def all_groups(self): + return self._all_groups + + @property + def all_subtasks(self): + return self._all_subtasks + + @property + def all_tags(self): + return self._all_tags + + @property + def task_index(self): + return self._task_index + + def list_all_tasks( + self, list_groups=True, list_tags=True, list_subtasks=True + ) -> str: + from pytablewriter import MarkdownTableWriter + + def sanitize_path(path): + # don't print full path if we are within the lm_eval/tasks dir ! + # if we aren't though, provide the full path. + if "lm_eval/tasks/" in path: + return "lm_eval/tasks/" + path.split("lm_eval/tasks/")[-1] + else: + return path + + group_table = MarkdownTableWriter() + group_table.headers = ["Group", "Config Location"] + gt_values = [] + for g in self.all_groups: + path = self.task_index[g]["yaml_path"] + if path == -1: + path = "---" + else: + path = sanitize_path(path) + gt_values.append([g, path]) + group_table.value_matrix = gt_values + + tag_table = MarkdownTableWriter() + tag_table.headers = ["Tag"] + tag_table.value_matrix = [[t] for t in self.all_tags] + + subtask_table = MarkdownTableWriter() + subtask_table.headers = ["Task", "Config Location", "Output Type"] + st_values = [] + for t in self.all_subtasks: + path = self.task_index[t]["yaml_path"] + + output_type = "" + + # read the yaml file to determine the output type + if path != -1: + config = utils.load_yaml_config(path, mode="simple") + if "output_type" in config: + output_type = config["output_type"] + elif ( + "include" in config + ): # if no output type, check if there is an include with an output type + include_path = path.split("/")[:-1] + config["include"] + include_config = utils.load_yaml_config(include_path, mode="simple") + if "output_type" in include_config: + output_type = include_config["output_type"] + + if path == -1: + path = "---" + else: + path = sanitize_path(path) + st_values.append([t, path, output_type]) + subtask_table.value_matrix = st_values + + result = "\n" + if list_groups: + result += group_table.dumps() + "\n\n" + if list_tags: + result += tag_table.dumps() + "\n\n" + if list_subtasks: + result += subtask_table.dumps() + "\n\n" + return result + + def match_tasks(self, task_list: list[str]) -> list[str]: + return utils.pattern_match(task_list, self.all_tasks) + + def _name_is_registered(self, name: str) -> bool: + if name in self.all_tasks: + return True + return False + + def _name_is_task(self, name: str) -> bool: + if self._name_is_registered(name) and (self.task_index[name]["type"] == "task"): + return True + return False + + def _name_is_tag(self, name: str) -> bool: + if self._name_is_registered(name) and (self.task_index[name]["type"] == "tag"): + return True + return False + + def _name_is_group(self, name: str) -> bool: + if self._name_is_registered(name) and ( + self.task_index[name]["type"] == "group" + ): + return True + return False + + def _name_is_python_task(self, name: str) -> bool: + if self._name_is_registered(name) and ( + self.task_index[name]["type"] == "python_task" + ): + return True + return False + + def _config_is_task(self, config: dict) -> bool: + if ("task" in config) and isinstance(config["task"], str): + return True + return False + + def _config_is_group(self, config: dict) -> bool: + if ("task" in config) and isinstance(config["task"], list): + return True + return False + + def _config_is_python_task(self, config: dict) -> bool: + if "class" in config: + return True + return False + + def _get_yaml_path(self, name: str): + if name not in self.task_index: + raise ValueError + return self.task_index[name]["yaml_path"] + + def _get_config(self, name): + if name not in self.task_index: + raise ValueError + yaml_path = self._get_yaml_path(name) + if yaml_path == -1: + return {} + else: + return utils.load_yaml_config(yaml_path, mode="full") + + def _get_tasklist(self, name): + if self._name_is_task(name): + raise ValueError + return self.task_index[name]["task"] + + def _process_alias(self, config, group=None): + # If the group is not the same as the original + # group which the group alias was intended for, + # Set the group_alias to None instead. + if ("group_alias" in config) and ("group" in config) and group is not None: + if config["group"] != group: + config["group_alias"] = None + return config + + def _class_has_config_in_constructor(self, cls): + constructor = getattr(cls, "__init__", None) + return ( + "config" in inspect.signature(constructor).parameters + if constructor + else False + ) + + def _load_individual_task_or_group( + self, + name_or_config: Optional[Union[str, dict]] = None, + parent_name: Optional[str] = None, + update_config: Optional[dict] = None, + ) -> Mapping: + def _load_task(config, task): + if "include" in config: + config = { + **utils.load_yaml_config( + yaml_path=None, + yaml_config={"include": config.pop("include")}, + mode="full", + ), + **config, + } + if self._config_is_python_task(config): + if self._class_has_config_in_constructor(config["class"]): + task_object = config["class"](config=config) + else: + task_object = config["class"]() + if isinstance(task_object, ConfigurableTask): + # very scuffed: set task name here. TODO: fixme? + task_object.config.task = task + else: + if self.metadata is not None: + config["metadata"] = config.get("metadata", {}) | self.metadata + else: + config["metadata"] = config.get("metadata", {}) + task_object = ConfigurableTask(config=config) + + return {task: task_object} + + def _get_group_and_subtask_from_config( + config: dict, + ) -> tuple[ConfigurableGroup, list[str]]: + if self.metadata is not None: + config["metadata"] = config.get("metadata", {}) | self.metadata + group_name = ConfigurableGroup(config=config) + subtask_list = [] + for task in group_name.config["task"]: + if isinstance(task, str) and self._name_is_tag(task): + subtask_list.extend(self._get_tasklist(task)) + else: + subtask_list.append(task) + return group_name, subtask_list + + def _process_group_config( + config: dict, update_config: dict = None + ) -> tuple[dict, dict]: + if update_config is not None: + config = {**config, **update_config} + _update_config = { + k: v for k, v in config.items() if k not in GROUP_ONLY_KEYS + } + if not bool(_update_config): + _update_config = None + + group_config = {k: v for k, v in config.items() if k in GROUP_ONLY_KEYS} + return group_config, _update_config + + if isinstance(name_or_config, str): + if update_config is not None: + # Process name_or_config as a dict instead + name_or_config = {"task": name_or_config, **update_config} + elif self._name_is_task(name_or_config) or self._name_is_python_task( + name_or_config + ): + task_config = self._get_config(name_or_config) + return _load_task(task_config, task=name_or_config) + else: + subtask_list = self._get_tasklist(name_or_config) + if subtask_list == -1: + group_config = self._get_config(name_or_config) + group_config, update_config = _process_group_config(group_config) + group_name, subtask_list = _get_group_and_subtask_from_config( + group_config + ) + else: + if self._name_is_tag(name_or_config): + fn = partial( + self._load_individual_task_or_group, + update_config=name_or_config + if isinstance(name_or_config, dict) + else None, + ) + return dict( + collections.ChainMap(*map(fn, reversed(subtask_list))) + ) + else: + group_name = ConfigurableGroup( + config={"group": name_or_config, "task": subtask_list} + ) + + if isinstance(name_or_config, dict): + if self._config_is_task(name_or_config): + name = name_or_config.pop("task") + if update_config is not None: + name_or_config = {**name_or_config, **update_config} + # If the name is registered as a group + if self._name_is_group(name): + group_config = self._get_config(name) + + group_config, update_config = _process_group_config( + group_config, name_or_config + ) + group_name, subtask_list = _get_group_and_subtask_from_config( + group_config + ) + elif self._name_is_tag(name): + subtask_list = self._get_tasklist(name) + fn = partial( + self._load_individual_task_or_group, + update_config=name_or_config, + ) + return dict(collections.ChainMap(*map(fn, reversed(subtask_list)))) + else: + if self._name_is_registered(name): + base_task_config = self._get_config(name) + + # Check if this is a duplicate. + if parent_name is not None: + num_duplicate = len( + list( + filter( + lambda x: x.startswith(name), + self.task_group_map[parent_name], + ) + ) + ) + if num_duplicate > 0: + name = f"{name}-{num_duplicate}" + self.task_group_map[parent_name].append(name) + + task_config = { + **base_task_config, + **name_or_config, + } + else: + task_config = name_or_config + return _load_task(task_config, task=name) + else: + group_config, update_config = _process_group_config(name_or_config) + group_name, subtask_list = _get_group_and_subtask_from_config( + group_config + ) + + fn = partial( + self._load_individual_task_or_group, + parent_name=group_name, + update_config=update_config, + ) + return { + group_name: dict(collections.ChainMap(*map(fn, reversed(subtask_list)))) + } + + def load_task_or_group(self, task_list: Optional[Union[str, list]] = None) -> dict: + """Loads a dictionary of task objects from a list + + :param task_list: Union[str, list] = None + Single string or list of string of task names to be loaded + + :return + Dictionary of task objects + """ + if isinstance(task_list, str): + task_list = [task_list] + + all_loaded_tasks = dict( + collections.ChainMap( + *map( + lambda task: self._load_individual_task_or_group(task), + task_list, + ) + ) + ) + return all_loaded_tasks + + def load_config(self, config: Dict): + return self._load_individual_task_or_group(config) + + def _get_task_and_group(self, task_dir: str): + """Creates a dictionary of tasks index with the following metadata, + - `type`, that can be either `task`, `python_task`, `group` or `tags`. + `task` refer to regular task configs, `python_task` are special + yaml files that only consists of `task` and `class` parameters. + `group` are group configs. `tags` are labels that can be assigned + to tasks to assist in sorting and calling tasks of certain themes. + - `yaml_path`, path to the yaml file. If the entry is a `group` that + was configured through a task config, the yaml_path will be -1 + and all subtasks will be listed in `task` (see below) + - `task`, reserved for entries with `type` as `group`. This will list + all subtasks. When a group config is created (as opposed to task + config having `group` parameter set), this will be set to -1 to + avoid recursive indexing. The whole list of subtasks will be loaded + at evaluation. + + :param task_dir: str + A directory to check for tasks + + :return + Dictionary of task names as key and task metadata + """ + + def _populate_tags_and_groups(config, task, tasks_and_groups, print_info): + # TODO: remove group in next release + if "tag" in config: + attr_list = config["tag"] + if isinstance(attr_list, str): + attr_list = [attr_list] + + for tag in attr_list: + if tag not in tasks_and_groups: + tasks_and_groups[tag] = { + "type": "tag", + "task": [task], + "yaml_path": -1, + } + elif tasks_and_groups[tag]["type"] != "tag": + eval_logger.info( + f"The tag '{tag}' is already registered as a group, this tag will not be registered. " + "This may affect tasks you want to call." + ) + break + else: + tasks_and_groups[tag]["task"].append(task) + + # TODO: remove group in next release + print_info = True + ignore_dirs = [ + "__pycache__", + ".ipynb_checkpoints", + ] + tasks_and_groups = collections.defaultdict() + for root, dirs, file_list in os.walk(task_dir): + dirs[:] = [d for d in dirs if d not in ignore_dirs] + for f in file_list: + if f.endswith(".yaml"): + yaml_path = os.path.join(root, f) + config = utils.load_yaml_config(yaml_path, mode="simple") + if self._config_is_python_task(config): + # This is a python class config + task = config["task"] + tasks_and_groups[task] = { + "type": "python_task", + "yaml_path": yaml_path, + } + _populate_tags_and_groups( + config, task, tasks_and_groups, print_info + ) + elif self._config_is_group(config): + # This is a group config + tasks_and_groups[config["group"]] = { + "type": "group", + "task": -1, # This signals that + # we don't need to know + # the task list for indexing + # as it can be loaded + # when called. + "yaml_path": yaml_path, + } + + # # Registered the level 1 tasks from a group config + # for config in config["task"]: + # if isinstance(config, dict) and self._config_is_task(config): + # task = config["task"] + # tasks_and_groups[task] = { + # "type": "task", + # "yaml_path": yaml_path, + # } + + elif self._config_is_task(config): + # This is a task config + task = config["task"] + tasks_and_groups[task] = { + "type": "task", + "yaml_path": yaml_path, + } + _populate_tags_and_groups( + config, task, tasks_and_groups, print_info + ) + else: + eval_logger.debug(f"File {f} in {root} could not be loaded") + + return tasks_and_groups + + +def get_task_name_from_config(task_config: Dict[str, str]) -> str: + if "task" in task_config: + return task_config["task"] + if "dataset_name" in task_config: + return "{dataset_path}_{dataset_name}".format(**task_config) + else: + return "{dataset_path}".format(**task_config) + + +def get_task_name_from_object(task_object): + if hasattr(task_object, "config"): + return task_object._config["task"] + + # TODO: scrap this + # this gives a mechanism for non-registered tasks to have a custom name anyways when reporting + return ( + task_object.EVAL_HARNESS_NAME + if hasattr(task_object, "EVAL_HARNESS_NAME") + else type(task_object).__name__ + ) + + +def _check_duplicates(task_dict: dict) -> None: + """helper function solely used in validating get_task_dict output. + Takes the output of lm_eval.evaluator_utils.get_subtask_list and + returns a list of all leaf subtasks contained within, and errors if any such leaf subtasks are + "oversubscribed" to several disjoint groups. + """ + subtask_names = [] + for key, value in task_dict.items(): + subtask_names.extend(value) + + duplicate_tasks = { + task_name for task_name in subtask_names if subtask_names.count(task_name) > 1 + } + + # locate the potentially problematic groups that seem to 'compete' for constituent subtasks + competing_groups = [ + group + for group in task_dict.keys() + if len(set(task_dict[group]).intersection(duplicate_tasks)) > 0 + ] + + if len(duplicate_tasks) > 0: + raise ValueError( + f"Found 1 or more tasks while trying to call get_task_dict() that were members of more than 1 called group: {list(duplicate_tasks)}. Offending groups: {competing_groups}. Please call groups which overlap their constituent tasks in separate evaluation runs." + ) + + +def get_task_dict( + task_name_list: Union[str, List[Union[str, Dict, Task]]], + task_manager: Optional[TaskManager] = None, +): + """Creates a dictionary of task objects from either a name of task, config, or prepared Task object. + + :param task_name_list: List[Union[str, Dict, Task]] + Name of model or LM object, see lm_eval.models.get_model + :param task_manager: TaskManager = None + A TaskManager object that stores indexed tasks. If not set, + task_manager will load one. This should be set by the user + if there are additional paths that want to be included + via `include_path` + + :return + Dictionary of task objects + """ + + task_name_from_string_dict = {} + task_name_from_config_dict = {} + task_name_from_object_dict = {} + + if isinstance(task_name_list, str): + task_name_list = [task_name_list] + elif isinstance(task_name_list, list): + if not all([isinstance(task, (str, dict, Task)) for task in task_name_list]): + raise TypeError( + "Expected all list items to be of types 'str', 'dict', or 'Task', but at least one entry did not match." + ) + else: + raise TypeError( + f"Expected a 'str' or 'list' but received {type(task_name_list)}." + ) + + string_task_name_list = [task for task in task_name_list if isinstance(task, str)] + others_task_name_list = [ + task for task in task_name_list if not isinstance(task, str) + ] + if len(string_task_name_list) > 0: + if task_manager is None: + task_manager = TaskManager() + + task_name_from_string_dict = task_manager.load_task_or_group( + string_task_name_list + ) + + for task_element in others_task_name_list: + if isinstance(task_element, dict): + task_name_from_config_dict = { + **task_name_from_config_dict, + **task_manager.load_config(config=task_element), + } + + elif isinstance(task_element, Task): + task_name_from_object_dict = { + **task_name_from_object_dict, + get_task_name_from_object(task_element): task_element, + } + + if not set(task_name_from_string_dict.keys()).isdisjoint( + set(task_name_from_object_dict.keys()) + ): + raise ValueError + + final_task_dict = { + **task_name_from_string_dict, + **task_name_from_config_dict, + **task_name_from_object_dict, + } + + # behavior can get odd if one tries to invoke several groups that "compete" for the same task. + # (notably, because one could request several num_fewshot values at once in GroupConfig overrides for the subtask + # and we'd be unsure which to use and report.) + # we explicitly check and error in this case. + _check_duplicates(get_subtask_list(final_task_dict)) + + return final_task_dict diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/README.md b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7c148d9fbec8be41fd89a01aa8590deabd2c4cad --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/README.md @@ -0,0 +1,59 @@ +# GPQA + +### Paper + +Title: GPQA: A Graduate-Level Google-Proof Q&A Benchmark + +Abstract: https://arxiv.org/abs/2311.12022 + +We present GPQA, a challenging dataset of 448 multiple-choice questions written by domain experts in biology, physics, and chemistry. We ensure that the questions are high-quality and extremely difficult: experts who have or are pursuing PhDs in the corresponding domains reach 65% accuracy (74% when discounting clear mistakes the experts identified in retrospect), while highly skilled non-expert validators only reach 34% accuracy, despite spending on average over 30 minutes with unrestricted access to the web (i.e., the questions are “Google-proof”). The questions are also difficult for state-of-the-art AI systems, with our strongest GPT-4–based baseline achieving 39% accuracy. If we are to use future AI systems to help us answer very hard questions—for example, when developing new scientific knowledge—we need to develop *scalable oversight* methods that enable humans to supervise their outputs, which may be difficult even if the supervisors are themselves skilled and knowledgeable. The difficulty of GPQA both for skilled non-experts and frontier AI systems should enable realistic scalable oversight experiments, which we hope can help devise ways for human experts to reliably get truthful information from AI systems that surpass human capabilities. + +Homepage: `https://github.com/idavidrein/gpqa/tree/main` + +### Citation + +``` +@misc{rein2023gpqa, + title={GPQA: A Graduate-Level Google-Proof Q&A Benchmark}, + author={David Rein and Betty Li Hou and Asa Cooper Stickland and Jackson Petty and Richard Yuanzhe Pang and Julien Dirani and Julian Michael and Samuel R. Bowman}, + year={2023}, + eprint={2311.12022}, + archivePrefix={arXiv}, + primaryClass={cs.AI} +} +``` + +This dataset is gated, so you will have to accept the terms of use at https://huggingface.co/datasets/Idavidrein/gpqa and login via `huggingface-cli login` using your HF Hub token before running this task. + +### Groups, Tags, and Tasks + +#### Groups + +None + +#### Tags + +* `gpqa`: runs all GPQA variants. + +#### Tasks + +* `gpqa_{main, diamond, extended}_zeroshot` +* `gpqa_{main, diamond, extended}_n_shot` +* `gpqa_{main, diamond, extended}_generative_n_shot` +* `gpqa_{main, diamond, extended}_cot_zeroshot` +* `gpqa_{main, diamond, extended}_cot_n_shot` + +### Checklist + +For adding novel benchmarks/datasets to the library: + +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: + +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/cot_n_shot/_generate_configs.py b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/cot_n_shot/_generate_configs.py new file mode 100644 index 0000000000000000000000000000000000000000..73ccb876a449a1e8eda5984d977194f6b0c064d9 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/cot_n_shot/_generate_configs.py @@ -0,0 +1,26 @@ +import yaml +from tqdm import tqdm + + +def main() -> None: + subset = ["extended", "diamond", "main"] + setting = "cot_n_shot" + for task in tqdm(subset): + file_name = f"gpqa_{task}_{setting}.yaml" + try: + with open(f"{file_name}", "w") as f: + f.write("# Generated by _generate_configs.py\n") + yaml.dump( + { + "include": f"_gpqa_{setting}_yaml", + "task": f"gpqa_{task}_{setting}", + "dataset_name": f"gpqa_{task}", + }, + f, + ) + except FileExistsError: + pass + + +if __name__ == "__main__": + main() diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/cot_n_shot/_gpqa_cot_n_shot_yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/cot_n_shot/_gpqa_cot_n_shot_yaml new file mode 100644 index 0000000000000000000000000000000000000000..97c0603bcc94f0c689269ea9859b62bdfab7644e --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/cot_n_shot/_gpqa_cot_n_shot_yaml @@ -0,0 +1,38 @@ +dataset_path: Idavidrein/gpqa +tag: gpqa +output_type: generate_until +process_docs: !function utils.process_docs +training_split: train +# Because huggingface dataset only has train split +validation_split: train +test_split: null +description: "Here are some example questions from experts. Answer the final question yourself, following the format of the previous questions exactly.\n" +doc_to_text: "Question: {{Question}}\nChoices:\n(A) {{choice1}}\n(B) {{choice2}}\n(C) {{choice3}}\n(D) {{choice4}}\nLet's think step by step: " +doc_to_target: answer +filter_list: + - name: "strict-match" + filter: + - function: "regex" + regex_pattern: "(?<=The answer is )(.*)(?=.)" + - function: "take_first" + - name: "flexible-extract" + filter: + - function: "multi_choice_regex" + group_select: -1 + ignore_case: true + ignore_punctuation: true + regex_pattern: "(\\([A-Z]\\))" + - function: "take_first" +generation_kwargs: + until: + - "" + do_sample: false + temperature: 0.0 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 2.0 diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/cot_n_shot/gpqa_diamond_cot_n_shot.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/cot_n_shot/gpqa_diamond_cot_n_shot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..24e5f4f90f1f770f9f792e4aeef51e08d3aa08d9 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/cot_n_shot/gpqa_diamond_cot_n_shot.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: gpqa_diamond +include: _gpqa_cot_n_shot_yaml +task: gpqa_diamond_cot_n_shot diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/cot_n_shot/gpqa_extended_cot_n_shot.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/cot_n_shot/gpqa_extended_cot_n_shot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..002ede9a82110e3679bf3e1e958ded4342e408e3 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/cot_n_shot/gpqa_extended_cot_n_shot.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: gpqa_extended +include: _gpqa_cot_n_shot_yaml +task: gpqa_extended_cot_n_shot diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/cot_n_shot/gpqa_main_cot_n_shot.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/cot_n_shot/gpqa_main_cot_n_shot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..916b6ea06a2e22042344b668191adbb3c91c4e75 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/cot_n_shot/gpqa_main_cot_n_shot.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: gpqa_main +include: _gpqa_cot_n_shot_yaml +task: gpqa_main_cot_n_shot diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/cot_n_shot/utils.py b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/cot_n_shot/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..96bcd52b140fd0a5896f55c0a52ea2fd5453fd53 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/cot_n_shot/utils.py @@ -0,0 +1,39 @@ +import random +import re + +import datasets + + +def preprocess(text): + if text is None: + return " " + text = text.strip() + text = text.replace(" [title]", ". ") + text = re.sub("\\[.*?\\]", "", text) + text = text.replace(" ", " ") + return text + + +def process_docs(dataset: datasets.Dataset) -> datasets.Dataset: + def _process_doc(doc): + choices = [ + preprocess(doc["Incorrect Answer 1"]), + preprocess(doc["Incorrect Answer 2"]), + preprocess(doc["Incorrect Answer 3"]), + preprocess(doc["Correct Answer"]), + ] + + random.shuffle(choices) + correct_answer_index = choices.index(preprocess(doc["Correct Answer"])) + + out_doc = { + "choice1": choices[0], + "choice2": choices[1], + "choice3": choices[2], + "choice4": choices[3], + "choices": [choices[0], choices[1], choices[2], choices[3]], + "answer": f"({chr(65 + correct_answer_index)})", + } + return out_doc + + return dataset.map(_process_doc) diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/cot_zeroshot/_generate_configs.py b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/cot_zeroshot/_generate_configs.py new file mode 100644 index 0000000000000000000000000000000000000000..bda00784cc2fa26b5f0d488cf7b6aea37243353d --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/cot_zeroshot/_generate_configs.py @@ -0,0 +1,26 @@ +import yaml +from tqdm import tqdm + + +def main() -> None: + subset = ["extended", "diamond", "main"] + setting = "cot_zeroshot" + for task in tqdm(subset): + file_name = f"gpqa_{task}_{setting}.yaml" + try: + with open(f"{file_name}", "w") as f: + f.write("# Generated by _generate_configs.py\n") + yaml.dump( + { + "include": f"_gpqa_{setting}_yaml", + "task": f"gpqa_{task}_{setting}", + "dataset_name": f"gpqa_{task}", + }, + f, + ) + except FileExistsError: + pass + + +if __name__ == "__main__": + main() diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/cot_zeroshot/_gpqa_cot_zeroshot_yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/cot_zeroshot/_gpqa_cot_zeroshot_yaml new file mode 100644 index 0000000000000000000000000000000000000000..8c487a8c4a3e3806bfa265fa7dc7a3f897ddedff --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/cot_zeroshot/_gpqa_cot_zeroshot_yaml @@ -0,0 +1,38 @@ +dataset_path: Idavidrein/gpqa +tag: gpqa +output_type: generate_until +process_docs: !function utils.process_docs +training_split: train +# Because huggingface dataset only has train split +validation_split: train +test_split: null +doc_to_text: "What is the correct answer to this question:{{Question}}\nChoices:\n(A) {{choice1}}\n(B) {{choice2}}\n(C) {{choice3}}\n(D) {{choice4}}\nLet's think step by step: " +doc_to_target: answer +filter_list: + - name: "strict-match" + filter: + - function: "regex" + regex_pattern: "(?<=The answer is )(.*)(?=.)" + - function: "take_first" + - name: "flexible-extract" + filter: + - function: "multi_choice_regex" + group_select: -1 + ignore_case: true + ignore_punctuation: true + regex_pattern: "(\\([A-Z]\\))" + - function: "take_first" +generation_kwargs: + until: + - "" + do_sample: false + temperature: 0.0 +num_fewshot: 0 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/cot_zeroshot/gpqa_diamond_cot_zeroshot.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/cot_zeroshot/gpqa_diamond_cot_zeroshot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e6a840fa1815096f5fa180ed06223e3523a06214 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/cot_zeroshot/gpqa_diamond_cot_zeroshot.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: gpqa_diamond +include: _gpqa_cot_zeroshot_yaml +task: gpqa_diamond_cot_zeroshot diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/cot_zeroshot/gpqa_extended_cot_zeroshot.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/cot_zeroshot/gpqa_extended_cot_zeroshot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9f542a6148f231e2d7e7e2a5a3437047459e3856 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/cot_zeroshot/gpqa_extended_cot_zeroshot.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: gpqa_extended +include: _gpqa_cot_zeroshot_yaml +task: gpqa_extended_cot_zeroshot diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/cot_zeroshot/gpqa_main_cot_zeroshot.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/cot_zeroshot/gpqa_main_cot_zeroshot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8c14604854294c4551e2602e573488c6a7fef254 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/cot_zeroshot/gpqa_main_cot_zeroshot.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: gpqa_main +include: _gpqa_cot_zeroshot_yaml +task: gpqa_main_cot_zeroshot diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/cot_zeroshot/utils.py b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/cot_zeroshot/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..96bcd52b140fd0a5896f55c0a52ea2fd5453fd53 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/cot_zeroshot/utils.py @@ -0,0 +1,39 @@ +import random +import re + +import datasets + + +def preprocess(text): + if text is None: + return " " + text = text.strip() + text = text.replace(" [title]", ". ") + text = re.sub("\\[.*?\\]", "", text) + text = text.replace(" ", " ") + return text + + +def process_docs(dataset: datasets.Dataset) -> datasets.Dataset: + def _process_doc(doc): + choices = [ + preprocess(doc["Incorrect Answer 1"]), + preprocess(doc["Incorrect Answer 2"]), + preprocess(doc["Incorrect Answer 3"]), + preprocess(doc["Correct Answer"]), + ] + + random.shuffle(choices) + correct_answer_index = choices.index(preprocess(doc["Correct Answer"])) + + out_doc = { + "choice1": choices[0], + "choice2": choices[1], + "choice3": choices[2], + "choice4": choices[3], + "choices": [choices[0], choices[1], choices[2], choices[3]], + "answer": f"({chr(65 + correct_answer_index)})", + } + return out_doc + + return dataset.map(_process_doc) diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/generative/_generate_configs.py b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/generative/_generate_configs.py new file mode 100644 index 0000000000000000000000000000000000000000..e2c011ea02d25ca1d3550210f4a4644c97fa52c2 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/generative/_generate_configs.py @@ -0,0 +1,26 @@ +import yaml +from tqdm import tqdm + + +def main() -> None: + subset = ["extended", "diamond", "main"] + setting = "generative_n_shot" + for task in tqdm(subset): + file_name = f"gpqa_{task}_{setting}.yaml" + try: + with open(f"{file_name}", "w") as f: + f.write("# Generated by _generate_configs.py\n") + yaml.dump( + { + "include": f"_gpqa_{setting}_yaml", + "task": f"gpqa_{task}_{setting}", + "dataset_name": f"gpqa_{task}", + }, + f, + ) + except FileExistsError: + pass + + +if __name__ == "__main__": + main() diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/generative/_gpqa_generative_n_shot_yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/generative/_gpqa_generative_n_shot_yaml new file mode 100644 index 0000000000000000000000000000000000000000..f43a9a414cb4e53e7d5e83787ae6c1e5de109111 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/generative/_gpqa_generative_n_shot_yaml @@ -0,0 +1,39 @@ +dataset_path: Idavidrein/gpqa +tag: gpqa +output_type: generate_until +process_docs: !function utils.process_docs +training_split: train +# Because huggingface dataset only has train split +validation_split: train +test_split: null +description: "Here are some example questions from experts. Answer the final question yourself, following the format of the previous questions exactly.\n" +doc_to_text: "Question: {{Question}}\nChoices:\n(A) {{choice1}}\n(B) {{choice2}}\n(C) {{choice3}}\n(D) {{choice4}}\nAnswer:" +doc_to_target: answer +filter_list: + - name: "strict-match" + filter: + - function: "regex" + regex_pattern: "(?<=The answer is )(.*)(?=.)" + - function: "take_first" + - name: "flexible-extract" + filter: + - function: "multi_choice_regex" + group_select: -1 + ignore_case: true + ignore_punctuation: true + regex_pattern: "(\\([A-Z]\\))" + - function: "take_first" +generation_kwargs: + until: + - "" + - "Question:" + - "<|im_end|>" + temperature: 0.0 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 2.0 diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/generative/gpqa_diamond_generative_n_shot.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/generative/gpqa_diamond_generative_n_shot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3a42094e8ba8ef6037820255b74a8830d550b8a9 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/generative/gpqa_diamond_generative_n_shot.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: gpqa_diamond +include: _gpqa_generative_n_shot_yaml +task: gpqa_diamond_generative_n_shot diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/generative/gpqa_extended_generative_n_shot.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/generative/gpqa_extended_generative_n_shot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fc40c2d97684c50b3992f5adf894ebe0c138b4ae --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/generative/gpqa_extended_generative_n_shot.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: gpqa_extended +include: _gpqa_generative_n_shot_yaml +task: gpqa_extended_generative_n_shot diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/generative/gpqa_main_generative_n_shot.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/generative/gpqa_main_generative_n_shot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..865f3cb5efa3d4b8641843cfde7db3c95bd8b8b3 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/generative/gpqa_main_generative_n_shot.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: gpqa_main +include: _gpqa_generative_n_shot_yaml +task: gpqa_main_generative_n_shot diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/generative/utils.py b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/generative/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..96bcd52b140fd0a5896f55c0a52ea2fd5453fd53 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/generative/utils.py @@ -0,0 +1,39 @@ +import random +import re + +import datasets + + +def preprocess(text): + if text is None: + return " " + text = text.strip() + text = text.replace(" [title]", ". ") + text = re.sub("\\[.*?\\]", "", text) + text = text.replace(" ", " ") + return text + + +def process_docs(dataset: datasets.Dataset) -> datasets.Dataset: + def _process_doc(doc): + choices = [ + preprocess(doc["Incorrect Answer 1"]), + preprocess(doc["Incorrect Answer 2"]), + preprocess(doc["Incorrect Answer 3"]), + preprocess(doc["Correct Answer"]), + ] + + random.shuffle(choices) + correct_answer_index = choices.index(preprocess(doc["Correct Answer"])) + + out_doc = { + "choice1": choices[0], + "choice2": choices[1], + "choice3": choices[2], + "choice4": choices[3], + "choices": [choices[0], choices[1], choices[2], choices[3]], + "answer": f"({chr(65 + correct_answer_index)})", + } + return out_doc + + return dataset.map(_process_doc) diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/n_shot/_generate_configs.py b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/n_shot/_generate_configs.py new file mode 100644 index 0000000000000000000000000000000000000000..c01f208e767cb813e6d2116caf74c3d0b2fccfb3 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/n_shot/_generate_configs.py @@ -0,0 +1,26 @@ +import yaml +from tqdm import tqdm + + +def main() -> None: + subset = ["extended", "diamond", "main"] + + for task in tqdm(subset): + file_name = f"gpqa_{task}_n_shot.yaml" + try: + with open(f"{file_name}", "w") as f: + f.write("# Generated by _generate_configs.py\n") + yaml.dump( + { + "include": "_gpqa_n_shot_yaml", + "task": f"gpqa_{task}_n_shot", + "dataset_name": f"gpqa_{task}", + }, + f, + ) + except FileExistsError: + pass + + +if __name__ == "__main__": + main() diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/n_shot/_gpqa_n_shot_yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/n_shot/_gpqa_n_shot_yaml new file mode 100644 index 0000000000000000000000000000000000000000..8406f8aabfa9d10eec18ef7a8565b6393a0bfc03 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/n_shot/_gpqa_n_shot_yaml @@ -0,0 +1,21 @@ +dataset_path: Idavidrein/gpqa +tag: gpqa +output_type: multiple_choice +process_docs: !function utils.process_docs +training_split: train +# Because huggingface dataset only has train split +validation_split: train +test_split: null +description: "Here are some example questions from experts. Answer the final question yourself, following the format of the previous questions exactly.\n" +doc_to_text: "Question: {{Question}}\nChoices:\n(A) {{choice1}}\n(B) {{choice2}}\n(C) {{choice3}}\n(D) {{choice4}}\nAnswer:" +doc_to_target: answer +doc_to_choice: ["(A)", "(B)", "(C)", "(D)"] +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 2.0 diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/n_shot/gpqa_diamond_n_shot.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/n_shot/gpqa_diamond_n_shot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3043a7e53647ff72d535abc113dfccebaa1bd43c --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/n_shot/gpqa_diamond_n_shot.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: gpqa_diamond +include: _gpqa_n_shot_yaml +task: gpqa_diamond_n_shot diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/n_shot/gpqa_extended_n_shot.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/n_shot/gpqa_extended_n_shot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5d16b505b355bccb3d6fd70eb16b307c12d06a09 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/n_shot/gpqa_extended_n_shot.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: gpqa_extended +include: _gpqa_n_shot_yaml +task: gpqa_extended_n_shot diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/n_shot/gpqa_main_n_shot.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/n_shot/gpqa_main_n_shot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7e5f3e9532ab41c0158409e6afb47393806c4177 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/n_shot/gpqa_main_n_shot.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: gpqa_main +include: _gpqa_n_shot_yaml +task: gpqa_main_n_shot diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/n_shot/utils.py b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/n_shot/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..e0b886d2879216094214ce534438e4db0c5e60f8 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/n_shot/utils.py @@ -0,0 +1,41 @@ +import random +import re + +import datasets + + +def preprocess(text): + if text is None: + return " " + text = text.strip() + text = text.replace(" [title]", ". ") + text = re.sub("\\[.*?\\]", "", text) + text = text.replace(" ", " ") + return text + + +rng = random.Random(42) + + +def process_docs(dataset: datasets.Dataset) -> datasets.Dataset: + def _process_doc(doc): + choices = [ + preprocess(doc["Incorrect Answer 1"]), + preprocess(doc["Incorrect Answer 2"]), + preprocess(doc["Incorrect Answer 3"]), + preprocess(doc["Correct Answer"]), + ] + + rng.shuffle(choices) + correct_answer_index = choices.index(preprocess(doc["Correct Answer"])) + + out_doc = { + "choice1": choices[0], + "choice2": choices[1], + "choice3": choices[2], + "choice4": choices[3], + "answer": f"({chr(65 + correct_answer_index)})", + } + return out_doc + + return dataset.map(_process_doc) diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/zeroshot/_generate_configs.py b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/zeroshot/_generate_configs.py new file mode 100644 index 0000000000000000000000000000000000000000..79afbd6f1d8d4b2eb54455d734f6245357580bd3 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/zeroshot/_generate_configs.py @@ -0,0 +1,26 @@ +import yaml +from tqdm import tqdm + + +def main() -> None: + subset = ["extended", "diamond", "main"] + setting = "zeroshot" + for task in tqdm(subset): + file_name = f"gpqa_{task}_{setting}.yaml" + try: + with open(f"{file_name}", "w") as f: + f.write("# Generated by _generate_configs.py\n") + yaml.dump( + { + "include": f"_gpqa_{setting}_yaml", + "task": f"gpqa_{task}_{setting}", + "dataset_name": f"gpqa_{task}", + }, + f, + ) + except FileExistsError: + pass + + +if __name__ == "__main__": + main() diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/zeroshot/_gpqa_zeroshot_yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/zeroshot/_gpqa_zeroshot_yaml new file mode 100644 index 0000000000000000000000000000000000000000..500f1921bec3db0d1282b8501b7a0841ebbb79c4 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/zeroshot/_gpqa_zeroshot_yaml @@ -0,0 +1,21 @@ +dataset_path: Idavidrein/gpqa +tag: gpqa +output_type: multiple_choice +process_docs: !function utils.process_docs +training_split: train +# Because huggingface dataset only has train split +validation_split: train +test_split: null +doc_to_text: "What is the correct answer to this question:{{Question}}\nChoices:\n(A) {{choice1}}\n(B) {{choice2}}\n(C) {{choice3}}\n(D) {{choice4}}\nAnswer:" +doc_to_target: answer +doc_to_choice: ["(A)", "(B)", "(C)", "(D)"] +num_fewshot: 0 +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/zeroshot/gpqa_diamond_zeroshot.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/zeroshot/gpqa_diamond_zeroshot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c3a7921c30b3ff09e82aacb4c0e915010f698966 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/zeroshot/gpqa_diamond_zeroshot.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: gpqa_diamond +include: _gpqa_zeroshot_yaml +task: gpqa_diamond_zeroshot diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/zeroshot/gpqa_extended_zeroshot.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/zeroshot/gpqa_extended_zeroshot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e7347f11154351ad4560200a3f3bf54106a1a8f --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/zeroshot/gpqa_extended_zeroshot.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: gpqa_extended +include: _gpqa_zeroshot_yaml +task: gpqa_extended_zeroshot diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/zeroshot/gpqa_main_zeroshot.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/zeroshot/gpqa_main_zeroshot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a8d7fb59025d148130f2a468cb1bbdfad959102 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/zeroshot/gpqa_main_zeroshot.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: gpqa_main +include: _gpqa_zeroshot_yaml +task: gpqa_main_zeroshot diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/zeroshot/utils.py b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/zeroshot/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c2317e02efd132aea27ec8c8fad284df55ccd382 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gpqa/zeroshot/utils.py @@ -0,0 +1,38 @@ +import random +import re + +import datasets + + +def preprocess(text): + if text is None: + return " " + text = text.strip() + text = text.replace(" [title]", ". ") + text = re.sub("\\[.*?\\]", "", text) + text = text.replace(" ", " ") + return text + + +def process_docs(dataset: datasets.Dataset) -> datasets.Dataset: + def _process_doc(doc): + choices = [ + preprocess(doc["Incorrect Answer 1"]), + preprocess(doc["Incorrect Answer 2"]), + preprocess(doc["Incorrect Answer 3"]), + preprocess(doc["Correct Answer"]), + ] + + random.shuffle(choices) + correct_answer_index = choices.index(preprocess(doc["Correct Answer"])) + + out_doc = { + "choice1": choices[0], + "choice2": choices[1], + "choice3": choices[2], + "choice4": choices[3], + "answer": f"({chr(65 + correct_answer_index)})", + } + return out_doc + + return dataset.map(_process_doc) diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gsm8k/README.md b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gsm8k/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1556151f821f526cf57388f15bb5c867af904a15 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gsm8k/README.md @@ -0,0 +1,62 @@ +# GSM8k + +## Paper +Training Verifiers to Solve Math Word Problems +https://arxiv.org/abs/2110.14168 + +State-of-the-art language models can match human performance on many tasks, but +they still struggle to robustly perform multi-step mathematical reasoning. To +diagnose the failures of current models and support research, we introduce GSM8K, +a dataset of 8.5K high quality linguistically diverse grade school math word problems. +We find that even the largest transformer models fail to achieve high test performance, +despite the conceptual simplicity of this problem distribution. + +NOTE: See the official implementation of the task: + https://github.com/openai/grade-school-math/blob/master/grade_school_math/calculator.py +for how to make use of the dataset's calculator annotations in your language +model's sample/generation function. + +Homepage: https://github.com/openai/grade-school-math + + +## Citation +``` +@misc{cobbe2021training, + title={Training Verifiers to Solve Math Word Problems}, + author={Karl Cobbe and Vineet Kosaraju and Mohammad Bavarian and Jacob Hilton and Reiichiro Nakano and Christopher Hesse and John Schulman}, + year={2021}, + eprint={2110.14168}, + archivePrefix={arXiv}, + primaryClass={cs.LG} +} +``` + +### Groups and Tasks + +#### Groups + +- `math_word_problems` +- `chain_of_thought` +- `self_consistency` + +#### Tasks + +- `gsm8k_yaml` +- `gsm8k_cot`: GSM8K with Chain-of-Thought +- `gsm8k_cot_self_consistency`: GSM8K with Chain-of-Thought and Self-Consistency +- `gsm8k_cot_llama`: GSM8K with prompt formatting modified to conform to the evaluation settings described by Meta here: https://huggingface.co/datasets/meta-llama/Meta-Llama-3.1-8B-Instruct-evals/viewer/Meta-Llama-3.1-8B-Instruct-evals__gsm8k__details?row=0 + - Use this task with --fewshot_as_multiturn and --apply_chat_template to replicate Meta's reported performance. + + +### Checklist + +- [x] Is in Eval-harness v1.0 ? +- [ ] Has been checked for regression from v1.0? +- [ ] Has been checked for equivalence with original paper methodology? +- [ ] "Main" checked variant clearly denoted? + +### Variant Wishlist + +- [ ] Variant with Calculator (see https://github.com/openai/grade-school-math/blob/master/grade_school_math/calculator.py for example implementation) +- [ ] Using Verifiers +- [ ] Majority voting "without CoT" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gsm8k/gsm8k-cot-llama.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gsm8k/gsm8k-cot-llama.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e7948eeb8e3e7039f0c9c1738ac89aa19f4c4bb --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gsm8k/gsm8k-cot-llama.yaml @@ -0,0 +1,84 @@ +dataset_name: main +dataset_path: gsm8k +doc_to_target: '{{answer.split(''####'')[-1].strip() if answer is defined else target}}' +doc_to_text: "Given the following problem, reason and give a final answer to the problem.\nProblem: {{question}}\nYour response should end with \"The final answer is [answer]\" where [answer] is the response to the problem.\n" +fewshot_config: + sampler: first_n + samples: + - question: There are 15 trees in the grove. Grove workers will plant trees in the + grove today. After they are done, there will be 21 trees. How many trees did + the grove workers plant today? + target: There are 15 trees originally. Then there were 21 trees after some more + were planted. So there must have been 21 - 15 = 6. The final answer is 6 + - question: If there are 3 cars in the parking lot and 2 more cars arrive, how many + cars are in the parking lot? + target: There are originally 3 cars. 2 more cars arrive. 3 + 2 = 5. The final answer + is 5 + - question: Leah had 32 chocolates and her sister had 42. If they ate 35, how many + pieces do they have left in total? + target: Originally, Leah had 32 chocolates. Her sister had 42. So in total they + had 32 + 42 = 74. After eating 35, they had 74 - 35 = 39. The final answer is 39 + - question: Jason had 20 lollipops. He gave Denny some lollipops. Now Jason has 12 + lollipops. How many lollipops did Jason give to Denny? + target: Jason started with 20 lollipops. Then he had 12 after giving some to Denny. + So he gave Denny 20 - 12 = 8. The final answer is 8 + - question: Shawn has five toys. For Christmas, he got two toys each from his mom and + dad. How many toys does he have now? + target: Shawn started with 5 toys. If he got 2 toys each from his mom and dad, + then that is 4 more toys. 5 + 4 = 9. The final answer is 9 + - question: There were nine computers in the server room. Five more computers were + installed each day, from monday to thursday. How many computers are now in the + server room? + target: There were originally 9 computers. For each of 4 days, 5 more computers + were added. So 5 * 4 = 20 computers were added. 9 + 20 is 29. The final answer is + 29 + - question: Michael had 58 golf balls. On tuesday, he lost 23 golf balls. On wednesday, + he lost 2 more. How many golf balls did he have at the end of wednesday? + target: Michael started with 58 golf balls. After losing 23 on tuesday, he had + 58 - 23 = 35. After losing 2 more, he had 35 - 2 = 33 golf balls. The final answer + is 33 + - question: Olivia has $23. She bought five bagels for $3 each. How much money does + she have left? + target: Olivia had 23 dollars. 5 bagels for 3 dollars each will be 5 x 3 = 15 + dollars. So she has 23 - 15 dollars left. 23 - 15 is 8. The final answer is 8 +filter_list: +- filter: + - function: regex + group_select: -1 + regex_pattern: The final answer is ((-?[$0-9.,]{2,})|(-?[0-9]+)) + - function: take_first + name: strict-match +- filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +generation_kwargs: + do_sample: false + until: + - '<|eot_id|>' + - '<|start_header_id|>user<|end_header_id|>' + - 'Q:' + - + - <|im_end|> +tag: +- chain_of_thought +metadata: + version: 3.0 +metric_list: +- aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: false + metric: exact_match + regexes_to_ignore: + - ',' + - \$ + - '(?s).*#### ' + - \.$ +num_fewshot: 8 +output_type: generate_until +repeats: 1 +task: gsm8k_cot_llama +test_split: test diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gsm8k/gsm8k-cot-self-consistency.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gsm8k/gsm8k-cot-self-consistency.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0994081b049c0815ae85b9539b627e4c8df00dd3 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gsm8k/gsm8k-cot-self-consistency.yaml @@ -0,0 +1,34 @@ +include: gsm8k-cot.yaml +tag: + - chain_of_thought + - self_consistency +task: gsm8k_cot_self_consistency +generation_kwargs: + until: + - "Q:" + - "\n\n" + do_sample: true + temperature: 0.2 +repeats: 64 +filter_list: + - name: "score-first" # pick only the first response, and report metrics on that + filter: + - function: "regex" + regex_pattern: "The answer is (\\-?[0-9\\.\\,]*[0-9]+)" + - function: "take_first" + - name: "maj@64" + filter: + - function: "regex" + regex_pattern: "The answer is (\\-?[0-9\\.\\,]*[0-9]+)" + - function: "majority_vote" + - function: "take_first" + - name: "maj@8" # get Maj@8 , via selecting the first 8 responses. Using a better estimator would be optimal. + filter: + - function: "take_first_k" + k: 8 + - function: "regex" + regex_pattern: "The answer is (\\-?[0-9\\.\\,]*[0-9]+)" + - function: "majority_vote" + - function: "take_first" +metadata: + version: 2.0 diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gsm8k/gsm8k-cot-zeroshot.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gsm8k/gsm8k-cot-zeroshot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c112d324acf707e5934432068abd2ad6143438ac --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gsm8k/gsm8k-cot-zeroshot.yaml @@ -0,0 +1,44 @@ +tag: + - math_word_problems +task: gsm8k_cot_zeroshot +dataset_path: gsm8k +dataset_name: main +output_type: generate_until +training_split: train +fewshot_split: train +test_split: test +doc_to_text: "Q: {{question}}\nA: Let's think step by step." +doc_to_target: "{{answer}}" #" {{answer.split('### ')[-1].rstrip()}}" +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: false + regexes_to_ignore: + - "," + - "\\$" + - "(?s).*#### " + - "\\.$" +generation_kwargs: + until: + - "Q:" + - "" + - "<|im_end|>" + do_sample: false +repeats: 1 +num_fewshot: 0 +filter_list: + - name: "strict-match" + filter: + - function: "regex" + regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)." + - function: "take_first" + - name: "flexible-extract" + filter: + - function: "regex" + group_select: -1 + regex_pattern: "(-?[$0-9.,]{2,})|(-?[0-9]+)" + - function: "take_first" +metadata: + version: 3.0 diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gsm8k/gsm8k-cot.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gsm8k/gsm8k-cot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d125b0198535122fd5b12a388e903b03ee5f6020 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gsm8k/gsm8k-cot.yaml @@ -0,0 +1,83 @@ +dataset_name: main +dataset_path: gsm8k +doc_to_target: '{{answer.split(''####'')[-1].strip() if answer is defined else target}}' +doc_to_text: 'Q: {{question}} + + A:' +fewshot_config: + sampler: first_n + samples: + - question: There are 15 trees in the grove. Grove workers will plant trees in the + grove today. After they are done, there will be 21 trees. How many trees did + the grove workers plant today? + target: There are 15 trees originally. Then there were 21 trees after some more + were planted. So there must have been 21 - 15 = 6. The answer is 6. + - question: If there are 3 cars in the parking lot and 2 more cars arrive, how many + cars are in the parking lot? + target: There are originally 3 cars. 2 more cars arrive. 3 + 2 = 5. The answer + is 5. + - question: Leah had 32 chocolates and her sister had 42. If they ate 35, how many + pieces do they have left in total? + target: Originally, Leah had 32 chocolates. Her sister had 42. So in total they + had 32 + 42 = 74. After eating 35, they had 74 - 35 = 39. The answer is 39. + - question: Jason had 20 lollipops. He gave Denny some lollipops. Now Jason has 12 + lollipops. How many lollipops did Jason give to Denny? + target: Jason started with 20 lollipops. Then he had 12 after giving some to Denny. + So he gave Denny 20 - 12 = 8. The answer is 8. + - question: Shawn has five toys. For Christmas, he got two toys each from his mom and + dad. How many toys does he have now? + target: Shawn started with 5 toys. If he got 2 toys each from his mom and dad, + then that is 4 more toys. 5 + 4 = 9. The answer is 9. + - question: There were nine computers in the server room. Five more computers were + installed each day, from monday to thursday. How many computers are now in the + server room? + target: There were originally 9 computers. For each of 4 days, 5 more computers + were added. So 5 * 4 = 20 computers were added. 9 + 20 is 29. The answer is + 29. + - question: Michael had 58 golf balls. On tuesday, he lost 23 golf balls. On wednesday, + he lost 2 more. How many golf balls did he have at the end of wednesday? + target: Michael started with 58 golf balls. After losing 23 on tuesday, he had + 58 - 23 = 35. After losing 2 more, he had 35 - 2 = 33 golf balls. The answer + is 33. + - question: Olivia has $23. She bought five bagels for $3 each. How much money does + she have left? + target: Olivia had 23 dollars. 5 bagels for 3 dollars each will be 5 x 3 = 15 + dollars. So she has 23 - 15 dollars left. 23 - 15 is 8. The answer is 8. +filter_list: +- filter: + - function: regex + regex_pattern: The answer is (\-?[0-9\.\,]+). + - function: take_first + name: strict-match +- filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +generation_kwargs: + do_sample: false + until: + - 'Q:' + - + - <|im_end|> +tag: +- chain_of_thought +metadata: + version: 3.0 +metric_list: +- aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: false + metric: exact_match + regexes_to_ignore: + - ',' + - \$ + - '(?s).*#### ' + - \.$ +num_fewshot: 8 +output_type: generate_until +repeats: 1 +task: gsm8k_cot +test_split: test diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gsm8k/gsm8k.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gsm8k/gsm8k.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a9d5bb39aedc0e2b991f0d79f2de6face47a31cf --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/gsm8k/gsm8k.yaml @@ -0,0 +1,45 @@ +tag: + - math_word_problems +task: gsm8k +dataset_path: gsm8k +dataset_name: main +output_type: generate_until +training_split: train +fewshot_split: train +test_split: test +doc_to_text: "Question: {{question}}\nAnswer:" +doc_to_target: "{{answer}}" #" {{answer.split('### ')[-1].rstrip()}}" +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: false + regexes_to_ignore: + - "," + - "\\$" + - "(?s).*#### " + - "\\.$" +generation_kwargs: + until: + - "Question:" + - "" + - "<|im_end|>" + do_sample: false + temperature: 0.0 +repeats: 1 +num_fewshot: 5 +filter_list: + - name: "strict-match" + filter: + - function: "regex" + regex_pattern: "#### (\\-?[0-9\\.\\,]+)" + - function: "take_first" + - name: "flexible-extract" + filter: + - function: "regex" + group_select: -1 + regex_pattern: "(-?[$0-9.,]{2,})|(-?[0-9]+)" + - function: "take_first" +metadata: + version: 3.0 diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/humaneval/README.md b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/humaneval/README.md new file mode 100644 index 0000000000000000000000000000000000000000..44a2cb829233370020319f39e6ae7323e601aabb --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/humaneval/README.md @@ -0,0 +1,52 @@ +# HumanEval + +## Paper +Evaluating Large Language Models Trained on Code +https://arxiv.org/abs/2107.03374 + +We introduce Codex, a GPT language model fine-tuned on publicly available code from GitHub, and study its Python code-writing capabilities. A distinct production version of Codex powers GitHub Copilot. On HumanEval, a new evaluation set we release to measure functional correctness for synthesizing programs from docstrings, our model solves 28.8% of the problems, while GPT-3 solves 0% and GPT-J solves 11.4%. Furthermore, we find that repeated sampling from the model is a surprisingly effective strategy for producing working solutions to difficult prompts. Using this method, we solve 70.2% of our problems with 100 samples per problem. Careful investigation of our model reveals its limitations, including difficulty with docstrings describing long chains of operations and with binding operations to variables. Finally, we discuss the potential broader impacts of deploying powerful code generation technologies, covering safety, security, and economics. + +Homepage: https://github.com/openai/human-eval + +Note: For instruct tuned models, we recommend the instruct variant. That uses a gen_prefix to ensure the model completes the partial code snippet (might not work with all APIs) + +## Citation +``` +@article{chen2021codex, + title={Evaluating Large Language Models Trained on Code}, + author={Mark Chen and Jerry Tworek and Heewoo Jun and Qiming Yuan and Henrique Ponde de Oliveira Pinto and Jared Kaplan and Harri Edwards and Yuri Burda and Nicholas Joseph and Greg Brockman and Alex Ray and Raul Puri and Gretchen Krueger and Michael Petrov and Heidy Khlaaf and Girish Sastry and Pamela Mishkin and Brooke Chan and Scott Gray and Nick Ryder and Mikhail Pavlov and Alethea Power and Lukasz Kaiser and Mohammad Bavarian and Clemens Winter and Philippe Tillet and Felipe Petroski Such and Dave Cummings and Matthias Plappert and Fotios Chantzis and Elizabeth Barnes and Ariel Herbert-Voss and William Hebgen Guss and Alex Nichol and Alex Paino and Nikolas Tezak and Jie Tang and Igor Babuschkin and Suchir Balaji and Shantanu Jain and William Saunders and Christopher Hesse and Andrew N. Carr and Jan Leike and Josh Achiam and Vedant Misra and Evan Morikawa and Alec Radford and Matthew Knight and Miles Brundage and Mira Murati and Katie Mayer and Peter Welinder and Bob McGrew and Dario Amodei and Sam McCandlish and Ilya Sutskever and Wojciech Zaremba}, + year={2021}, + eprint={2107.03374}, + archivePrefix={arXiv}, + primaryClass={cs.LG} +} +``` + +### Groups and Tasks + +#### Groups + +* Not part of a group yet. + +#### Tasks + +- `humaneval` pass@1 +- `humaneval_64` pass@64 variant +- `humaneval_instruct`: pass@1 with config more appropriate for instruct models. (implementation taken from llama [evals](https://huggingface.co/datasets/meta-llama/Llama-3.1-8B-Instruct-evals/viewer/Llama-3.1-8B-Instruct-evals__human_eval__details?row=0)) +- `humaneval_instruct_64`: pass@64 variant + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [ ] Is the task an existing benchmark in the literature? + * [ ] Have you referenced the original paper that introduced the task? + * [ ] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? + +### Changelog +v2 20-MAR-2025: `humaneval_instruct`, `humaneval_instruct_64`: fixed typo in gen_prefix diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/humaneval/humaneval.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/humaneval/humaneval.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b22f67b77ce67db8de66bb8ade37e173a89854b2 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/humaneval/humaneval.yaml @@ -0,0 +1,26 @@ +task: humaneval +dataset_path: openai/openai_humaneval +unsafe_code: true +output_type: generate_until +test_split: test +doc_to_text: "{{prompt}}" +doc_to_target: "{{test}}\ncheck({{entry_point}})" +metric_list: + - metric: !function utils.pass_at_k + aggregation: mean + higher_is_better: true + k: [1] +generation_kwargs: + until: + - "[DONE]" + max_gen_toks: 1024 + do_sample: false +repeats: 1 +num_fewshot: 0 +filter_list: + - name: "create_test" + filter: + - function: "custom" + filter_fn: !function utils.build_predictions +metadata: + version: 1.0 \ No newline at end of file diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/humaneval/humaneval_5.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/humaneval/humaneval_5.yaml new file mode 100644 index 0000000000000000000000000000000000000000..01be8ec130a1c1f6669e6b5dc0a0245bbc246e16 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/humaneval/humaneval_5.yaml @@ -0,0 +1,19 @@ +include: humaneval.yaml +task: humaneval_5 +repeats: 5 +metric_list: + - metric: !function utils.pass_at_k + aggregation: mean + higher_is_better: true + k: [1,2,3,4,5] +generation_kwargs: + until: + - "\nclass" + - "\ndef" + - "\n#" + - "\nif" + - "\nprint" + max_gen_toks: 1024 + do_sample: true + temperature: 0.2 + top_p: 0.95 diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/humaneval/humaneval_5_instruct.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/humaneval/humaneval_5_instruct.yaml new file mode 100644 index 0000000000000000000000000000000000000000..953d10de29fa6044b7db2a160f162fd67c945b19 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/humaneval/humaneval_5_instruct.yaml @@ -0,0 +1,11 @@ +include: humaneval_5.yaml +task: humaneval_5_instruct +doc_to_text: "Write a solution to the following problem and make sure that it passes the tests:\n```{{prompt}}" +gen_prefix: "Here is the completed function:\n```python\n{{prompt}}\n" +filter_list: + - name: "create_test" + filter: + - function: "custom" + filter_fn: !function utils.build_predictions_instruct +metadata: + version: 2.0 diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/humaneval/humaneval_5_instruct_noprefix.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/humaneval/humaneval_5_instruct_noprefix.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6cb0f51cfc1cd4c38f266518a1ad1931dec5d9d3 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/humaneval/humaneval_5_instruct_noprefix.yaml @@ -0,0 +1,15 @@ +include: humaneval_5.yaml +task: humaneval_5_instruct_noprefix +doc_to_text: "Write a solution to the following problem and make sure that it passes the tests:\n```{{prompt}}" +gen_prefix: "```python\n" +generation_kwargs: + until: + - "\nassert" + - "\n# Test" +filter_list: + - name: "create_test" + filter: + - function: "custom" + filter_fn: !function utils.build_predictions_instruct +metadata: + version: 2.0 diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/humaneval/humaneval_64.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/humaneval/humaneval_64.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1720ae7c77c1d1b0bae96a4b1153ea6b0893c64d --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/humaneval/humaneval_64.yaml @@ -0,0 +1,19 @@ +include: humaneval.yaml +task: humaneval_64 +repeats: 64 +metric_list: + - metric: !function utils.pass_at_k + aggregation: mean + higher_is_better: true + k: [2,8,16,32,64] +generation_kwargs: + until: + - "\nclass" + - "\ndef" + - "\n#" + - "\nif" + - "\nprint" + max_gen_toks: 1024 + do_sample: true + temperature: 0.2 + top_p: 0.95 diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/humaneval/humaneval_64_instruct.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/humaneval/humaneval_64_instruct.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ca0f38c31e8d6b8d6b3ae8e7847fd6141f187492 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/humaneval/humaneval_64_instruct.yaml @@ -0,0 +1,11 @@ +include: humaneval_64.yaml +task: humaneval_64_instruct +doc_to_text: "Write a solution to the following problem and make sure that it passes the tests:\n```{{prompt}}" +gen_prefix: "Here is the completed function:\n```python\n{{prompt}}\n" +filter_list: + - name: "create_test" + filter: + - function: "custom" + filter_fn: !function utils.build_predictions_instruct +metadata: + version: 2.0 diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/humaneval/humaneval_instruct.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/humaneval/humaneval_instruct.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0dcbd49e46760d63f1d72fe09194c4102b6a0415 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/humaneval/humaneval_instruct.yaml @@ -0,0 +1,11 @@ +include: humaneval.yaml +task: humaneval_instruct +doc_to_text: "Write a solution to the following problem and make sure that it passes the tests:\n```{{prompt}}" +gen_prefix: "Here is the completed function:\n```python\n{{prompt}}\n" +filter_list: + - name: "create_test" + filter: + - function: "custom" + filter_fn: !function utils.build_predictions_instruct +metadata: + version: 2.0 \ No newline at end of file diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/humaneval/humaneval_instruct_noprefix.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/humaneval/humaneval_instruct_noprefix.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2f11a7aea0985234d5d39ce8d1f163eea9803bdb --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/humaneval/humaneval_instruct_noprefix.yaml @@ -0,0 +1,15 @@ +include: humaneval.yaml +task: humaneval_instruct_noprefix +doc_to_text: "Write a solution to the following problem and make sure that it passes the tests:\n```{{prompt}}```" +gen_prefix: "```python\n" +generation_kwargs: + until: + - "\nassert" + - "\n# Test" +filter_list: + - name: "create_test" + filter: + - function: "custom" + filter_fn: !function utils.build_predictions_instruct +metadata: + version: 2.0 diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/humaneval/humaneval_plus.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/humaneval/humaneval_plus.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e31772050e8ecc713827804cba3b81c3da3cfc2 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/humaneval/humaneval_plus.yaml @@ -0,0 +1,3 @@ +include: humaneval.yaml +task: humaneval_plus +dataset_path: evalplus/humanevalplus diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/humaneval/sanitize_utils.py b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/humaneval/sanitize_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..a8acf070cc2d47a4aee9fe37e0b73ecda22fd9ea --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/humaneval/sanitize_utils.py @@ -0,0 +1,121 @@ +import ast +import traceback + +from typing import Dict, List, Optional, Set, Tuple + +def refine_text(text: str) -> str: + text = text.replace("\t", " ") + text = text.replace("\r\n", "\n").replace("\r", "\n") + return text.strip() + "\n" + +def syntax_check(code, verbose = False): + try: + ast.parse(code) + return True + except (SyntaxError, MemoryError): + if verbose: + traceback.print_exc() + return False + +def extract_longest_valid_code(text: str) -> str: + lines = text.splitlines() + + if len(lines) > 100: + lines = lines[:100] + max_valid_lines = 0 + max_valid_snippet = "" + + for i in range(len(lines)): + for j in range(i, len(lines)): + current_snippet = "\n".join(lines[i:j+1]) + if syntax_check(current_snippet): + valid_line_count = sum(1 for line in lines[i:j+1] if line.strip()) + if valid_line_count > max_valid_lines: + max_valid_lines = valid_line_count + max_valid_snippet = current_snippet + + return max_valid_snippet + +def get_deps(nodes: List[Tuple[str, ast.AST]]) -> Dict[str, Set[str]]: + name2deps = {} + for name, node in nodes: + deps = set() + stack = [node] + while stack: + current = stack.pop() + for child in ast.iter_child_nodes(current): + if isinstance(child, ast.Name): + deps.add(child.id) + elif isinstance(child, ast.Attribute): + deps.add(child.attr) + else: + stack.append(child) + name2deps[name] = deps + return name2deps + +def get_function_dependency(entrypoint: str, call_graph: Dict[str, Set[str]]) -> Set[str]: + visited = set() + to_visit = [entrypoint] + + while to_visit: + current = to_visit.pop(0) + if current not in visited: + visited.add(current) + to_visit.extend(call_graph.get(current, set()) - visited) + + return visited + +def get_definition_name(node: ast.AST) -> Optional[str]: + if isinstance(node, (ast.FunctionDef, ast.ClassDef)): + return node.name + elif isinstance(node, ast.Assign): + targets = node.targets + if targets and isinstance(targets[0], ast.Name): + return targets[0].id + return None + +def has_return_statement(node: ast.AST) -> bool: + return any(isinstance(n, ast.Return) for n in ast.walk(node)) + +def sanitize(text: str, entrypoint: Optional[str] = None) -> str: + + text = refine_text(text) + + # text = python_extract(text) + + code = extract_longest_valid_code(text) + tree = ast.parse(code) + + definitions = {} + + imports = [] + + for node in tree.body: + if isinstance(node, (ast.Import, ast.ImportFrom)): + imports.append(node) + elif isinstance(node, ast.ClassDef): + name = node.name + definitions[name] = ('class', node) + elif isinstance(node, ast.FunctionDef): + name = node.name + if has_return_statement(node): + definitions[name] = ('function', node) + elif isinstance(node, ast.Assign): + name = get_definition_name(node) + if name: + definitions[name] = ('variable', node) + + if entrypoint: + name2deps = get_deps([(name, node) for name, (_, node) in definitions.items()]) + reachable = get_function_dependency(entrypoint, name2deps) + + sanitized_output = [] + + for node in imports: + sanitized_output.append(ast.unparse(node)) + + for name, (_, node) in definitions.items(): + if not entrypoint or name in reachable: + sanitized_output.append(ast.unparse(node)) + + return "\n".join(sanitized_output) \ No newline at end of file diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/humaneval/utils.py b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/humaneval/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fc98f5d4f71963e8d93b0ccd6aec39d0691960d2 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/humaneval/utils.py @@ -0,0 +1,52 @@ +import evaluate as hf_evaluate + +from lm_eval.tasks.humaneval.sanitize_utils import sanitize + + +try: + compute_ = hf_evaluate.load("code_eval") + test_cases = ["assert add(2, 3)==5"] + candidates = [["def add(a,b): return a*b"]] + results = compute_.compute(references=test_cases, predictions=candidates, k=[1]) +except Exception as e: + raise e + + +def pass_at_k(references: list[str], predictions: list[list[str]], k: list[int] = None): + global compute_ + assert k is not None + if isinstance(k, int): + k = [k] + + processed_predictions = [] + for preds in predictions: + processed_preds = [] + for p in preds: + processed_preds.append(p.strip("```")[0] if "```" in p else p) + processed_predictions.append(processed_preds) + + res = compute_.compute( + references=references, + predictions=predictions, + k=k, + ) + return res[0] + + +def build_predictions(resps: list[list[str]], docs: list[dict]) -> list[list[str]]: + return [[doc["prompt"] + r for r in resp] for resp, doc in zip(resps, docs)] + + +def build_predictions_instruct( + resps: list[list[str]], docs: list[dict] +) -> list[list[str]]: + return [ + [ + sanitize( + doc["prompt"] + "\n" + r.split('```python\n', 1)[-1].split('```')[0], + doc["entry_point"] + ) + for r in resp + ] + for resp, doc in zip(resps, docs) + ] \ No newline at end of file diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/ifeval/README.md b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/ifeval/README.md new file mode 100644 index 0000000000000000000000000000000000000000..aced2e78bb5adfd0ff413b4ee72c53e0fa3d5cc6 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/ifeval/README.md @@ -0,0 +1,45 @@ +# IFEval + +### Paper + +Title: Instruction-Following Evaluation for Large Language Models +Abstract: https://arxiv.org/abs/2311.07911 + +One core capability of Large Language Models (LLMs) is to follow natural language instructions. However, the evaluation of such abilities is not standardized: Human evaluations are expensive, slow, and not objectively reproducible, while LLM-based auto-evaluation is potentially biased or limited by the ability of the evaluator LLM. To overcome these issues, we introduce Instruction-Following Eval (IFEval) for large language models. IFEval is a straightforward and easy-to-reproduce evaluation benchmark. It focuses on a set of "verifiable instructions" such as "write in more than 400 words" and "mention the keyword of AI at least 3 times". We identified 25 types of those verifiable instructions and constructed around 500 prompts, with each prompt containing one or more verifiable instructions. We show evaluation results of two widely available LLMs on the market. Our code and data can be found at https://github.com/google-research/google-research/tree/master/instruction_following_eval + +Homepage: https://github.com/google-research/google-research/tree/master/instruction_following_eval + + +### Citation + +``` +@article{zhou2023instructionfollowing, + title={Instruction-Following Evaluation for Large Language Models}, + author={Jeffrey Zhou and Tianjian Lu and Swaroop Mishra and Siddhartha Brahma and Sujoy Basu and Yi Luan and Denny Zhou and Le Hou}, + journal={arXiv preprint arXiv:2311.07911}, + year={2023}, +} +``` + +### Groups and Tasks + +#### Groups + +* Not part of a group yet + +#### Tasks + +* `ifeval` + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/ifeval/ifeval.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/ifeval/ifeval.yaml new file mode 100644 index 0000000000000000000000000000000000000000..508a63a9452874109cd949f5d5a5e00ad5f66b36 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/ifeval/ifeval.yaml @@ -0,0 +1,29 @@ +task: ifeval +dataset_path: google/IFEval +dataset_name: null +output_type: generate_until +test_split: train +num_fewshot: 0 +doc_to_text: prompt +doc_to_target: 0 +generation_kwargs: + until: [] + do_sample: false + temperature: 0.0 + max_gen_toks: 1280 +process_results: !function utils.process_results +metric_list: + - metric: prompt_level_strict_acc + aggregation: mean + higher_is_better: true + - metric: inst_level_strict_acc + aggregation: !function utils.agg_inst_level_acc + higher_is_better: true + - metric: prompt_level_loose_acc + aggregation: mean + higher_is_better: true + - metric: inst_level_loose_acc + aggregation: !function utils.agg_inst_level_acc + higher_is_better: true +metadata: + version: 4.0 diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/ifeval/instructions.py b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/ifeval/instructions.py new file mode 100644 index 0000000000000000000000000000000000000000..9a7bcce13b0f29b829f21dea14b8f7ce5baeaac1 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/ifeval/instructions.py @@ -0,0 +1,1612 @@ +# Copyright 2023 The Google Research Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Library of instructions.""" + +import collections +import json +import logging +import random +import re +import string +from typing import Dict, Optional, Sequence, Union + +import langdetect + +from lm_eval.tasks.ifeval import instructions_util + + +logger = logging.getLogger(__name__) + +_InstructionArgsDtype = Optional[Dict[str, Union[int, str, Sequence[str]]]] + +_LANGUAGES = instructions_util.LANGUAGE_CODES + +# The relational operation for comparison. +_COMPARISON_RELATION = ("less than", "at least") + +# The maximum number of sentences. +_MAX_NUM_SENTENCES = 20 + +# The number of placeholders. +_NUM_PLACEHOLDERS = 4 + +# The number of bullet lists. +_NUM_BULLETS = 5 + +# The options of constrained response. +_CONSTRAINED_RESPONSE_OPTIONS = ( + "My answer is yes.", + "My answer is no.", + "My answer is maybe.", +) + +# The options of starter keywords. +_STARTER_OPTIONS = ( + "I would say", + "My answer is", + "I believe", + "In my opinion", + "I think", + "I reckon", + "I feel", + "From my perspective", + "As I see it", + "According to me", + "As far as I'm concerned", + "To my understanding", + "In my view", + "My take on it is", + "As per my perception", +) + +# The options of ending keywords. +# TODO(jeffreyzhou) add more ending options +_ENDING_OPTIONS = ("Any other questions?", "Is there anything else I can help with?") + +# The number of highlighted sections. +_NUM_HIGHLIGHTED_SECTIONS = 4 + +# The section splitter. +_SECTION_SPLITER = ("Section", "SECTION") + +# The number of sections. +_NUM_SECTIONS = 5 + +# The number of paragraphs. +_NUM_PARAGRAPHS = 5 + +# The postscript marker. +_POSTSCRIPT_MARKER = ("P.S.", "P.P.S") + +# The number of keywords. +_NUM_KEYWORDS = 2 + +# The occurrences of a single keyword. +_KEYWORD_FREQUENCY = 3 + +# The occurrences of a single letter. +_LETTER_FREQUENCY = 10 + +# The occurrences of words with all capital letters. +_ALL_CAPITAL_WORD_FREQUENCY = 20 + +# The number of words in the response. +_NUM_WORDS_LOWER_LIMIT = 100 +_NUM_WORDS_UPPER_LIMIT = 500 + + +class Instruction: + """An instruction template.""" + + def __init__(self, instruction_id): + self.id = instruction_id + + def build_description(self, **kwargs): + raise NotImplementedError("`build_description` not implemented.") + + def get_instruction_args(self): + raise NotImplementedError("`get_instruction_args` not implemented.") + + def get_instruction_args_keys(self): + raise NotImplementedError("`get_instruction_args_keys` not implemented.") + + def check_following(self, value): + raise NotImplementedError("`check_following` not implemented.") + + +class ResponseLanguageChecker(Instruction): + """Check the language of the entire response.""" + + def build_description(self, *, language=None): + """Build the instruction description. + + Args: + language: A string representing the expected language of the response. The + language has to comply to the 97 types defined in + `langid.py` (https://pypi.org/project/langid/1.1.5/), which follows + ISO 639-1 codes (https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes); + for example, `en` for English, `zh` for Chinese, `fr` for French. + + Returns: + A string representing the instruction description. + """ + self._language = language + if self._language is None: + self._language = random.choice(list(_LANGUAGES.keys())) + # TODO(tianjianlu): opens the description generation to more choices. + self._description_pattern = ( + "Your ENTIRE response should be in {language} language, no other " + + "language is allowed." + ) + return self._description_pattern.format(language=_LANGUAGES[self._language]) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"language": self._language} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["language"] + + def check_following(self, value): + """Check if the language of the entire response follows the instruction. + + Args: + value: A string representing the response. + + Returns: + True if the language of `value` follows instruction; otherwise False. + """ + assert isinstance(value, str) + + try: + return langdetect.detect(value) == self._language + except langdetect.LangDetectException as e: + # Count as instruction is followed. + logging.error( + "Unable to detect language for text %s due to %s", value, e + ) # refex: disable=pytotw.037 + return True + + +class NumberOfSentences(Instruction): + """Check the number of sentences.""" + + def build_description(self, *, num_sentences=None, relation=None): + """Build the instruction description. + + Args: + num_sentences: An integer specifying the number of sentences as a + threshold. + relation: A string in (`less than`, `at least`), defining the relational + operator for comparison. + Two relational comparisons are supported for now: + if 'less than', the actual number of sentences < the threshold; + if 'at least', the actual number of sentences >= the threshold. + + Returns: + A string representing the instruction description. + """ + # The number of sentences as a threshold for comparison. + self._num_sentences_threshold = num_sentences + if self._num_sentences_threshold is None or self._num_sentences_threshold < 0: + self._num_sentences_threshold = random.randint(1, _MAX_NUM_SENTENCES) + + if relation is None: + self._comparison_relation = random.choice(_COMPARISON_RELATION) + elif relation not in _COMPARISON_RELATION: + raise ValueError( + "The supported relation for comparison must be in " + f"{_COMPARISON_RELATION}, but {relation} is given." + ) + else: + self._comparison_relation = relation + + self._description_pattern = ( + "Your response should contain {relation} {num_sentences} sentences." + ) + return self._description_pattern.format( + relation=self._comparison_relation, + num_sentences=self._num_sentences_threshold, + ) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return { + "num_sentences": self._num_sentences_threshold, + "relation": self._comparison_relation, + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_sentences", "relation"] + + def check_following(self, value): + """Check if the number of sentences follows the instruction. + + Args: + value: A string representing the response. + + Returns: + True if the response follows the instruction. + + Raise: + ValueError if the string in `instruction_args` is not in + [`less_than`, `at_least`]. + """ + num_sentences = instructions_util.count_sentences(value) + if self._comparison_relation == _COMPARISON_RELATION[0]: + return num_sentences < self._num_sentences_threshold + elif self._comparison_relation == _COMPARISON_RELATION[1]: + return num_sentences >= self._num_sentences_threshold + + +class PlaceholderChecker(Instruction): + """Check the placeholders in template writing.""" + + def build_description(self, *, num_placeholders=None): + """Build the instruction description. + + Args: + num_placeholders: An integer denoting the minimum number of + placeholders required in the response. + + Returns: + A string representing the instruction description. + """ + self._num_placeholders = num_placeholders + if self._num_placeholders is None or self._num_placeholders < 0: + self._num_placeholders = random.randint(1, _NUM_PLACEHOLDERS) + self._description_pattern = ( + "The response must contain at least {num_placeholders} placeholders " + + "represented by square brackets, such as [address]." + ) + return self._description_pattern.format(num_placeholders=self._num_placeholders) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"num_placeholders": self._num_placeholders} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_placeholders"] + + def check_following(self, value): + """Check if the number of placeholders follows the instruction. + + Args: + value: A string representing the response. + + Returns: + True if the actual number of placeholders in the response is greater than + or equal to `num_placeholders`; otherwise, False. + """ + placeholders = re.findall(r"\[.*?\]", value) + num_placeholders = len(placeholders) + return num_placeholders >= self._num_placeholders + + +class BulletListChecker(Instruction): + """Checks the bullet list in the prompt.""" + + def build_description(self, *, num_bullets=None): + """Build the instruction description. + + Args: + num_bullets: An integer specifying the exact number of bullet lists + that is required to appear in the response. + + Returns: + A string representing the instruction description. + """ + self._num_bullets = num_bullets + if self._num_bullets is None or self._num_bullets < 0: + self._num_bullets = random.randint(1, _NUM_BULLETS) + self._description_pattern = ( + "Your answer must contain exactly {num_bullets} bullet points. " + + "Use the markdown bullet points such as:\n" + + "* This is point 1. \n" + + "* This is point 2" + ) + return self._description_pattern.format(num_bullets=self._num_bullets) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"num_bullets": self._num_bullets} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_bullets"] + + def check_following(self, value): + r"""Check if the number of bullet lists meets the requirement. + + Args: + value: A string representing the response. The response is expected to + contain some bullet lists that start with `\*`. + + Returns: + True if the actual number of bullet lists in the response meets the + requirement. + """ + bullet_lists = re.findall(r"^\s*\*[^\*].*$", value, flags=re.MULTILINE) + bullet_lists_2 = re.findall(r"^\s*-.*$", value, flags=re.MULTILINE) + num_bullet_lists = len(bullet_lists) + len(bullet_lists_2) + return num_bullet_lists == self._num_bullets + + +class ConstrainedResponseChecker(Instruction): + """Checks the constrained response.""" + + def build_description(self): + """Build the instruction description.""" + # A sequence of string(s) representing the options of the expected response. + self._constrained_responses = _CONSTRAINED_RESPONSE_OPTIONS + self._description_pattern = ( + "Answer with one of the following options: {response_options}" + ) + return self._description_pattern.format( + response_options=self._constrained_responses + ) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response matches the constrained options. + + Args: + value: A string representing the response. + + Returns: + True if the actual response contains one of the options in the constrained + responses; otherwise False. + """ + value = value.strip() + for constrained_response in self._constrained_responses: + if constrained_response in value: + return True + return False + + +class ConstrainedStartChecker(Instruction): + """Checks the response start.""" + + def build_description(self, *, starter=None): + """Build the instruction description. + + Args: + starter: A string representing the keyword that the response should start + with. + + Returns: + A string representing the instruction description. + """ + self._starter = starter.strip() if isinstance(starter, str) else starter + if self._starter is None: + self._starter = random.choice(_STARTER_OPTIONS) + self._description_pattern = ( + "During the conversation, when it is your turn, " + + "please always start with {starter}" + ) + return self._description_pattern.format(starter=self._starter) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"starter": self._starter} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["starter"] + + def check_following(self, value): + """Checks if the response starts with the constrained keyword or phrase. + + Args: + value: A string representing the response. + + Returns: + True if the response starts with the given phrase or keyword that is + contained in `instruction_args`; otherwise, False. + """ + response_pattern = r"^\s*" + self._starter + r".*$" + response_with_constrained_start = re.search( + response_pattern, value, flags=re.MULTILINE + ) + return True if response_with_constrained_start else False + + +class HighlightSectionChecker(Instruction): + """Checks the highlighted section.""" + + def build_description(self, *, num_highlights=None): + """Build the instruction description. + + Args: + num_highlights: An integer specifying the minimum number of highlighted + sections. + + Returns: + A string representing the instruction description. + """ + self._num_highlights = num_highlights + if self._num_highlights is None or self._num_highlights < 0: + self._num_highlights = random.randint(1, _NUM_HIGHLIGHTED_SECTIONS) + + self._description_pattern = ( + "Highlight at least {num_highlights} sections in your answer with " + + "markdown, i.e. *highlighted section*." + ) + + return self._description_pattern.format(num_highlights=self._num_highlights) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"num_highlights": self._num_highlights} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_highlights"] + + def check_following(self, value): + """Checks if the number of highlighted sections meets the requirement. + + Args: + value: a string representing the response. The response is expected to + contain highlighted sections in the format of *highlighted*. + + Returns: + True if the actual number of highlighted sections in the format of + *highlighted sections* meets the minimum requirement; otherwise False. + """ + num_highlights = 0 + highlights = re.findall(r"\*[^\n\*]*\*", value) + double_highlights = re.findall(r"\*\*[^\n\*]*\*\*", value) + for highlight in highlights: + if highlight.strip("*").strip(): + num_highlights += 1 + for highlight in double_highlights: + if highlight.removeprefix("**").removesuffix("**").strip(): + num_highlights += 1 + + return num_highlights >= self._num_highlights + + +class SectionChecker(Instruction): + """Checks the sections.""" + + def build_description(self, *, section_spliter=None, num_sections=None): + """Build the instruction description. + + Args: + section_spliter: A string represents the section spliter keyword that + marks a new section, i.e., `Section` or `SECTION`. + num_sections: An integer specifying the number of sections. + + Returns: + A string representing the instruction description. + """ + self._section_spliter = ( + section_spliter.strip() + if isinstance(section_spliter, str) + else section_spliter + ) + if self._section_spliter is None: + self._section_spliter = random.choice(_SECTION_SPLITER) + + self._num_sections = num_sections + if self._num_sections is None or self._num_sections < 0: + self._num_sections = random.randint(1, _NUM_SECTIONS) + + self._description_pattern = ( + "Your response must have {num_sections} sections. Mark the beginning " + + "of each section with {section_spliter} X, such as:\n" + + "{section_spliter} 1\n" + + "[content of section 1]\n" + + "{section_spliter} 2\n" + + "[content of section 2]" + ) + + return self._description_pattern.format( + num_sections=self._num_sections, section_spliter=self._section_spliter + ) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return { + "section_spliter": self._section_spliter, + "num_sections": self._num_sections, + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["section_spliter", "num_sections"] + + def check_following(self, value): + """Checks the response contains multiple sections. + + Args: + value: A string representing the response. The response is expected + to contain multiple sections (number of sections is greater than 1). + A new section starts with `Section 1`, where the number denotes the + section index. + + Returns: + True if the number of sections in the response is greater than or equal to + the minimum number of sections; otherwise, False. + """ + section_splitter_patten = r"\s?" + self._section_spliter + r"\s?\d+\s?" + sections = re.split(section_splitter_patten, value) + num_sections = len(sections) - 1 + return num_sections >= self._num_sections + + +class ParagraphChecker(Instruction): + """Checks the paragraphs.""" + + def build_description(self, *, num_paragraphs=None): + """Build the instruction description. + + Args: + num_paragraphs: An integer specifying the number of paragraphs. + + Returns: + A string representing the instruction description. + """ + self._num_paragraphs = num_paragraphs + if self._num_paragraphs is None or self._num_paragraphs < 0: + self._num_paragraphs = random.randint(1, _NUM_PARAGRAPHS) + + self._description_pattern = ( + "There should be {num_paragraphs} paragraphs. " + + "Paragraphs are separated with the markdown divider: ***" + ) + + return self._description_pattern.format(num_paragraphs=self._num_paragraphs) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"num_paragraphs": self._num_paragraphs} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_paragraphs"] + + def check_following(self, value): + """Checks the response contains required number of paragraphs. + + Args: + value: A string representing the response. The response may contain + paragraphs that are separated by the markdown divider: `***`. + + Returns: + True if the actual number of paragraphs is the same as required; + otherwise, False. + """ + paragraphs = re.split(r"\s?\*\*\*\s?", value) + num_paragraphs = len(paragraphs) + + for index, paragraph in enumerate(paragraphs): + if not paragraph.strip(): + if index == 0 or index == len(paragraphs) - 1: + num_paragraphs -= 1 + else: + return False + + return num_paragraphs == self._num_paragraphs + + +class PostscriptChecker(Instruction): + """Checks the postscript.""" + + def build_description(self, *, postscript_marker=None): + """Build the instruction description. + + Args: + postscript_marker: A string containing the keyword that marks the start + of the postscript section. + + Returns: + A string representing the instruction description. + """ + self._postscript_marker = ( + postscript_marker.strip() + if isinstance(postscript_marker, str) + else postscript_marker + ) + if self._postscript_marker is None: + self._postscript_marker = random.choice(_POSTSCRIPT_MARKER) + + self._description_pattern = ( + "At the end of your response, please explicitly add a postscript " + + "starting with {postscript}" + ) + + return self._description_pattern.format(postscript=self._postscript_marker) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"postscript_marker": self._postscript_marker} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["postscript_marker"] + + def check_following(self, value): + """Checks if the response follows the postscript format. + + Args: + value: a string representing the response. The response is expected to + contain a postscript section. + + Returns: + True if the response contains a postscript section starting with + the keyword containing in the `instruction_args`; otherwise False. + """ + value = value.lower() + if self._postscript_marker == "P.P.S": + postscript_pattern = r"\s*p\.\s?p\.\s?s.*$" + elif self._postscript_marker == "P.S.": + postscript_pattern = r"\s*p\.\s?s\..*$" + else: + postscript_pattern = r"\s*" + self._postscript_marker.lower() + r".*$" + postscript = re.findall(postscript_pattern, value, flags=re.MULTILINE) + return True if postscript else False + + +class RephraseChecker(Instruction): + """Checks the rephrase.""" + + def build_description(self, *, original_message): + """Build the instruction description. + + Args: + original_message: A string representing the original message. The + rephrased response should only change its words/sentences in between + its two asterisks, for example, *change me*. Both original and rephrased + messages should contain the changes in the form of *change me*. + + Returns: + A string representing the instruction description. + """ + if not self.is_change(original_message): + raise ValueError( + f"Message {original_message} does not contain changes " + "in the form of *change me*." + ) + + self._reference_without_change = original_message + self._description = ( + "Rephrasing: Your rephrased response should only" + + "change the words/sentences in between two asterisks" + + "such as *change me*." + ) + return self._description + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"original_message": self._reference_without_change} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["original_message"] + + def check_following(self, value): + r"""Checks if the rephrasing follows the instruction. + + Args: + value: A string representing the response, which is expected to rephras + the string of `instruction_args`. + + Returns: + True if `value` and `instruction_args` only differ by the words/sentences + in between two asterisks such as *change me*; otherwise, False. + """ + + if not self.is_change(value): + raise ValueError( + f"value {value} does not contain changes in the form of *change me*." + ) + + response_without_changes = self.strip_changes(value) + reference_without_changes = self.strip_changes(self._reference_without_change) + + return response_without_changes == reference_without_changes + + def is_change(self, response): + """Check if there is change in the response in the form of *change me*.""" + return re.search(r"\*.*\*", response) + + def strip_changes(self, response): + """Strips off the changes.""" + return re.sub(r"\*.*\*", "", response) + + +class KeywordChecker(Instruction): + """Check the exisitence of certain keywords.""" + + def build_description(self, *, keywords=None): + """Build the instruction description. + + Args: + keywords: A sequence of strings representing the keywords that are + expected in the response. + + Returns: + A string representing the instruction description. + """ + + if not keywords: + self._keywords = instructions_util.generate_keywords( + num_keywords=_NUM_KEYWORDS + ) + else: + self._keywords = keywords + self._keywords = sorted(self._keywords) + + self._description_pattern = "Include keywords {keywords} in the response." + + return self._description_pattern.format(keywords=self._keywords) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"keywords": self._keywords} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["keywords"] + + def check_following(self, value): + """Check if the response contain the expected keywords.""" + for keyword in self._keywords: + if not re.search(keyword, value, flags=re.IGNORECASE): + return False + return True + + +class KeywordFrequencyChecker(Instruction): + """Check the keyword frequency.""" + + def build_description(self, *, keyword=None, frequency=None, relation=None): + """Build the instruction description. + + Args: + keyword: A string representing a keyword that is expected in the response. + frequency: An integer specifying the number of times `keyword` is expected + to appear in the response. + relation: A string in (`less than`, `at least`), defining the relational + operator for comparison. + Two relational comparisons are supported for now: + if 'less than', the actual number of occurrences < frequency; + if 'at least', the actual number of occurrences >= frequency. + + Returns: + A string representing the instruction description. + """ + if not keyword: + self._keyword = instructions_util.generate_keywords(num_keywords=1)[0] + else: + self._keyword = keyword.strip() + + self._frequency = frequency + if self._frequency is None or self._frequency < 0: + self._frequency = random.randint(1, _KEYWORD_FREQUENCY) + + if relation is None: + self._comparison_relation = random.choice(_COMPARISON_RELATION) + elif relation not in _COMPARISON_RELATION: + raise ValueError( + "The supported relation for comparison must be in " + f"{_COMPARISON_RELATION}, but {relation} is given." + ) + else: + self._comparison_relation = relation + + self._description_pattern = ( + "In your response, the word {keyword} should appear {relation} " + + "{frequency} times." + ) + + return self._description_pattern.format( + keyword=self._keyword, + relation=self._comparison_relation, + frequency=self._frequency, + ) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return { + "keyword": self._keyword, + "frequency": self._frequency, + "relation": self._comparison_relation, + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["keyword", "frequency", "relation"] + + def check_following(self, value): + """Checks if the response contain the keyword with required frequency.""" + actual_occurrences = len(re.findall(self._keyword, value, flags=re.IGNORECASE)) + + if self._comparison_relation == _COMPARISON_RELATION[0]: + return actual_occurrences < self._frequency + elif self._comparison_relation == _COMPARISON_RELATION[1]: + return actual_occurrences >= self._frequency + + +class NumberOfWords(Instruction): + """Checks the number of words.""" + + def build_description(self, *, num_words=None, relation=None): + """Build the instruction description. + + Args: + num_words: An integer specifying the number of words contained in the + response. + relation: A string in (`less than`, `at least`), defining the relational + operator for comparison. + Two relational comparisons are supported for now: + if 'less than', the actual number of words < num_words; + if 'at least', the actual number of words >= num_words. + + Returns: + A string representing the instruction description. + """ + + self._num_words = num_words + if self._num_words is None or self._num_words < 0: + self._num_words = random.randint( + _NUM_WORDS_LOWER_LIMIT, _NUM_WORDS_UPPER_LIMIT + ) + + if relation is None: + self._comparison_relation = random.choice(_COMPARISON_RELATION) + elif relation not in _COMPARISON_RELATION: + raise ValueError( + "The supported relation for comparison must be in " + f"{_COMPARISON_RELATION}, but {relation} is given." + ) + else: + self._comparison_relation = relation + + self._description_pattern = "Answer with {relation} {num_words} words." + + return self._description_pattern.format( + relation=self._comparison_relation, num_words=self._num_words + ) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"num_words": self._num_words, "relation": self._comparison_relation} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_words", "relation"] + + def check_following(self, value): + """Checks if the response contains the expected number of words.""" + num_words = instructions_util.count_words(value) + + if self._comparison_relation == _COMPARISON_RELATION[0]: + return num_words < self._num_words + elif self._comparison_relation == _COMPARISON_RELATION[1]: + return num_words >= self._num_words + + +class JsonFormat(Instruction): + """Check the Json format.""" + + def build_description(self): + self._description_pattern = ( + "Entire output should be wrapped in JSON format. You can use markdown" + " ticks such as ```." + ) + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + value = ( + value.strip() + .removeprefix("```json") + .removeprefix("```Json") + .removeprefix("```JSON") + .removeprefix("```") + .removesuffix("```") + .strip() + ) + try: + json.loads(value) + except ValueError: + return False + return True + + +class ParagraphFirstWordCheck(Instruction): + """Check the paragraph and the first word of the nth paragraph.""" + + def build_description( + self, num_paragraphs=None, nth_paragraph=None, first_word=None + ): + r"""Build the instruction description. + + Args: + num_paragraphs: An integer indicating the number of paragraphs expected + in the response. A paragraph is a subset of the string that is + expected to be separated by '\n\n'. + nth_paragraph: An integer indicating the paragraph number that we look at. + Note that n starts from 1. + first_word: A string that represent the first word of the bth paragraph. + + Returns: + A string representing the instruction description. + """ + self._num_paragraphs = num_paragraphs + if self._num_paragraphs is None or self._num_paragraphs < 0: + self._num_paragraphs = random.randint(1, _NUM_PARAGRAPHS) + + self._nth_paragraph = nth_paragraph + if ( + self._nth_paragraph is None + or self._nth_paragraph <= 0 + or self._nth_paragraph > self._num_paragraphs + ): + self._nth_paragraph = random.randint(1, self._num_paragraphs + 1) + + self._first_word = first_word + if self._first_word is None: + self._first_word = instructions_util.generate_keywords(num_keywords=1)[0] + self._first_word = self._first_word.lower() + + self._description_pattern = ( + "There should be {num_paragraphs} paragraphs. " + + "Paragraphs and only paragraphs are separated with each other by two " + + "new lines as if it was '\\n\\n' in python. " + + "Paragraph {nth_paragraph} must start with word {first_word}." + ) + + return self._description_pattern.format( + num_paragraphs=self._num_paragraphs, + nth_paragraph=self._nth_paragraph, + first_word=self._first_word, + ) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return { + "num_paragraphs": self._num_paragraphs, + "nth_paragraph": self._nth_paragraph, + "first_word": self._first_word, + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_paragraphs", "nth_paragraph", "first_word"] + + def check_following(self, value): + """Checks for required number of paragraphs and correct first word. + + Args: + value: a string representing the response. The response may contain + paragraphs that are separated by two new lines and the first word of + the nth paragraph will have to match a specified word. + + Returns: + True if the number of paragraphs is the same as required and the first + word of the specified paragraph is the same as required. Otherwise, false. + """ + + paragraphs = re.split(r"\n\n", value) + num_paragraphs = len(paragraphs) + + for paragraph in paragraphs: + if not paragraph.strip(): + num_paragraphs -= 1 + + # check that index doesn't go out of bounds + if self._nth_paragraph <= num_paragraphs: + paragraph = paragraphs[self._nth_paragraph - 1].strip() + if not paragraph: + return False + else: + return False + + first_word = "" + punctuation = {".", ",", "?", "!", "'", '"'} + + # get first word and remove punctuation + word = paragraph.split()[0].strip() + # TODO(jeffrey): make more complex? + word = word.lstrip("'") + word = word.lstrip('"') + + for letter in word: + if letter in punctuation: + break + first_word += letter.lower() + + return num_paragraphs == self._num_paragraphs and first_word == self._first_word + + +# TODO(jeffrey) add relation - at least/at most? +class KeySentenceChecker(Instruction): + """Check the existence of certain key sentences.""" + + def build_description(self, key_sentences=None, num_sentences=None): + """Build the instruction description. + + Args: + key_sentences: A sequences of strings representing the key sentences that + are expected in the response. + num_sentences: The number of key sentences that are expected to be seen in + the response. + + Returns: + A string representing the instruction description. + """ + + if not key_sentences: + # TODO(jeffrey) make a generate sentences function? wonderwords package + self._key_sentences = set(["For now, this is fine."]) + else: + self._key_sentences = key_sentences + + if not num_sentences: + self._num_sentences = random.randint(1, len(self._key_sentences)) + else: + self._num_sentences = num_sentences + + self._description_pattern = ( + "Include {num_sentences} of the following sentences {key_sentences}" + ) + + return self._description_pattern.format( + num_sentences=self._num_sentences, key_sentences=self._key_sentences + ) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return { + "num_sentences": self._num_sentences, + "key_sentences": list(self._key_sentences), + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_sentences", "key_sentences"] + + def check_following(self, value): + """Checks if the response contains the expected key sentences.""" + count = 0 + sentences = instructions_util.split_into_sentences(value) + for sentence in self._key_sentences: + if sentence in sentences: + count += 1 + + return count == self._num_sentences + + +class ForbiddenWords(Instruction): + """Checks that specified words are not used in response.""" + + def build_description(self, forbidden_words=None): + """Build the instruction description. + + Args: + forbidden_words: A sequences of strings representing words that are not + allowed in the response. + + Returns: + A string representing the instruction description. + """ + + if not forbidden_words: + self._forbidden_words = instructions_util.generate_keywords( + num_keywords=_NUM_KEYWORDS + ) + else: + self._forbidden_words = list(set(forbidden_words)) + self._forbidden_words = sorted(self._forbidden_words) + self._description_pattern = ( + "Do not include keywords {forbidden_words} in the response." + ) + + return self._description_pattern.format(forbidden_words=self._forbidden_words) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"forbidden_words": self._forbidden_words} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["forbidden_words"] + + def check_following(self, value): + """Check if the response does not contain the expected keywords.""" + for word in self._forbidden_words: + if re.search(r"\b" + word + r"\b", value, flags=re.IGNORECASE): + return False + return True + + +class RephraseParagraph(Instruction): + """Checks that the paragraph is rephrased.""" + + def build_description(self, *, original_paragraph, low, high): + """Builds the instruction description. + + Args: + original_paragraph: A string presenting the original paragraph. The + rephrases response should have betweeb low-high words in common. + low: An integer presenting the lower bound of similar words. + high: An integer representing the upper bound of similar words. + + Returns: + A string representing the instruction description. + """ + # TODO(jeffrey) make more encompassing + self._original_paragraph = original_paragraph + self._low = low + self._high = high + + self._description = ( + "Rephrase the following paragraph: " + + "{original_paragraph}\nYour response should have " + + "between {low} and {high} of the same words. " + + "Words are the same if and only if all of the " + + "letters, ignoring cases, are the same. For " + + "example, 'run' is the same as 'Run' but different " + + "to 'ran'." + ) + + return self._description.format( + original_paragraph=original_paragraph, low=self._low, high=self._high + ) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return { + "original_paragraph": self._original_paragraph, + "low": self._low, + "high": self._high, + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["original_paragraph", "low", "high"] + + def check_following(self, value): + val_words = re.findall(r"\w+", value.lower()) + original_words = re.findall(r"\w+", self._original_paragraph.lower()) + similar_words = 0 + + dict_val = collections.Counter(val_words) + dict_original = collections.Counter(original_words) + + for word in dict_original: + similar_words += min(dict_original[word], dict_val[word]) + + return similar_words >= self._low and similar_words <= self._high + + +class TwoResponsesChecker(Instruction): + """Check that two responses were given.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Give two different responses. Responses and only responses should" + " be separated by 6 asterisk symbols: ******." + ) + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response has two different answers. + + Args: + value: A string representing the response. + + Returns: + True if two responses are detected and false otherwise. + """ + valid_responses = list() + responses = value.split("******") + for index, response in enumerate(responses): + if not response.strip(): + if index != 0 and index != len(responses) - 1: + return False + else: + valid_responses.append(response) + return ( + len(valid_responses) == 2 + and valid_responses[0].strip() != valid_responses[1].strip() + ) + + +class RepeatPromptThenAnswer(Instruction): + """Checks that Prompt is first repeated then answered.""" + + def build_description(self, *, prompt_to_repeat=None): + """Build the instruction description. + + Args: + prompt_to_repeat: The prompt that is meant to be repeated. + + Returns: + A string representing the instruction description. + """ + if not prompt_to_repeat: + raise ValueError("prompt_to_repeat must be set.") + else: + self._prompt_to_repeat = prompt_to_repeat + self._description_pattern = ( + "First repeat the request word for word without change," + " then give your answer (1. do not say any words or characters" + " before repeating the request; 2. the request you need to repeat" + " does not include this sentence)" + ) + return self._description_pattern + + def get_instruction_args(self): + return {"prompt_to_repeat": self._prompt_to_repeat} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["prompt_to_repeat"] + + def check_following(self, value): + if value.strip().lower().startswith(self._prompt_to_repeat.strip().lower()): + return True + return False + + +class EndChecker(Instruction): + """Checks that the prompt ends with a given phrase.""" + + def build_description(self, *, end_phrase=None): + """Build the instruction description. + + Args: + end_phrase: A string representing the phrase the response should end with. + + Returns: + A string representing the instruction description. + """ + self._end_phrase = ( + end_phrase.strip() if isinstance(end_phrase, str) else end_phrase + ) + if self._end_phrase is None: + self._end_phrase = random.choice(_ENDING_OPTIONS) + self._description_pattern = ( + "Finish your response with this exact phrase {ender}. " + "No other words should follow this phrase." + ) + return self._description_pattern.format(ender=self._end_phrase) + + def get_instruction_args(self): + return {"end_phrase": self._end_phrase} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["end_phrase"] + + def check_following(self, value): + """Checks if the response ends with the expected phrase.""" + value = value.strip().strip('"').lower() + self._end_phrase = self._end_phrase.strip().lower() + return value.endswith(self._end_phrase) + + +class TitleChecker(Instruction): + """Checks the response for a title.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Your answer must contain a title, wrapped in double angular brackets," + " such as <>." + ) + return self._description_pattern + + def get_instruction_args(self): + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response contains a title.""" + pattern = r"<<[^\n]+>>" + re_pattern = re.compile(pattern) + titles = re.findall(re_pattern, value) + + for title in titles: + if title.lstrip("<").rstrip(">").strip(): + return True + return False + + +class LetterFrequencyChecker(Instruction): + """Checks letter frequency.""" + + def build_description(self, *, letter=None, let_frequency=None, let_relation=None): + """Build the instruction description. + + Args: + letter: A string representing a letter that is expected in the response. + let_frequency: An integer specifying the number of times `keyword` is + expected to appear in the response. + let_relation: A string in (`less than`, `at least`), defining the + relational operator for comparison. Two relational comparisons are + supported for now; if 'less than', the actual number of + occurrences < frequency; if 'at least', the actual number of + occurrences >= frequency. + + Returns: + A string representing the instruction description. + """ + if ( + not letter + or len(letter) > 1 + or ord(letter.lower()) < 97 + or ord(letter.lower()) > 122 + ): + self._letter = random.choice(list(string.ascii_letters)) + else: + self._letter = letter.strip() + self._letter = self._letter.lower() + + self._frequency = let_frequency + if self._frequency is None or self._frequency < 0: + self._frequency = random.randint(1, _LETTER_FREQUENCY) + + if let_relation is None: + self._comparison_relation = random.choice(_COMPARISON_RELATION) + elif let_relation not in _COMPARISON_RELATION: + raise ValueError( + "The supported relation for comparison must be in " + f"{_COMPARISON_RELATION}, but {let_relation} is given." + ) + else: + self._comparison_relation = let_relation + + self._description_pattern = ( + "In your response, the letter {letter} should appear {let_relation}" + " {let_frequency} times." + ) + + return self._description_pattern.format( + letter=self._letter, + let_frequency=self._frequency, + let_relation=self._comparison_relation, + ) + + def get_instruction_args(self): + """Returns the keyword args of build description.""" + return { + "letter": self._letter, + "let_frequency": self._frequency, + "let_relation": self._comparison_relation, + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["letter", "let_frequency", "let_relation"] + + def check_following(self, value): + """Checks that the response contains the letter at the right frequency.""" + value = value.lower() + letters = collections.Counter(value) + + if self._comparison_relation == _COMPARISON_RELATION[0]: + return letters[self._letter] < self._frequency + else: + return letters[self._letter] >= self._frequency + + +class CapitalLettersEnglishChecker(Instruction): + """Checks that the response is in english and is in all capital letters.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Your entire response should be in English, and in all capital letters." + ) + return self._description_pattern + + def get_instruction_args(self): + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks that the response is in English and in all capital letters.""" + assert isinstance(value, str) + + try: + return value.isupper() and langdetect.detect(value) == "en" + except langdetect.LangDetectException as e: + # Count as instruction is followed. + logging.error( + "Unable to detect language for text %s due to %s", value, e + ) # refex: disable=pytotw.037 + return True + + +class LowercaseLettersEnglishChecker(Instruction): + """Checks that the response is in english and is in all lowercase letters.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Your entire response should be in English, and in all lowercase" + " letters. No capital letters are allowed." + ) + return self._description_pattern + + def get_instruction_args(self): + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks that the response is in English and in all lowercase letters.""" + assert isinstance(value, str) + + try: + return value.islower() and langdetect.detect(value) == "en" + except langdetect.LangDetectException as e: + # Count as instruction is followed. + logging.error( + "Unable to detect language for text %s due to %s", value, e + ) # refex: disable=pytotw.037 + return True + + +class CommaChecker(Instruction): + """Checks the response for no commas.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "In your entire response, refrain from the use of any commas." + ) + return self._description_pattern + + def get_instruction_args(self): + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks that the response does not contain commas.""" + return not re.search(r"\,", value) + + +class CapitalWordFrequencyChecker(Instruction): + """Checks frequency of words with all capital letters.""" + + def build_description( + self, + capital_frequency=None, + capital_relation=None, + ): + """Build the instruction description. + + Args: + capital_frequency: An integer that represents the number of words that + should be in all capital letters. + capital_relation: A string that is 'at least' or 'at most' that refers to + the frequency. + + Returns: + A string representing the instruction description. + """ + self._frequency = capital_frequency + if self._frequency is None: + self._frequency = random.randint(1, _ALL_CAPITAL_WORD_FREQUENCY) + + self._comparison_relation = capital_relation + if capital_relation is None: + self._comparison_relation = random.choice(_COMPARISON_RELATION) + elif capital_relation not in _COMPARISON_RELATION: + raise ValueError( + "The supported relation for comparison must be in " + f"{_COMPARISON_RELATION}, but {capital_relation} is given." + ) + + self._description_pattern = ( + "In your response, words with all capital letters should appear" + " {relation} {frequency} times." + ) + + return self._description_pattern.format( + frequency=self._frequency, relation=self._comparison_relation + ) + + def get_instruction_args(self): + """Returns the keyword args of build description.""" + return { + "capital_frequency": self._frequency, + "capital_relation": self._comparison_relation, + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["capital_frequency", "capital_relation"] + + def check_following(self, value): + """Checks the frequency of words with all capital letters.""" + # Hyphenated words will count as one word + words = instructions_util.nltk.word_tokenize(value) + capital_words = [word for word in words if word.isupper()] + + capital_words = len(capital_words) + + if self._comparison_relation == _COMPARISON_RELATION[0]: + return capital_words < self._frequency + else: + return capital_words >= self._frequency + + +class QuotationChecker(Instruction): + """Checks response is wrapped with double quotation marks.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Wrap your entire response with double quotation marks." + ) + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of build description.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response is wrapped with double quotation marks.""" + value = value.strip() + return len(value) > 1 and value[0] == '"' and value[-1] == '"' diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/ifeval/instructions_registry.py b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/ifeval/instructions_registry.py new file mode 100644 index 0000000000000000000000000000000000000000..00d9a1de1985beacead34215952ecf4642d1ea35 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/ifeval/instructions_registry.py @@ -0,0 +1,168 @@ +# Copyright 2023 The Google Research Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Registry of all instructions.""" + +from lm_eval.tasks.ifeval import instructions + + +_KEYWORD = "keywords:" + +_LANGUAGE = "language:" + +_LENGTH = "length_constraints:" + +_CONTENT = "detectable_content:" + +_FORMAT = "detectable_format:" + +_MULTITURN = "multi-turn:" + +_COMBINATION = "combination:" + +_STARTEND = "startend:" + +_CHANGE_CASES = "change_case:" + +_PUNCTUATION = "punctuation:" + +INSTRUCTION_DICT = { + _KEYWORD + "existence": instructions.KeywordChecker, + _KEYWORD + "frequency": instructions.KeywordFrequencyChecker, + # TODO(jeffreyzhou): make a proper set of sentences to choose from + # _KEYWORD + "key_sentences": instructions.KeySentenceChecker, + _KEYWORD + "forbidden_words": instructions.ForbiddenWords, + _KEYWORD + "letter_frequency": instructions.LetterFrequencyChecker, + _LANGUAGE + "response_language": instructions.ResponseLanguageChecker, + _LENGTH + "number_sentences": instructions.NumberOfSentences, + _LENGTH + "number_paragraphs": instructions.ParagraphChecker, + _LENGTH + "number_words": instructions.NumberOfWords, + _LENGTH + "nth_paragraph_first_word": instructions.ParagraphFirstWordCheck, + _CONTENT + "number_placeholders": instructions.PlaceholderChecker, + _CONTENT + "postscript": instructions.PostscriptChecker, + _FORMAT + "number_bullet_lists": instructions.BulletListChecker, + # TODO(jeffreyzhou): Pre-create paragraph or use prompt to replace + # _CONTENT + "rephrase_paragraph": instructions.RephraseParagraph, + _FORMAT + "constrained_response": instructions.ConstrainedResponseChecker, + _FORMAT + "number_highlighted_sections": (instructions.HighlightSectionChecker), + _FORMAT + "multiple_sections": instructions.SectionChecker, + # TODO(tianjianlu): Re-enable rephrasing with preprocessing the message. + # _FORMAT + "rephrase": instructions.RephraseChecker, + _FORMAT + "json_format": instructions.JsonFormat, + _FORMAT + "title": instructions.TitleChecker, + # TODO(tianjianlu): Re-enable with specific prompts. + # _MULTITURN + "constrained_start": instructions.ConstrainedStartChecker, + _COMBINATION + "two_responses": instructions.TwoResponsesChecker, + _COMBINATION + "repeat_prompt": instructions.RepeatPromptThenAnswer, + _STARTEND + "end_checker": instructions.EndChecker, + _CHANGE_CASES + "capital_word_frequency": instructions.CapitalWordFrequencyChecker, + _CHANGE_CASES + "english_capital": instructions.CapitalLettersEnglishChecker, + _CHANGE_CASES + "english_lowercase": instructions.LowercaseLettersEnglishChecker, + _PUNCTUATION + "no_comma": instructions.CommaChecker, + _STARTEND + "quotation": instructions.QuotationChecker, +} + +INSTRUCTION_CONFLICTS = { + _KEYWORD + "existence": {_KEYWORD + "existence"}, + _KEYWORD + "frequency": {_KEYWORD + "frequency"}, + # TODO(jeffreyzhou): make a proper set of sentences to choose from + # _KEYWORD + "key_sentences": instructions.KeySentenceChecker, + _KEYWORD + "forbidden_words": {_KEYWORD + "forbidden_words"}, + _KEYWORD + "letter_frequency": {_KEYWORD + "letter_frequency"}, + _LANGUAGE + "response_language": { + _LANGUAGE + "response_language", + _FORMAT + "multiple_sections", + _KEYWORD + "existence", + _KEYWORD + "frequency", + _KEYWORD + "forbidden_words", + _STARTEND + "end_checker", + _CHANGE_CASES + "english_capital", + _CHANGE_CASES + "english_lowercase", + }, + _LENGTH + "number_sentences": {_LENGTH + "number_sentences"}, + _LENGTH + "number_paragraphs": { + _LENGTH + "number_paragraphs", + _LENGTH + "nth_paragraph_first_word", + _LENGTH + "number_sentences", + _LENGTH + "nth_paragraph_first_word", + }, + _LENGTH + "number_words": {_LENGTH + "number_words"}, + _LENGTH + "nth_paragraph_first_word": { + _LENGTH + "nth_paragraph_first_word", + _LENGTH + "number_paragraphs", + }, + _CONTENT + "number_placeholders": {_CONTENT + "number_placeholders"}, + _CONTENT + "postscript": {_CONTENT + "postscript"}, + _FORMAT + "number_bullet_lists": {_FORMAT + "number_bullet_lists"}, + # TODO(jeffreyzhou): Pre-create paragraph or use prompt to replace + # _CONTENT + "rephrase_paragraph": instructions.RephraseParagraph, + _FORMAT + "constrained_response": set(INSTRUCTION_DICT.keys()), + _FORMAT + "number_highlighted_sections": {_FORMAT + "number_highlighted_sections"}, + _FORMAT + "multiple_sections": { + _FORMAT + "multiple_sections", + _LANGUAGE + "response_language", + _FORMAT + "number_highlighted_sections", + }, + # TODO(tianjianlu): Re-enable rephrasing with preprocessing the message. + # _FORMAT + "rephrase": instructions.RephraseChecker, + _FORMAT + "json_format": set(INSTRUCTION_DICT.keys()).difference( + {_KEYWORD + "forbidden_words", _KEYWORD + "existence"} + ), + _FORMAT + "title": {_FORMAT + "title"}, + # TODO(tianjianlu): Re-enable with specific prompts. + # _MULTITURN + "constrained_start": instructions.ConstrainedStartChecker, + _COMBINATION + "two_responses": set(INSTRUCTION_DICT.keys()).difference( + { + _KEYWORD + "forbidden_words", + _KEYWORD + "existence", + _LANGUAGE + "response_language", + _FORMAT + "title", + _PUNCTUATION + "no_comma", + } + ), + _COMBINATION + "repeat_prompt": set(INSTRUCTION_DICT.keys()).difference( + {_KEYWORD + "existence", _FORMAT + "title", _PUNCTUATION + "no_comma"} + ), + _STARTEND + "end_checker": {_STARTEND + "end_checker"}, + _CHANGE_CASES + "capital_word_frequency": { + _CHANGE_CASES + "capital_word_frequency", + _CHANGE_CASES + "english_lowercase", + _CHANGE_CASES + "english_capital", + }, + _CHANGE_CASES + "english_capital": {_CHANGE_CASES + "english_capital"}, + _CHANGE_CASES + "english_lowercase": { + _CHANGE_CASES + "english_lowercase", + _CHANGE_CASES + "english_capital", + }, + _PUNCTUATION + "no_comma": {_PUNCTUATION + "no_comma"}, + _STARTEND + "quotation": {_STARTEND + "quotation", _FORMAT + "title"}, +} + + +def conflict_make(conflicts): + """Makes sure if A conflicts with B, B will conflict with A. + + Args: + conflicts: Dictionary of potential conflicts where key is instruction id + and value is set of instruction ids that it conflicts with. + + Returns: + Revised version of the dictionary. All instructions conflict with + themselves. If A conflicts with B, B will conflict with A. + """ + for key in conflicts: + for k in conflicts[key]: + conflicts[k].add(key) + conflicts[key].add(key) + return conflicts diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/ifeval/instructions_util.py b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/ifeval/instructions_util.py new file mode 100644 index 0000000000000000000000000000000000000000..33e0a0a00c54f301334dc1bcd211dd588e6c9529 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/ifeval/instructions_util.py @@ -0,0 +1,1701 @@ +# Copyright 2023 The Google Research Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utility library of instructions.""" + +import functools +import os +import random +import re +from importlib.metadata import version + +import immutabledict +import nltk +from packaging.version import parse as parse_version + + +# Downloading 'punkt' with nltk<3.9 has a remote code vuln. +# see https://github.com/EleutherAI/lm-evaluation-harness/issues/2210 +# and https://github.com/nltk/nltk/issues/3266 +# for more information. +NLTK_MIN_VERSION = "3.9.1" +RANK = os.environ.get("LOCAL_RANK", "0") + + +def download_nltk_resources(): + """Download 'punkt' if not already installed""" + assert (nltk_version := parse_version(version("nltk"))) >= parse_version( + NLTK_MIN_VERSION + ), ( + f"`nltk` version {nltk_version} is not >= {NLTK_MIN_VERSION}. Please update `nltk` before proceeding--older versions are vulnerable to a remote code execution vulnerability." + ) + + try: + nltk.data.find("tokenizers/punkt_tab") + except LookupError: + if RANK == "0": + nltk.download("punkt_tab") + print("Downloaded punkt_tab on rank 0") + + +download_nltk_resources() + +WORD_LIST = [ + "western", + "sentence", + "signal", + "dump", + "spot", + "opposite", + "bottom", + "potato", + "administration", + "working", + "welcome", + "morning", + "good", + "agency", + "primary", + "wish", + "responsibility", + "press", + "problem", + "president", + "steal", + "brush", + "read", + "type", + "beat", + "trainer", + "growth", + "lock", + "bone", + "case", + "equal", + "comfortable", + "region", + "replacement", + "performance", + "mate", + "walk", + "medicine", + "film", + "thing", + "rock", + "tap", + "total", + "competition", + "ease", + "south", + "establishment", + "gather", + "parking", + "world", + "plenty", + "breath", + "claim", + "alcohol", + "trade", + "dear", + "highlight", + "street", + "matter", + "decision", + "mess", + "agreement", + "studio", + "coach", + "assist", + "brain", + "wing", + "style", + "private", + "top", + "brown", + "leg", + "buy", + "procedure", + "method", + "speed", + "high", + "company", + "valuable", + "pie", + "analyst", + "session", + "pattern", + "district", + "pleasure", + "dinner", + "swimming", + "joke", + "order", + "plate", + "department", + "motor", + "cell", + "spend", + "cabinet", + "difference", + "power", + "examination", + "engine", + "horse", + "dimension", + "pay", + "toe", + "curve", + "literature", + "bother", + "fire", + "possibility", + "debate", + "activity", + "passage", + "hello", + "cycle", + "background", + "quiet", + "author", + "effect", + "actor", + "page", + "bicycle", + "error", + "throat", + "attack", + "character", + "phone", + "tea", + "increase", + "outcome", + "file", + "specific", + "inspector", + "internal", + "potential", + "staff", + "building", + "employer", + "shoe", + "hand", + "direction", + "garden", + "purchase", + "interview", + "study", + "recognition", + "member", + "spiritual", + "oven", + "sandwich", + "weird", + "passenger", + "particular", + "response", + "reaction", + "size", + "variation", + "a", + "cancel", + "candy", + "exit", + "guest", + "condition", + "fly", + "price", + "weakness", + "convert", + "hotel", + "great", + "mouth", + "mind", + "song", + "sugar", + "suspect", + "telephone", + "ear", + "roof", + "paint", + "refrigerator", + "organization", + "jury", + "reward", + "engineering", + "day", + "possession", + "crew", + "bar", + "road", + "description", + "celebration", + "score", + "mark", + "letter", + "shower", + "suggestion", + "sir", + "luck", + "national", + "progress", + "hall", + "stroke", + "theory", + "offer", + "story", + "tax", + "definition", + "history", + "ride", + "medium", + "opening", + "glass", + "elevator", + "stomach", + "question", + "ability", + "leading", + "village", + "computer", + "city", + "grand", + "confidence", + "candle", + "priest", + "recommendation", + "point", + "necessary", + "body", + "desk", + "secret", + "horror", + "noise", + "culture", + "warning", + "water", + "round", + "diet", + "flower", + "bus", + "tough", + "permission", + "week", + "prompt", + "connection", + "abuse", + "height", + "save", + "corner", + "border", + "stress", + "drive", + "stop", + "rip", + "meal", + "listen", + "confusion", + "girlfriend", + "living", + "relation", + "significance", + "plan", + "creative", + "atmosphere", + "blame", + "invite", + "housing", + "paper", + "drink", + "roll", + "silver", + "drunk", + "age", + "damage", + "smoke", + "environment", + "pack", + "savings", + "influence", + "tourist", + "rain", + "post", + "sign", + "grandmother", + "run", + "profit", + "push", + "clerk", + "final", + "wine", + "swim", + "pause", + "stuff", + "singer", + "funeral", + "average", + "source", + "scene", + "tradition", + "personal", + "snow", + "nobody", + "distance", + "sort", + "sensitive", + "animal", + "major", + "negotiation", + "click", + "mood", + "period", + "arrival", + "expression", + "holiday", + "repeat", + "dust", + "closet", + "gold", + "bad", + "sail", + "combination", + "clothes", + "emphasis", + "duty", + "black", + "step", + "school", + "jump", + "document", + "professional", + "lip", + "chemical", + "front", + "wake", + "while", + "inside", + "watch", + "row", + "subject", + "penalty", + "balance", + "possible", + "adult", + "aside", + "sample", + "appeal", + "wedding", + "depth", + "king", + "award", + "wife", + "blow", + "site", + "camp", + "music", + "safe", + "gift", + "fault", + "guess", + "act", + "shame", + "drama", + "capital", + "exam", + "stupid", + "record", + "sound", + "swing", + "novel", + "minimum", + "ratio", + "machine", + "shape", + "lead", + "operation", + "salary", + "cloud", + "affair", + "hit", + "chapter", + "stage", + "quantity", + "access", + "army", + "chain", + "traffic", + "kick", + "analysis", + "airport", + "time", + "vacation", + "philosophy", + "ball", + "chest", + "thanks", + "place", + "mountain", + "advertising", + "red", + "past", + "rent", + "return", + "tour", + "house", + "construction", + "net", + "native", + "war", + "figure", + "fee", + "spray", + "user", + "dirt", + "shot", + "task", + "stick", + "friend", + "software", + "promotion", + "interaction", + "surround", + "block", + "purpose", + "practice", + "conflict", + "routine", + "requirement", + "bonus", + "hole", + "state", + "junior", + "sweet", + "catch", + "tear", + "fold", + "wall", + "editor", + "life", + "position", + "pound", + "respect", + "bathroom", + "coat", + "script", + "job", + "teach", + "birth", + "view", + "resolve", + "theme", + "employee", + "doubt", + "market", + "education", + "serve", + "recover", + "tone", + "harm", + "miss", + "union", + "understanding", + "cow", + "river", + "association", + "concept", + "training", + "recipe", + "relationship", + "reserve", + "depression", + "proof", + "hair", + "revenue", + "independent", + "lift", + "assignment", + "temporary", + "amount", + "loss", + "edge", + "track", + "check", + "rope", + "estimate", + "pollution", + "stable", + "message", + "delivery", + "perspective", + "mirror", + "assistant", + "representative", + "witness", + "nature", + "judge", + "fruit", + "tip", + "devil", + "town", + "emergency", + "upper", + "drop", + "stay", + "human", + "neck", + "speaker", + "network", + "sing", + "resist", + "league", + "trip", + "signature", + "lawyer", + "importance", + "gas", + "choice", + "engineer", + "success", + "part", + "external", + "worker", + "simple", + "quarter", + "student", + "heart", + "pass", + "spite", + "shift", + "rough", + "lady", + "grass", + "community", + "garage", + "youth", + "standard", + "skirt", + "promise", + "blind", + "television", + "disease", + "commission", + "positive", + "energy", + "calm", + "presence", + "tune", + "basis", + "preference", + "head", + "common", + "cut", + "somewhere", + "presentation", + "current", + "thought", + "revolution", + "effort", + "master", + "implement", + "republic", + "floor", + "principle", + "stranger", + "shoulder", + "grade", + "button", + "tennis", + "police", + "collection", + "account", + "register", + "glove", + "divide", + "professor", + "chair", + "priority", + "combine", + "peace", + "extension", + "maybe", + "evening", + "frame", + "sister", + "wave", + "code", + "application", + "mouse", + "match", + "counter", + "bottle", + "half", + "cheek", + "resolution", + "back", + "knowledge", + "make", + "discussion", + "screw", + "length", + "accident", + "battle", + "dress", + "knee", + "log", + "package", + "it", + "turn", + "hearing", + "newspaper", + "layer", + "wealth", + "profile", + "imagination", + "answer", + "weekend", + "teacher", + "appearance", + "meet", + "bike", + "rise", + "belt", + "crash", + "bowl", + "equivalent", + "support", + "image", + "poem", + "risk", + "excitement", + "remote", + "secretary", + "public", + "produce", + "plane", + "display", + "money", + "sand", + "situation", + "punch", + "customer", + "title", + "shake", + "mortgage", + "option", + "number", + "pop", + "window", + "extent", + "nothing", + "experience", + "opinion", + "departure", + "dance", + "indication", + "boy", + "material", + "band", + "leader", + "sun", + "beautiful", + "muscle", + "farmer", + "variety", + "fat", + "handle", + "director", + "opportunity", + "calendar", + "outside", + "pace", + "bath", + "fish", + "consequence", + "put", + "owner", + "go", + "doctor", + "information", + "share", + "hurt", + "protection", + "career", + "finance", + "force", + "golf", + "garbage", + "aspect", + "kid", + "food", + "boot", + "milk", + "respond", + "objective", + "reality", + "raw", + "ring", + "mall", + "one", + "impact", + "area", + "news", + "international", + "series", + "impress", + "mother", + "shelter", + "strike", + "loan", + "month", + "seat", + "anything", + "entertainment", + "familiar", + "clue", + "year", + "glad", + "supermarket", + "natural", + "god", + "cost", + "conversation", + "tie", + "ruin", + "comfort", + "earth", + "storm", + "percentage", + "assistance", + "budget", + "strength", + "beginning", + "sleep", + "other", + "young", + "unit", + "fill", + "store", + "desire", + "hide", + "value", + "cup", + "maintenance", + "nurse", + "function", + "tower", + "role", + "class", + "camera", + "database", + "panic", + "nation", + "basket", + "ice", + "art", + "spirit", + "chart", + "exchange", + "feedback", + "statement", + "reputation", + "search", + "hunt", + "exercise", + "nasty", + "notice", + "male", + "yard", + "annual", + "collar", + "date", + "platform", + "plant", + "fortune", + "passion", + "friendship", + "spread", + "cancer", + "ticket", + "attitude", + "island", + "active", + "object", + "service", + "buyer", + "bite", + "card", + "face", + "steak", + "proposal", + "patient", + "heat", + "rule", + "resident", + "broad", + "politics", + "west", + "knife", + "expert", + "girl", + "design", + "salt", + "baseball", + "grab", + "inspection", + "cousin", + "couple", + "magazine", + "cook", + "dependent", + "security", + "chicken", + "version", + "currency", + "ladder", + "scheme", + "kitchen", + "employment", + "local", + "attention", + "manager", + "fact", + "cover", + "sad", + "guard", + "relative", + "county", + "rate", + "lunch", + "program", + "initiative", + "gear", + "bridge", + "breast", + "talk", + "dish", + "guarantee", + "beer", + "vehicle", + "reception", + "woman", + "substance", + "copy", + "lecture", + "advantage", + "park", + "cold", + "death", + "mix", + "hold", + "scale", + "tomorrow", + "blood", + "request", + "green", + "cookie", + "church", + "strip", + "forever", + "beyond", + "debt", + "tackle", + "wash", + "following", + "feel", + "maximum", + "sector", + "sea", + "property", + "economics", + "menu", + "bench", + "try", + "language", + "start", + "call", + "solid", + "address", + "income", + "foot", + "senior", + "honey", + "few", + "mixture", + "cash", + "grocery", + "link", + "map", + "form", + "factor", + "pot", + "model", + "writer", + "farm", + "winter", + "skill", + "anywhere", + "birthday", + "policy", + "release", + "husband", + "lab", + "hurry", + "mail", + "equipment", + "sink", + "pair", + "driver", + "consideration", + "leather", + "skin", + "blue", + "boat", + "sale", + "brick", + "two", + "feed", + "square", + "dot", + "rush", + "dream", + "location", + "afternoon", + "manufacturer", + "control", + "occasion", + "trouble", + "introduction", + "advice", + "bet", + "eat", + "kill", + "category", + "manner", + "office", + "estate", + "pride", + "awareness", + "slip", + "crack", + "client", + "nail", + "shoot", + "membership", + "soft", + "anybody", + "web", + "official", + "individual", + "pizza", + "interest", + "bag", + "spell", + "profession", + "queen", + "deal", + "resource", + "ship", + "guy", + "chocolate", + "joint", + "formal", + "upstairs", + "car", + "resort", + "abroad", + "dealer", + "associate", + "finger", + "surgery", + "comment", + "team", + "detail", + "crazy", + "path", + "tale", + "initial", + "arm", + "radio", + "demand", + "single", + "draw", + "yellow", + "contest", + "piece", + "quote", + "pull", + "commercial", + "shirt", + "contribution", + "cream", + "channel", + "suit", + "discipline", + "instruction", + "concert", + "speech", + "low", + "effective", + "hang", + "scratch", + "industry", + "breakfast", + "lay", + "join", + "metal", + "bedroom", + "minute", + "product", + "rest", + "temperature", + "many", + "give", + "argument", + "print", + "purple", + "laugh", + "health", + "credit", + "investment", + "sell", + "setting", + "lesson", + "egg", + "middle", + "marriage", + "level", + "evidence", + "phrase", + "love", + "self", + "benefit", + "guidance", + "affect", + "you", + "dad", + "anxiety", + "special", + "boyfriend", + "test", + "blank", + "payment", + "soup", + "obligation", + "reply", + "smile", + "deep", + "complaint", + "addition", + "review", + "box", + "towel", + "minor", + "fun", + "soil", + "issue", + "cigarette", + "internet", + "gain", + "tell", + "entry", + "spare", + "incident", + "family", + "refuse", + "branch", + "can", + "pen", + "grandfather", + "constant", + "tank", + "uncle", + "climate", + "ground", + "volume", + "communication", + "kind", + "poet", + "child", + "screen", + "mine", + "quit", + "gene", + "lack", + "charity", + "memory", + "tooth", + "fear", + "mention", + "marketing", + "reveal", + "reason", + "court", + "season", + "freedom", + "land", + "sport", + "audience", + "classroom", + "law", + "hook", + "win", + "carry", + "eye", + "smell", + "distribution", + "research", + "country", + "dare", + "hope", + "whereas", + "stretch", + "library", + "if", + "delay", + "college", + "plastic", + "book", + "present", + "use", + "worry", + "champion", + "goal", + "economy", + "march", + "election", + "reflection", + "midnight", + "slide", + "inflation", + "action", + "challenge", + "guitar", + "coast", + "apple", + "campaign", + "field", + "jacket", + "sense", + "way", + "visual", + "remove", + "weather", + "trash", + "cable", + "regret", + "buddy", + "beach", + "historian", + "courage", + "sympathy", + "truck", + "tension", + "permit", + "nose", + "bed", + "son", + "person", + "base", + "meat", + "usual", + "air", + "meeting", + "worth", + "game", + "independence", + "physical", + "brief", + "play", + "raise", + "board", + "she", + "key", + "writing", + "pick", + "command", + "party", + "yesterday", + "spring", + "candidate", + "physics", + "university", + "concern", + "development", + "change", + "string", + "target", + "instance", + "room", + "bitter", + "bird", + "football", + "normal", + "split", + "impression", + "wood", + "long", + "meaning", + "stock", + "cap", + "leadership", + "media", + "ambition", + "fishing", + "essay", + "salad", + "repair", + "today", + "designer", + "night", + "bank", + "drawing", + "inevitable", + "phase", + "vast", + "chip", + "anger", + "switch", + "cry", + "twist", + "personality", + "attempt", + "storage", + "being", + "preparation", + "bat", + "selection", + "white", + "technology", + "contract", + "side", + "section", + "station", + "till", + "structure", + "tongue", + "taste", + "truth", + "difficulty", + "group", + "limit", + "main", + "move", + "feeling", + "light", + "example", + "mission", + "might", + "wait", + "wheel", + "shop", + "host", + "classic", + "alternative", + "cause", + "agent", + "consist", + "table", + "airline", + "text", + "pool", + "craft", + "range", + "fuel", + "tool", + "partner", + "load", + "entrance", + "deposit", + "hate", + "article", + "video", + "summer", + "feature", + "extreme", + "mobile", + "hospital", + "flight", + "fall", + "pension", + "piano", + "fail", + "result", + "rub", + "gap", + "system", + "report", + "suck", + "ordinary", + "wind", + "nerve", + "ask", + "shine", + "note", + "line", + "mom", + "perception", + "brother", + "reference", + "bend", + "charge", + "treat", + "trick", + "term", + "homework", + "bake", + "bid", + "status", + "project", + "strategy", + "orange", + "let", + "enthusiasm", + "parent", + "concentrate", + "device", + "travel", + "poetry", + "business", + "society", + "kiss", + "end", + "vegetable", + "employ", + "schedule", + "hour", + "brave", + "focus", + "process", + "movie", + "illegal", + "general", + "coffee", + "ad", + "highway", + "chemistry", + "psychology", + "hire", + "bell", + "conference", + "relief", + "show", + "neat", + "funny", + "weight", + "quality", + "club", + "daughter", + "zone", + "touch", + "tonight", + "shock", + "burn", + "excuse", + "name", + "survey", + "landscape", + "advance", + "satisfaction", + "bread", + "disaster", + "item", + "hat", + "prior", + "shopping", + "visit", + "east", + "photo", + "home", + "idea", + "father", + "comparison", + "cat", + "pipe", + "winner", + "count", + "lake", + "fight", + "prize", + "foundation", + "dog", + "keep", + "ideal", + "fan", + "struggle", + "peak", + "safety", + "solution", + "hell", + "conclusion", + "population", + "strain", + "alarm", + "measurement", + "second", + "train", + "race", + "due", + "insurance", + "boss", + "tree", + "monitor", + "sick", + "course", + "drag", + "appointment", + "slice", + "still", + "care", + "patience", + "rich", + "escape", + "emotion", + "royal", + "female", + "childhood", + "government", + "picture", + "will", + "sock", + "big", + "gate", + "oil", + "cross", + "pin", + "improvement", + "championship", + "silly", + "help", + "sky", + "pitch", + "man", + "diamond", + "most", + "transition", + "work", + "science", + "committee", + "moment", + "fix", + "teaching", + "dig", + "specialist", + "complex", + "guide", + "people", + "dead", + "voice", + "original", + "break", + "topic", + "data", + "degree", + "reading", + "recording", + "bunch", + "reach", + "judgment", + "lie", + "regular", + "set", + "painting", + "mode", + "list", + "player", + "bear", + "north", + "wonder", + "carpet", + "heavy", + "officer", + "negative", + "clock", + "unique", + "baby", + "pain", + "assumption", + "disk", + "iron", + "bill", + "drawer", + "look", + "double", + "mistake", + "finish", + "future", + "brilliant", + "contact", + "math", + "rice", + "leave", + "restaurant", + "discount", + "sex", + "virus", + "bit", + "trust", + "event", + "wear", + "juice", + "failure", + "bug", + "context", + "mud", + "whole", + "wrap", + "intention", + "draft", + "pressure", + "cake", + "dark", + "explanation", + "space", + "angle", + "word", + "efficiency", + "management", + "habit", + "star", + "chance", + "finding", + "transportation", + "stand", + "criticism", + "flow", + "door", + "injury", + "insect", + "surprise", + "apartment", +] # pylint: disable=line-too-long + +# ISO 639-1 codes to language names. +LANGUAGE_CODES = immutabledict.immutabledict( + { + "en": "English", + "es": "Spanish", + "pt": "Portuguese", + "ar": "Arabic", + "hi": "Hindi", + "fr": "French", + "ru": "Russian", + "de": "German", + "ja": "Japanese", + "it": "Italian", + "bn": "Bengali", + "uk": "Ukrainian", + "th": "Thai", + "ur": "Urdu", + "ta": "Tamil", + "te": "Telugu", + "bg": "Bulgarian", + "ko": "Korean", + "pl": "Polish", + "he": "Hebrew", + "fa": "Persian", + "vi": "Vietnamese", + "ne": "Nepali", + "sw": "Swahili", + "kn": "Kannada", + "mr": "Marathi", + "gu": "Gujarati", + "pa": "Punjabi", + "ml": "Malayalam", + "fi": "Finnish", + } +) + +_ALPHABETS = "([A-Za-z])" +_PREFIXES = "(Mr|St|Mrs|Ms|Dr)[.]" +_SUFFIXES = "(Inc|Ltd|Jr|Sr|Co)" +_STARTERS = r"(Mr|Mrs|Ms|Dr|Prof|Capt|Cpt|Lt|He\s|She\s|It\s|They\s|Their\s|Our\s|We\s|But\s|However\s|That\s|This\s|Wherever)" +_ACRONYMS = "([A-Z][.][A-Z][.](?:[A-Z][.])?)" +_WEBSITES = "[.](com|net|org|io|gov|edu|me)" +_DIGITS = "([0-9])" +_MULTIPLE_DOTS = r"\.{2,}" + + +def split_into_sentences(text): + """Split the text into sentences. + + Args: + text: A string that consists of more than or equal to one sentences. + + Returns: + A list of strings where each string is a sentence. + """ + text = " " + text + " " + text = text.replace("\n", " ") + text = re.sub(_PREFIXES, "\\1", text) + text = re.sub(_WEBSITES, "\\1", text) + text = re.sub(_DIGITS + "[.]" + _DIGITS, "\\1\\2", text) + text = re.sub( + _MULTIPLE_DOTS, + lambda match: "" * len(match.group(0)) + "", + text, + ) + if "Ph.D" in text: + text = text.replace("Ph.D.", "PhD") + text = re.sub(r"\s" + _ALPHABETS + "[.] ", " \\1 ", text) + text = re.sub(_ACRONYMS + " " + _STARTERS, "\\1 \\2", text) + text = re.sub( + _ALPHABETS + "[.]" + _ALPHABETS + "[.]" + _ALPHABETS + "[.]", + "\\1\\2\\3", + text, + ) + text = re.sub(_ALPHABETS + "[.]" + _ALPHABETS + "[.]", "\\1\\2", text) + text = re.sub(" " + _SUFFIXES + "[.] " + _STARTERS, " \\1 \\2", text) + text = re.sub(" " + _SUFFIXES + "[.]", " \\1", text) + text = re.sub(" " + _ALPHABETS + "[.]", " \\1", text) + if "”" in text: + text = text.replace(".”", "”.") + if '"' in text: + text = text.replace('."', '".') + if "!" in text: + text = text.replace('!"', '"!') + if "?" in text: + text = text.replace('?"', '"?') + text = text.replace(".", ".") + text = text.replace("?", "?") + text = text.replace("!", "!") + text = text.replace("", ".") + sentences = text.split("") + sentences = [s.strip() for s in sentences] + if sentences and not sentences[-1]: + sentences = sentences[:-1] + return sentences + + +def count_words(text): + """Counts the number of words.""" + tokenizer = nltk.tokenize.RegexpTokenizer(r"\w+") + tokens = tokenizer.tokenize(text) + num_words = len(tokens) + return num_words + + +@functools.lru_cache(maxsize=None) +def _get_sentence_tokenizer(): + return nltk.data.load("nltk:tokenizers/punkt/english.pickle") + + +def count_sentences(text): + """Count the number of sentences.""" + tokenizer = _get_sentence_tokenizer() + tokenized_sentences = tokenizer.tokenize(text) + return len(tokenized_sentences) + + +def generate_keywords(num_keywords): + """Randomly generates a few keywords.""" + return random.sample(WORD_LIST, k=num_keywords) diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/ifeval/utils.py b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/ifeval/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..985e8d5ae578c484267c7c2d90ee7c896028941f --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/ifeval/utils.py @@ -0,0 +1,134 @@ +import dataclasses +from typing import Dict, Optional, Union + +from lm_eval.tasks.ifeval import instructions_registry + + +@dataclasses.dataclass +class InputExample: + key: int + instruction_id_list: list[str] + prompt: str + kwargs: list[Dict[str, Optional[Union[str, int]]]] + + +@dataclasses.dataclass +class OutputExample: + instruction_id_list: list[str] + prompt: str + response: str + follow_all_instructions: bool + follow_instruction_list: list[bool] + + +def test_instruction_following_strict( + inp, + response, +): + """Tests response to see if instructions are followed.""" + instruction_list = inp.instruction_id_list + is_following_list = [] + + for index, instruction_id in enumerate(instruction_list): + instruction_cls = instructions_registry.INSTRUCTION_DICT[instruction_id] + instruction = instruction_cls(instruction_id) + + # Remove None values from kwargs to avoid unexpected keyword argument errors in build_description method. + kwargs = {k: v for k, v in inp.kwargs[index].items() if v} + instruction.build_description(**kwargs) + args = instruction.get_instruction_args() + if args and "prompt" in args: + instruction.build_description(prompt=inp.prompt) + + if response.strip() and instruction.check_following(response): + is_following_list.append(True) + else: + is_following_list.append(False) + + return OutputExample( + instruction_id_list=inp.instruction_id_list, + prompt=inp.prompt, + response=response, + follow_all_instructions=all(is_following_list), + follow_instruction_list=is_following_list, + ) + + +def test_instruction_following_loose( + inp, + response, +): + """Tests response for an upper bound for following instructions.""" + r = response.split("\n") + response_remove_first = "\n".join(r[1:]).strip() + response_remove_last = "\n".join(r[:-1]).strip() + response_remove_both = "\n".join(r[1:-1]).strip() + revised_response = response.replace("*", "") + revised_response_remove_first = response_remove_first.replace("*", "") + revised_response_remove_last = response_remove_last.replace("*", "") + revised_response_remove_both = response_remove_both.replace("*", "") + all_responses = [ + response, + revised_response, + response_remove_first, + response_remove_last, + response_remove_both, + revised_response_remove_first, + revised_response_remove_last, + revised_response_remove_both, + ] + instruction_list = inp.instruction_id_list + is_following_list = [] + + for index, instruction_id in enumerate(instruction_list): + instruction_cls = instructions_registry.INSTRUCTION_DICT[instruction_id] + instruction = instruction_cls(instruction_id) + + # Remove None values from kwargs to avoid unexpected keyword argument errors in build_description method. + kwargs = {k: v for k, v in inp.kwargs[index].items() if v} + instruction.build_description(**kwargs) + args = instruction.get_instruction_args() + if args and "prompt" in args: + instruction.build_description(prompt=inp.prompt) + + is_following = False + for r in all_responses: + if r.strip() and instruction.check_following(r): + is_following = True + break + + is_following_list.append(is_following) + + return OutputExample( + instruction_id_list=inp.instruction_id_list, + prompt=inp.prompt, + response=response, + follow_all_instructions=all(is_following_list), + follow_instruction_list=is_following_list, + ) + + +def process_results(doc, results): + inp = InputExample( + key=doc["key"], + instruction_id_list=doc["instruction_id_list"], + prompt=doc["prompt"], + kwargs=doc["kwargs"], + ) + response = results[0] + + out_strict = test_instruction_following_strict(inp, response) + out_loose = test_instruction_following_loose(inp, response) + + return { + "prompt_level_strict_acc": out_strict.follow_all_instructions, + "inst_level_strict_acc": out_strict.follow_instruction_list, + "prompt_level_loose_acc": out_loose.follow_all_instructions, + "inst_level_loose_acc": out_loose.follow_instruction_list, + } + + +def agg_inst_level_acc(items): + flat_items = [item for sublist in items for item in sublist] + inst_level_acc = sum(flat_items) / len(flat_items) + return inst_level_acc diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/math500/math500.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/math500/math500.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1fe2f7a38417fe863c1301953be514b618054707 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/math500/math500.yaml @@ -0,0 +1,12 @@ +task: math500 +dataset_path: HuggingFaceH4/MATH-500 +output_type: generate_until +test_split: test +doc_to_text: !function utils.math500_prompt +doc_to_target: "{{answer}}" +generation_kwargs: + until: + - "[NO_UNTIL_PLACEHOLDER]" + do_sample: false +repeats: 1 +num_fewshot: 0 diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/math500/utils.py b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/math500/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..e0585298c29c8b5c12ebeaa01dfff572267db601 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/math500/utils.py @@ -0,0 +1,14 @@ +def math500_prompt(doc): + system_prompt = ( + "You are a math expert. You will be given a question to solve. Solve it step by step. Wrap the final answer in a \\boxed{}. \n" + "Respond in the following format:\n" + "\n" + "Your reasoning here\n" + "\n" + "\n" + "\\boxed{...}\n" + "" + ) + + prompt = f"{system_prompt}\n\n{doc['problem']}\n\n" + return prompt diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mbpp/README.md b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mbpp/README.md new file mode 100644 index 0000000000000000000000000000000000000000..fd6df44fb76a1a9e017d60afc470200967696f19 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mbpp/README.md @@ -0,0 +1,43 @@ +# MBPP + +## Paper +Program Synthesis with Large Language Models +https://arxiv.org/abs/2108.07732 + +This paper explores the limits of the current generation of large language models for program synthesis in general purpose programming languages. We evaluate a collection of such models (with between 244M and 137B parameters) on two new benchmarks, MBPP and MathQA-Python, in both the few-shot and fine-tuning regimes. Our benchmarks are designed to measure the ability of these models to synthesize short Python programs from natural language descriptions. The Mostly Basic Programming Problems (MBPP) dataset contains 974 programming tasks, designed to be solvable by entry-level programmers. The MathQA-Python dataset, a Python version of the MathQA benchmark, contains 23914 problems that evaluate the ability of the models to synthesize code from more complex text. On both datasets, we find that synthesis performance scales log-linearly with model size. Our largest models, even without finetuning on a code dataset, can synthesize solutions to 59.6 percent of the problems from MBPP using few-shot learning with a well-designed prompt. Fine-tuning on a held-out portion of the dataset improves performance by about 10 percentage points across most model sizes. On the MathQA-Python dataset, the largest fine-tuned model achieves 83.8 percent accuracy. Going further, we study the model's ability to engage in dialog about code, incorporating human feedback to improve its solutions. We find that natural language feedback from a human halves the error rate compared to the model's initial prediction. Additionally, we conduct an error analysis to shed light on where these models fall short and what types of programs are most difficult to generate. Finally, we explore the semantic grounding of these models by fine-tuning them to predict the results of program execution. We find that even our best models are generally unable to predict the output of a program given a specific input. + +Homepage: https://github.com/google-research/google-research/tree/master/mbpp + + +## Citation +``` +@article{austin2021program, + title={Program synthesis with large language models}, + author={Austin, Jacob and Odena, Augustus and Nye, Maxwell and Bosma, Maarten and Michalewski, Henryk and Dohan, David and Jiang, Ellen and Cai, Carrie and Terry, Michael and Le, Quoc and others}, + journal={arXiv preprint arXiv:2108.07732}, + year={2021} +} +``` + +### Groups and Tasks + +#### Groups + +* Not part of a group yet. + +#### Tasks + +- `mbpp` + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [ ] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mbpp/mbpp.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mbpp/mbpp.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9819870348baef1f450df6199348b626d59e4793 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mbpp/mbpp.yaml @@ -0,0 +1,23 @@ +task: mbpp +dataset_path: google-research-datasets/mbpp +dataset_name: full +unsafe_code: true +output_type: generate_until +test_split: test +doc_to_text: "You are an expert Python programmer, and here is your task: {{text}} Your code should pass these tests:\n\n{{test_list[0]}}\n{{test_list[1]}}\n{{test_list[2]}}\n[BEGIN]\n" +doc_to_target: "{% if is_fewshot is defined %}{{code}}\n[DONE]{% else %}{{test_list[0]}}\n{{test_list[1]}}\n{{test_list[2]}}{% endif %}" +target_delimiter: "" +metric_list: + - metric: !function utils.pass_at_1 + aggregation: mean + higher_is_better: true +generation_kwargs: + until: + - "[DONE]" + do_sample: false +num_fewshot: 0 +fewshot_config: + sampler: first_n + samples: !function utils.list_fewshot_samples +metadata: + version: 1.0 diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mbpp/mbpp_instruct.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mbpp/mbpp_instruct.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1fac69ebc030494d7529ddd0677576eae2de17b2 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mbpp/mbpp_instruct.yaml @@ -0,0 +1,24 @@ +task: mbpp_instruct +dataset_path: google-research-datasets/mbpp +dataset_name: full +unsafe_code: true +output_type: generate_until +test_split: test +doc_to_text: "You are an expert Python programmer, and here is your task: {{text}} Your code should pass these tests:\n\n{{test_list[0]}}\n{{test_list[1]}}\n{{test_list[2]}}" +doc_to_target: "{% if is_fewshot is defined %}{{code}}\n[DONE]{% else %}{{test_list[0]}}\n{{test_list[1]}}\n{{test_list[2]}}{% endif %}" +gen_prefix: "Here is the completed function:\n```python\n" +target_delimiter: "" +metric_list: + - metric: !function utils.pass_at_1 + aggregation: mean + higher_is_better: true +generation_kwargs: + until: + - "```" + do_sample: false +num_fewshot: 0 +fewshot_config: + sampler: first_n + samples: !function utils.list_fewshot_samples +metadata: + version: 1.0 diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mbpp/mbpp_plus.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mbpp/mbpp_plus.yaml new file mode 100644 index 0000000000000000000000000000000000000000..133c393c55e175ca958e5861cac91c8ec4f6beed --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mbpp/mbpp_plus.yaml @@ -0,0 +1,5 @@ +include: mbpp.yaml +task: mbpp_plus +dataset_path: evalplus/mbppplus +dataset_name: null +doc_to_text: "You are an expert Python programmer, and here is your task: {{prompt if prompt is defined else text}} Your code should pass these tests:\n\n{{test_list[0]}}\n{{test_list[1]}}\n{{test_list[2]}}\n[BEGIN]\n" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mbpp/mbpp_plus_instruct.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mbpp/mbpp_plus_instruct.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f1124cc0629b4e12ecd1764bbd19d282320645f0 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mbpp/mbpp_plus_instruct.yaml @@ -0,0 +1,5 @@ +include: mbpp_instruct.yaml +task: mbpp_plus_instruct +dataset_path: evalplus/mbppplus +dataset_name: null +doc_to_text: "You are an expert Python programmer, and here is your task: {{prompt if prompt is defined else text}} Your code should pass these tests:\n\n{{test_list[0]}}\n{{test_list[1]}}\n{{test_list[2]}}" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mbpp/utils.py b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mbpp/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f5ee7cddc127575bae2ae77530936e8c792b0065 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mbpp/utils.py @@ -0,0 +1,59 @@ +import evaluate as hf_evaluate + + +try: + pass_at_k = hf_evaluate.load("code_eval") + + # run simple test to check code execution is enabled before model generation + test_cases = ["assert add(2, 3)==5"] + candidates = [["def add(a,b): return a*b"]] + results = pass_at_k.compute(references=test_cases, predictions=candidates, k=[1]) +except Exception as e: + raise e + + +def pass_at_1(references, predictions): + print(predictions) + return pass_at_k.compute( + references=references, + predictions=[predictions], + k=[1], + )[0]["pass@1"] + + +def list_fewshot_samples(): + return [ + { + "task_id": 2, + "text": "Write a function to find the similar elements from the given two tuple lists.", + "code": "def similar_elements(test_tup1, test_tup2):\r\n res = tuple(set(test_tup1) & set(test_tup2))\r\n return (res) ", + "test_list": [ + "assert similar_elements((3, 4, 5, 6),(5, 7, 4, 10)) == (4, 5)", + "assert similar_elements((1, 2, 3, 4),(5, 4, 3, 7)) == (3, 4)", + "assert similar_elements((11, 12, 14, 13),(17, 15, 14, 13)) == (13, 14)", + ], + "is_fewshot": True, + }, + { + "task_id": 3, + "text": "Write a python function to identify non-prime numbers.", + "code": "import math\r\ndef is_not_prime(n):\r\n result = False\r\n for i in range(2,int(math.sqrt(n)) + 1):\r\n if n % i == 0:\r\n result = True\r\n return result", + "test_list": [ + "assert is_not_prime(2) == False", + "assert is_not_prime(10) == True", + "assert is_not_prime(35) == True", + ], + "is_fewshot": True, + }, + { + "task_id": 4, + "text": "Write a function to find the largest integers from a given list of numbers using heap queue algorithm.", + "code": "import heapq as hq\r\ndef heap_queue_largest(nums,n):\r\n largest_nums = hq.nlargest(n, nums)\r\n return largest_nums", + "test_list": [ + "assert heap_queue_largest( [25, 35, 22, 85, 14, 65, 75, 22, 58],3)==[85, 75, 65] ", + "assert heap_queue_largest( [25, 35, 22, 85, 14, 65, 75, 22, 58],2)==[85, 75] ", + "assert heap_queue_largest( [25, 35, 22, 85, 14, 65, 75, 22, 58],5)==[85, 75, 65, 58, 35]", + ], + "is_fewshot": True, + }, + ] diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/minerva_math/README.md b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/minerva_math/README.md new file mode 100644 index 0000000000000000000000000000000000000000..4cd78f76eb927db8f059fbba1a2e2bbe5a7ce03f --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/minerva_math/README.md @@ -0,0 +1,68 @@ +# MATH +ℹ️ This is the 4-shot variant! +## Paper +Measuring Mathematical Problem Solving With the MATH Dataset +https://arxiv.org/abs/2103.03874 + +Many intellectual endeavors require mathematical problem solving, but this skill remains beyond the capabilities of computers. To measure this ability in machine learning models, we introduce MATH, a new dataset of 12,500 challenging competition mathematics problems. Each problem in MATH has a full step-by-step solution which can be used to teach models to generate answer derivations and explanations. + +NOTE: The few-shot and the generated answer extraction is based on the [Minerva](https://arxiv.org/abs/2206.14858) and exact match equivalence is calculated using the `sympy` library. This requires additional dependencies, which can be installed via the `lm-eval[math]` extra. + +Homepage: https://github.com/hendrycks/math + + +## Citation +``` +@article{hendrycksmath2021, + title={Measuring Mathematical Problem Solving With the MATH Dataset}, + author={Dan Hendrycks and Collin Burns and Saurav Kadavath and Akul Arora and Steven Basart and Eric Tang and Dawn Song and Jacob Steinhardt}, + journal={NeurIPS}, + year={2021} +} + +@misc{2206.14858, +Author = {Aitor Lewkowycz and Anders Andreassen and David Dohan and Ethan Dyer and Henryk Michalewski and Vinay Ramasesh and Ambrose Slone and Cem Anil and Imanol Schlag and Theo Gutman-Solo and Yuhuai Wu and Behnam Neyshabur and Guy Gur-Ari and Vedant Misra}, +Title = {Solving Quantitative Reasoning Problems with Language Models}, +Year = {2022}, +Eprint = {arXiv:2206.14858}, +} +``` + +### Groups and Tasks + +#### Groups + +- `minerva_math` + +#### Tasks + +- `minerva_math_algebra` +- `minerva_math_counting_and_prob` +- `minerva_math_geometry` +- `minerva_math_intermediate_algebra` +- `minerva_math_num_theory` +- `minerva_math_prealgebra` +- `minerva_math_precalc` + +### Checklist + +The checklist is the following: + +For adding novel benchmarks/datasets to the library: +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + * The implementation in the original paper is one where the model is first fine-tuned on the data. They do have a few-shot evaluation for GPT-3, however the few-shot context used here is sourced from [Lewkowycz et al](https://arxiv.org/abs/2206.14858). The achieved accuracy on Llama-2 models is comparable to that provided in the paper, though not identical. + + +If other tasks on this dataset are already supported: +* [x] Is the "Main" variant of this task clearly denoted? +* [x] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [x] Have you noted which, if any, published evaluation setups are matched by this variant? + +### Variant Wishlist + +- [ ] zero-shot variant + +### Changelog +version 2.0: (21-Feb-2025); added math_verify (extraction) metric. For details [see](https://huggingface.co/blog/math_verify_leaderboard) diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/minerva_math/minerva_math_algebra.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/minerva_math/minerva_math_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..daf84f0accbd322f24019aa632cae74988f2cb11 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/minerva_math/minerva_math_algebra.yaml @@ -0,0 +1,32 @@ +tag: + - math_word_problems +task: minerva_math_algebra +dataset_path: EleutherAI/hendrycks_math +process_docs: !function utils.process_docs +dataset_name: algebra +output_type: generate_until +training_split: train +test_split: test +doc_to_text: !function utils.doc_to_text +process_results: !function utils.process_results +doc_to_target: "{{answer if few_shot is undefined else solution}}" +generation_kwargs: + until: + - "Problem:" + do_sample: false + temperature: 0 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + - metric: math_verify + aggregation: mean + higher_is_better: true +num_fewshot: 0 +metadata: + version: 2.0 +dataset_kwargs: + trust_remote_code: true +fewshot_config: + sampler: first_n + samples: !function utils.list_fewshot_samples diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/minerva_math/minerva_math_counting_and_prob.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/minerva_math/minerva_math_counting_and_prob.yaml new file mode 100644 index 0000000000000000000000000000000000000000..688cd711c50d005d5d78ca55116ad333d96161ce --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/minerva_math/minerva_math_counting_and_prob.yaml @@ -0,0 +1,3 @@ +include: minerva_math_algebra.yaml +dataset_name: counting_and_probability +task: minerva_math_counting_and_prob diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/minerva_math/minerva_math_geometry.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/minerva_math/minerva_math_geometry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..079ee70e9ed8997f351d1732c0c88dad1e4896de --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/minerva_math/minerva_math_geometry.yaml @@ -0,0 +1,3 @@ +include: minerva_math_algebra.yaml +dataset_name: geometry +task: minerva_math_geometry diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/minerva_math/minerva_math_intermediate_algebra.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/minerva_math/minerva_math_intermediate_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7b3f063c36e10063dd06be93c290820a787ddd1d --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/minerva_math/minerva_math_intermediate_algebra.yaml @@ -0,0 +1,3 @@ +include: minerva_math_algebra.yaml +dataset_name: intermediate_algebra +task: minerva_math_intermediate_algebra diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/minerva_math/minerva_math_num_theory.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/minerva_math/minerva_math_num_theory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..44f587bce4cce5e4ab80d24b938b88488553d6da --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/minerva_math/minerva_math_num_theory.yaml @@ -0,0 +1,3 @@ +include: minerva_math_algebra.yaml +dataset_name: number_theory +task: minerva_math_num_theory diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/minerva_math/minerva_math_prealgebra.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/minerva_math/minerva_math_prealgebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..865e2f2c6e5397a07fb473a89f4d8eaf47d3eb52 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/minerva_math/minerva_math_prealgebra.yaml @@ -0,0 +1,3 @@ +include: minerva_math_algebra.yaml +dataset_name: prealgebra +task: minerva_math_prealgebra diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/minerva_math/minerva_math_precalc.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/minerva_math/minerva_math_precalc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..06e63abc7c206b43759217b38cd5db2395e554a9 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/minerva_math/minerva_math_precalc.yaml @@ -0,0 +1,3 @@ +include: minerva_math_algebra.yaml +dataset_name: precalculus +task: minerva_math_precalc diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/minerva_math/utils.py b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/minerva_math/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..984ba33f229d624c9fc6036fa8f05e4da9d5cca4 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/minerva_math/utils.py @@ -0,0 +1,306 @@ +import logging +import re +import signal +from importlib.metadata import version +from typing import Dict, List, Optional + +import datasets + + +eval_logger = logging.getLogger(__name__) + + +try: + import antlr4 + import sympy + from math_verify import parse, verify + from sympy.parsing.latex import parse_latex + + assert version("antlr4-python3-runtime").startswith("4.11") +except (ModuleNotFoundError, AssertionError) as e: + raise type(e)( + "`sympy`, `math_verify` and `antlr4-python3-runtime==4.11` are required for generating translation task prompt templates. " + "Please install the required packages via pip install lm-eval[math] or pip install -e .[math]" + ) from e + + +# taken from +# https://github.com/wellecks/lm-evaluation-harness/blob/master/lm_eval/tasks/minerva_math.py +def doc_to_text(doc: dict) -> str: + return "Problem:" + "\n" + doc["problem"] + "\n\n" + "Solution:" + + +def process_docs(dataset: datasets.Dataset) -> datasets.Dataset: + def _process_doc(doc: dict) -> dict: + out_doc = { + "problem": doc["problem"], + "solution": doc["solution"], + "answer": normalize_final_answer( + remove_boxed(last_boxed_only_string(doc["solution"])) + ), + } + if getattr(doc, "few_shot", None) is not None: + out_doc["few_shot"] = True + return out_doc + + return dataset.map(_process_doc) + + +def list_fewshot_samples() -> list[dict]: + return [ + { + "problem": "Find the domain of the expression $\\frac{\\sqrt{x-2}}{\\sqrt{5-x}}$.}", + "solution": "The expressions inside each square root must be non-negative. Therefore, $x-2 \\ge 0$, so $x\\ge2$, and $5 - x \\ge 0$, so $x \\le 5$. Also, the denominator cannot be equal to zero, so $5-x>0$, which gives $x<5$. Therefore, the domain of the expression is $\\boxed{[2,5)}$.\nFinal Answer: The final answer is $[2,5)$. I hope it is correct.", + "few_shot": "1", + }, + { + "problem": "If $\\det \\mathbf{A} = 2$ and $\\det \\mathbf{B} = 12,$ then find $\\det (\\mathbf{A} \\mathbf{B}).$", + "solution": "We have that $\\det (\\mathbf{A} \\mathbf{B}) = (\\det \\mathbf{A})(\\det \\mathbf{B}) = (2)(12) = \\boxed{24}.$\nFinal Answer: The final answer is $24$. I hope it is correct.", + "few_shot": "1", + }, + { + "problem": "Terrell usually lifts two 20-pound weights 12 times. If he uses two 15-pound weights instead, how many times must Terrell lift them in order to lift the same total weight?", + "solution": "If Terrell lifts two 20-pound weights 12 times, he lifts a total of $2\\cdot 12\\cdot20=480$ pounds of weight. If he lifts two 15-pound weights instead for $n$ times, he will lift a total of $2\\cdot15\\cdot n=30n$ pounds of weight. Equating this to 480 pounds, we can solve for $n$:\n\\begin{align*}\n30n&=480\\\n\\Rightarrow\\qquad n&=480/30=\\boxed{16}\n\\end{align*}\nFinal Answer: The final answer is $16$. I hope it is correct.", + "few_shot": "1", + }, + { + "problem": "If the system of equations\n\n\\begin{align*}\n6x-4y&=a,\\\n6y-9x &=b.\n\\end{align*}has a solution $(x, y)$ where $x$ and $y$ are both nonzero,\nfind $\\frac{a}{b},$ assuming $b$ is nonzero.", + "solution": "If we multiply the first equation by $-\\frac{3}{2}$, we obtain\n\n$$6y-9x=-\\frac{3}{2}a.$$Since we also know that $6y-9x=b$, we have\n\n$$-\\frac{3}{2}a=b\\Rightarrow\\frac{a}{b}=\\boxed{-\\frac{2}{3}}.$$\nFinal Answer: The final answer is $-\\frac{2}{3}$. I hope it is correct.", + "few_shot": "1", + }, + ] + + +def process_results(doc: dict, results: List[str]) -> Dict[str, int]: + candidates = results[0] + + unnormalized_answer = get_unnormalized_answer(candidates) + answer = normalize_final_answer(unnormalized_answer) + + if is_equiv(answer, doc["answer"]): + retval = 1 + else: + retval = 0 + + # math_verify + res = verify(parse(doc["answer"]), parse(candidates)) + mathval = 1 if res else 0 + + results = { + "exact_match": retval, + "math_verify": mathval, + } + return results + + +def last_boxed_only_string(string: str) -> Optional[str]: + idx = string.rfind("\\boxed") + if "\\boxed " in string: + return "\\boxed " + string.split("\\boxed ")[-1].split("$")[0] + if idx < 0: + idx = string.rfind("\\fbox") + if idx < 0: + return None + + i = idx + right_brace_idx = None + num_left_braces_open = 0 + while i < len(string): + if string[i] == "{": + num_left_braces_open += 1 + if string[i] == "}": + num_left_braces_open -= 1 + if num_left_braces_open == 0: + right_brace_idx = i + break + i += 1 + + if right_brace_idx is None: + retval = None + else: + retval = string[idx : right_brace_idx + 1] + + return retval + + +def remove_boxed(s: str) -> str: + if "\\boxed " in s: + left = "\\boxed " + assert s[: len(left)] == left + return s[len(left) :] + + left = "\\boxed{" + + assert s[: len(left)] == left + assert s[-1] == "}" + + return s[len(left) : -1] + + +class timeout: + def __init__(self, seconds=1, error_message="Timeout"): + self.seconds = seconds + self.error_message = error_message + + def handle_timeout(self, signum, frame): + raise TimeoutError(self.error_message) + + def __enter__(self): + signal.signal(signal.SIGALRM, self.handle_timeout) + signal.alarm(self.seconds) + + def __exit__(self, type, value, traceback): + signal.alarm(0) + + +def is_equiv(x1: str, x2: str) -> bool: + """ + x1 and x2 are normalized latex string + """ + try: + with timeout(seconds=5): + try: + parsed_x1 = parse_latex(x1) + parsed_x2 = parse_latex(x2) + except ( + sympy.parsing.latex.errors.LaTeXParsingError, + sympy.SympifyError, + TypeError, + ): + eval_logger.debug(f"couldn't parse one of {x1} or {x2}") + return False + + try: + diff = parsed_x1 - parsed_x2 + except TypeError: + eval_logger.debug(f"couldn't subtract {x1} and {x2}") + return False + + try: + if sympy.simplify(diff) == 0: + return True + else: + return False + except ValueError: + eval_logger.debug( + f"Had some trouble simplifying when comparing {x1} and {x2}" + ) + except TimeoutError: + eval_logger.debug(f"Timed out comparing {x1} and {x2}") + return False + except ImportError as e: + eval_logger.error(e) + raise + except Exception as e: + eval_logger.debug(f"Failed comparing {x1} and {x2} with {e}") + return False + + +def get_unnormalized_answer(text: str) -> str: + INVALID_ANSWER = "[invalidanswer]" + end_seq = "I hope it is correct." + text += end_seq + match = re.search( + r"Final Answer: The final answer is(.*?). I hope it is correct.", + text, + ) + if match: + return match.group(1).strip() + else: + return INVALID_ANSWER + + +SUBSTITUTIONS = [ + ("an ", ""), + ("a ", ""), + (".$", "$"), + ("\\$", ""), + (r"\ ", ""), + (" ", ""), + ("mbox", "text"), + (",\\text{and}", ","), + ("\\text{and}", ","), + ("\\text{m}", "\\text{}"), +] +REMOVED_EXPRESSIONS = [ + "square", + "ways", + "integers", + "dollars", + "mph", + "inches", + "ft", + "hours", + "km", + "units", + "\\ldots", + "sue", + "points", + "feet", + "minutes", + "digits", + "cents", + "degrees", + "cm", + "gm", + "pounds", + "meters", + "meals", + "edges", + "students", + "childrentickets", + "multiples", + "\\text{s}", + "\\text{.}", + "\\text{\ns}", + "\\text{}^2", + "\\text{}^3", + "\\text{\n}", + "\\text{}", + r"\mathrm{th}", + r"^\circ", + r"^{\circ}", + r"\;", + r",\!", + "{,}", + '"', + "\\dots", +] + + +def normalize_final_answer(final_answer: str) -> str: + """ + Normalize a final answer to a quantitative reasoning question. + + Copied character for character from appendix D of Lewkowycz et al. (2022) + """ + final_answer = final_answer.split("=")[-1] + + for before, after in SUBSTITUTIONS: + final_answer = final_answer.replace(before, after) + for expr in REMOVED_EXPRESSIONS: + final_answer = final_answer.replace(expr, "") + + # Extract answer that is in LaTeX math, is bold, + # is surrounded by a box, etc. + final_answer = re.sub(r"(.*?)(\$)(.*?)(\$)(.*)", "$\\3$", final_answer) + final_answer = re.sub(r"(\\text\{)(.*?)(\})", "\\2", final_answer) + final_answer = re.sub(r"(\\textbf\{)(.*?)(\})", "\\2", final_answer) + final_answer = re.sub(r"(\\overline\{)(.*?)(\})", "\\2", final_answer) + final_answer = re.sub(r"(\\boxed\{)(.*)(\})", "\\2", final_answer) + + # Normalize shorthand TeX: + # \fracab -> \frac{a}{b} + # \frac{abc}{bef} -> \frac{abc}{bef} + # \fracabc -> \frac{a}{b}c + # \sqrta -> \sqrt{a} + # \sqrtab -> sqrt{a}b + final_answer = re.sub(r"(frac)([^{])(.)", "frac{\\2}{\\3}", final_answer) + final_answer = re.sub(r"(sqrt)([^{])", "sqrt{\\2}", final_answer) + final_answer = final_answer.replace("$", "") + + # Normalize 100,000 -> 100000 + if final_answer.replace(",", "").isdigit(): + final_answer = final_answer.replace(",", "") + + return final_answer diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/README.md b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/README.md new file mode 100644 index 0000000000000000000000000000000000000000..a3425d517654a6b93e03ee1bb681e07de18c4016 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/README.md @@ -0,0 +1,73 @@ +# Task-name + +### Paper + +Title: `Measuring Massive Multitask Language Understanding` + +Abstract: `https://arxiv.org/abs/2009.03300` + +`The test covers 57 tasks including elementary mathematics, US history, computer science, law, and more.` + +Homepage: `https://github.com/hendrycks/test` + +Note: The `Flan` variants are derived from [here](https://github.com/jasonwei20/flan-2), and as described in Appendix D.1 of [Scaling Instruction-Finetuned Language Models](https://arxiv.org/abs/2210.11416). + +### Citation + +``` +@article{hendryckstest2021, + title={Measuring Massive Multitask Language Understanding}, + author={Dan Hendrycks and Collin Burns and Steven Basart and Andy Zou and Mantas Mazeika and Dawn Song and Jacob Steinhardt}, + journal={Proceedings of the International Conference on Learning Representations (ICLR)}, + year={2021} +} + +@article{hendrycks2021ethics, + title={Aligning AI With Shared Human Values}, + author={Dan Hendrycks and Collin Burns and Steven Basart and Andrew Critch and Jerry Li and Dawn Song and Jacob Steinhardt}, + journal={Proceedings of the International Conference on Learning Representations (ICLR)}, + year={2021} +} +``` + +### Groups, Tags, and Tasks + +#### Groups + +* `mmlu`: `Original multiple-choice MMLU benchmark` +* `mmlu_continuation`: `MMLU but with continuation prompts` +* `mmlu_generation`: `MMLU generation` + +MMLU is the original benchmark as implemented by Hendrycks et al. with the choices in context and the answer letters (e.g `A`, `B`, `C`, `D`) in the continuation. +`mmlu_continuation` is a cloze-style variant without the choices in context and the full answer choice in the continuation. +`mmlu_generation` is a generation variant, similar to the original but the LLM is asked to generate the correct answer letter. + + +#### Subgroups + +* `mmlu_stem' +* `mmlu_humanities' +* `mmlu_social_sciences' +* `mmlu_other' + +Subgroup variants are prefixed with the subgroup name, e.g. `mmlu_stem_continuation`. + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [x] Is the "Main" variant of this task clearly denoted? +* [x] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [x] Have you noted which, if any, published evaluation setups are matched by this variant? + +# changelog +ver 1: PR #497 +switch to original implementation + +ver 2: PR #2116 +add missing newline in description. diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/_generate_configs.py b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/_generate_configs.py new file mode 100644 index 0000000000000000000000000000000000000000..88a7a2c2e63a5066b7f60a0bee8e8839173969e4 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/_generate_configs.py @@ -0,0 +1,159 @@ +# noqa +""" +Take in a YAML, and output all "other" splits with this YAML +""" + +import argparse +import logging +import os + +import yaml +from tqdm import tqdm + + +eval_logger = logging.getLogger(__name__) + + +SUBJECTS = { + "abstract_algebra": "stem", + "anatomy": "stem", + "astronomy": "stem", + "business_ethics": "other", + "clinical_knowledge": "other", + "college_biology": "stem", + "college_chemistry": "stem", + "college_computer_science": "stem", + "college_mathematics": "stem", + "college_medicine": "other", + "college_physics": "stem", + "computer_security": "stem", + "conceptual_physics": "stem", + "econometrics": "social_sciences", + "electrical_engineering": "stem", + "elementary_mathematics": "stem", + "formal_logic": "humanities", + "global_facts": "other", + "high_school_biology": "stem", + "high_school_chemistry": "stem", + "high_school_computer_science": "stem", + "high_school_european_history": "humanities", + "high_school_geography": "social_sciences", + "high_school_government_and_politics": "social_sciences", + "high_school_macroeconomics": "social_sciences", + "high_school_mathematics": "stem", + "high_school_microeconomics": "social_sciences", + "high_school_physics": "stem", + "high_school_psychology": "social_sciences", + "high_school_statistics": "stem", + "high_school_us_history": "humanities", + "high_school_world_history": "humanities", + "human_aging": "other", + "human_sexuality": "social_sciences", + "international_law": "humanities", + "jurisprudence": "humanities", + "logical_fallacies": "humanities", + "machine_learning": "stem", + "management": "other", + "marketing": "other", + "medical_genetics": "other", + "miscellaneous": "other", + "moral_disputes": "humanities", + "moral_scenarios": "humanities", + "nutrition": "other", + "philosophy": "humanities", + "prehistory": "humanities", + "professional_accounting": "other", + "professional_law": "humanities", + "professional_medicine": "other", + "professional_psychology": "social_sciences", + "public_relations": "social_sciences", + "security_studies": "social_sciences", + "sociology": "social_sciences", + "us_foreign_policy": "social_sciences", + "virology": "other", + "world_religions": "humanities", +} + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--base_yaml_path", required=True) + parser.add_argument("--save_prefix_path", default="mmlu") + parser.add_argument("--cot_prompt_path", default=None) + parser.add_argument("--task_prefix", default="") + parser.add_argument("--group_prefix", default="") + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + + # get filename of base_yaml so we can `"include": ` it in our "other" YAMLs. + base_yaml_name = os.path.split(args.base_yaml_path)[-1] + with open(args.base_yaml_path, encoding="utf-8") as f: + base_yaml = yaml.full_load(f) + + if args.cot_prompt_path is not None: + import json + + with open(args.cot_prompt_path, encoding="utf-8") as f: + cot_file = json.load(f) + + ALL_CATEGORIES = [] + for subject, category in tqdm(SUBJECTS.items()): + if category not in ALL_CATEGORIES: + ALL_CATEGORIES.append(category) + + if args.cot_prompt_path is not None: + description = cot_file[subject] + else: + description = f"The following are multiple choice questions (with answers) about {' '.join(subject.split('_'))}.\n\n" + + yaml_dict = { + "include": base_yaml_name, + "tag": f"mmlu_{args.task_prefix}_{category}" + if args.task_prefix != "" + else f"mmlu_{category}", + "task": f"mmlu_{args.task_prefix}_{subject}" + if args.task_prefix != "" + else f"mmlu_{subject}", + "task_alias": subject.replace("_", " "), + "dataset_name": subject, + "description": description, + } + + file_save_path = args.save_prefix_path + f"_{subject}.yaml" + eval_logger.info(f"Saving yaml for subset {subject} to {file_save_path}") + with open(file_save_path, "w", encoding="utf-8") as yaml_file: + yaml.dump( + yaml_dict, + yaml_file, + allow_unicode=True, + default_style='"', + ) + + if args.task_prefix != "": + mmlu_subcategories = [ + f"mmlu_{args.task_prefix}_{category}" for category in ALL_CATEGORIES + ] + else: + mmlu_subcategories = [f"mmlu_{category}" for category in ALL_CATEGORIES] + + if args.group_prefix != "": + file_save_path = args.group_prefix + ".yaml" + else: + file_save_path = args.save_prefix_path + ".yaml" + + eval_logger.info(f"Saving benchmark config to {file_save_path}") + with open(file_save_path, "w", encoding="utf-8") as yaml_file: + yaml.dump( + { + "group": f"mmlu_{args.task_prefix}" + if args.task_prefix != "" + else "mmlu", + "task": mmlu_subcategories, + }, + yaml_file, + indent=4, + default_flow_style=False, + ) diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/_continuation_template_yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/_continuation_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..264e27a5e8ebde9a203094c7cc9735ecf8ef3993 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/_continuation_template_yaml @@ -0,0 +1,13 @@ +dataset_path: hails/mmlu_no_train # a copy of `cais/mmlu` with no auxiliary_train split +output_type: multiple_choice +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +doc_to_text: "Question: {{question.strip()}}\nAnswer:" +doc_to_choice: "{{choices}}" +doc_to_target: "{{answer}}" +metadata: + version: 1.0 +dataset_kwargs: + trust_remote_code: true diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/_mmlu.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/_mmlu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c0cabf04b8ac1e1f9c809600214c589cfefbba79 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/_mmlu.yaml @@ -0,0 +1,32 @@ +group: mmlu_continuation +group_alias: mmlu (continuation) +task: + - group: stem + task: + - mmlu_continuation_stem + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: other + task: + - mmlu_continuation_other + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: social sciences + task: + - mmlu_continuation_social_sciences + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: humanities + task: + - mmlu_continuation_humanities + aggregate_metric_list: + - metric: acc + weight_by_size: True +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 2 diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_abstract_algebra.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f4e29c0fb5147d883ee993d95822dde10b69d4e --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_abstract_algebra.yaml @@ -0,0 +1,6 @@ +"dataset_name": "abstract_algebra" +"description": "The following are questions (with answers) about abstract\ + \ algebra.\n\n" +"tag": "mmlu_continuation_stem" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_abstract_algebra" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_anatomy.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc3de9c4e6679ba4c9f66494c908d99781adf5bb --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_anatomy.yaml @@ -0,0 +1,6 @@ +"dataset_name": "anatomy" +"description": "The following are questions (with answers) about anatomy.\n\ + \n" +"tag": "mmlu_continuation_stem" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_anatomy" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_astronomy.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..76aabcbfcf13a12e66e1af1daae2811b9b388fc8 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_astronomy.yaml @@ -0,0 +1,6 @@ +"dataset_name": "astronomy" +"description": "The following are questions (with answers) about astronomy.\n\ + \n" +"tag": "mmlu_continuation_stem" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_astronomy" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_business_ethics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e64d0920b9d1ac151712aac84a9e9c3f522c3c9f --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_business_ethics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "business_ethics" +"description": "The following are questions (with answers) about business\ + \ ethics.\n\n" +"tag": "mmlu_continuation_other" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_business_ethics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_clinical_knowledge.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e79805df6f73782f25be4a302c738b73ecd2f2a2 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_clinical_knowledge.yaml @@ -0,0 +1,6 @@ +"dataset_name": "clinical_knowledge" +"description": "The following are questions (with answers) about clinical\ + \ knowledge.\n\n" +"tag": "mmlu_continuation_other" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_clinical_knowledge" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_college_biology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..936f6ffe49245d558c0ef8fdf04b600dc177c375 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_college_biology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_biology" +"description": "The following are questions (with answers) about college\ + \ biology.\n\n" +"tag": "mmlu_continuation_stem" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_college_biology" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_college_chemistry.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..289364ee44351c3d1bcee1193563babe6abe2a63 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_college_chemistry.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_chemistry" +"description": "The following are questions (with answers) about college\ + \ chemistry.\n\n" +"tag": "mmlu_continuation_stem" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_college_chemistry" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_college_computer_science.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c7d3c5696067f09f9a68fdd9c3f7a1002d264128 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_college_computer_science.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_computer_science" +"description": "The following are questions (with answers) about college\ + \ computer science.\n\n" +"tag": "mmlu_continuation_stem" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_college_computer_science" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_college_mathematics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2dbc0932f63c0782e106db5fc27e96da9d816dec --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_college_mathematics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_mathematics" +"description": "The following are questions (with answers) about college\ + \ mathematics.\n\n" +"tag": "mmlu_continuation_stem" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_college_mathematics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_college_medicine.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..38abd2426f844916087795c4cc04355d8d6c2776 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_college_medicine.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_medicine" +"description": "The following are questions (with answers) about college\ + \ medicine.\n\n" +"tag": "mmlu_continuation_other" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_college_medicine" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_college_physics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ee6b42584c834a5e92506650ee3aba58ed1cfd66 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_college_physics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_physics" +"description": "The following are questions (with answers) about college\ + \ physics.\n\n" +"tag": "mmlu_continuation_stem" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_college_physics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_computer_security.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ebb487dfbf634d390d2b2f9aa0e31e5a2f68fc6 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_computer_security.yaml @@ -0,0 +1,6 @@ +"dataset_name": "computer_security" +"description": "The following are questions (with answers) about computer\ + \ security.\n\n" +"tag": "mmlu_continuation_stem" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_computer_security" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_conceptual_physics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c554caf07da77e4a9bb0bea9672dfcee4777b91 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_conceptual_physics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "conceptual_physics" +"description": "The following are questions (with answers) about conceptual\ + \ physics.\n\n" +"tag": "mmlu_continuation_stem" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_conceptual_physics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_econometrics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..848ce4e1f0dbff32d304c28f3d60d453e591a30f --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_econometrics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "econometrics" +"description": "The following are questions (with answers) about econometrics.\n\ + \n" +"tag": "mmlu_continuation_social_sciences" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_econometrics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_electrical_engineering.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d71dd16481a2bb5289ef5b713218dae0292bb11a --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_electrical_engineering.yaml @@ -0,0 +1,6 @@ +"dataset_name": "electrical_engineering" +"description": "The following are questions (with answers) about electrical\ + \ engineering.\n\n" +"tag": "mmlu_continuation_stem" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_electrical_engineering" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_elementary_mathematics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe8aa09718cb8aef0dad48c21926f7dacc7b8ee9 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_elementary_mathematics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "elementary_mathematics" +"description": "The following are questions (with answers) about elementary\ + \ mathematics.\n\n" +"tag": "mmlu_continuation_stem" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_elementary_mathematics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_formal_logic.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb5dbd2e505e3fb4604dd75f2d5fe1a35fce3391 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_formal_logic.yaml @@ -0,0 +1,6 @@ +"dataset_name": "formal_logic" +"description": "The following are questions (with answers) about formal\ + \ logic.\n\n" +"tag": "mmlu_continuation_humanities" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_formal_logic" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_global_facts.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..280a50d2ee229b5f047a02024298474225203e54 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_global_facts.yaml @@ -0,0 +1,6 @@ +"dataset_name": "global_facts" +"description": "The following are questions (with answers) about global\ + \ facts.\n\n" +"tag": "mmlu_continuation_other" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_global_facts" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_biology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e518a5239a6da013ad31bfca284a3b7096bce840 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_biology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_biology" +"description": "The following are questions (with answers) about high\ + \ school biology.\n\n" +"tag": "mmlu_continuation_stem" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_high_school_biology" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_chemistry.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c38d60a7706306b215e156d4c27f05585945f7b4 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_chemistry.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_chemistry" +"description": "The following are questions (with answers) about high\ + \ school chemistry.\n\n" +"tag": "mmlu_continuation_stem" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_high_school_chemistry" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_computer_science.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5fe34f7af35456657c1acf40e05b3aaabc7893e8 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_computer_science.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_computer_science" +"description": "The following are questions (with answers) about high\ + \ school computer science.\n\n" +"tag": "mmlu_continuation_stem" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_high_school_computer_science" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_european_history.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..666c2742d1b762c103bbd02ff121676a047fb3e5 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_european_history.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_european_history" +"description": "The following are questions (with answers) about high\ + \ school european history.\n\n" +"tag": "mmlu_continuation_humanities" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_high_school_european_history" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_geography.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..41f6caf3e7f3b762af7c0350ca9a73d39bede2b8 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_geography.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_geography" +"description": "The following are questions (with answers) about high\ + \ school geography.\n\n" +"tag": "mmlu_continuation_social_sciences" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_high_school_geography" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_government_and_politics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e80233dc891e6890a5dec384ed2fbe5b82aca094 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_government_and_politics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_government_and_politics" +"description": "The following are questions (with answers) about high\ + \ school government and politics.\n\n" +"tag": "mmlu_continuation_social_sciences" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_high_school_government_and_politics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_macroeconomics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ce7fa9d5e3caa8dd3ec8e25172afda5f997b6c0c --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_macroeconomics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_macroeconomics" +"description": "The following are questions (with answers) about high\ + \ school macroeconomics.\n\n" +"tag": "mmlu_continuation_social_sciences" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_high_school_macroeconomics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_mathematics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2598dcb38eb9f8fdacced20c57d62c83dacb8a40 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_mathematics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_mathematics" +"description": "The following are questions (with answers) about high\ + \ school mathematics.\n\n" +"tag": "mmlu_continuation_stem" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_high_school_mathematics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_microeconomics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..96c414d3c411c6380cf83dca3b7aedc325598220 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_microeconomics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_microeconomics" +"description": "The following are questions (with answers) about high\ + \ school microeconomics.\n\n" +"tag": "mmlu_continuation_social_sciences" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_high_school_microeconomics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_physics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..45ab0a539a02ae322f66db689d8eddf13c8b856a --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_physics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_physics" +"description": "The following are questions (with answers) about high\ + \ school physics.\n\n" +"tag": "mmlu_continuation_stem" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_high_school_physics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_psychology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..48dedf5c5ed94a836e0d802398ab05d7ab7db6ce --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_psychology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_psychology" +"description": "The following are questions (with answers) about high\ + \ school psychology.\n\n" +"tag": "mmlu_continuation_social_sciences" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_high_school_psychology" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_statistics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ee2418c7ff5235c1e31cf381502f5b21db60230 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_statistics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_statistics" +"description": "The following are questions (with answers) about high\ + \ school statistics.\n\n" +"tag": "mmlu_continuation_stem" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_high_school_statistics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_us_history.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a00f16ceba2cfd3f313c8fe0d2df4a43e4bbe23d --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_us_history.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_us_history" +"description": "The following are questions (with answers) about high\ + \ school us history.\n\n" +"tag": "mmlu_continuation_humanities" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_high_school_us_history" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_world_history.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dc4cddf553bf0144b5d4ecc5eabe8efef0cf0367 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_high_school_world_history.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_world_history" +"description": "The following are questions (with answers) about high\ + \ school world history.\n\n" +"tag": "mmlu_continuation_humanities" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_high_school_world_history" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_human_aging.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..314edeb6c26c6a6be2d819b7c66e047cd48f8933 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_human_aging.yaml @@ -0,0 +1,6 @@ +"dataset_name": "human_aging" +"description": "The following are questions (with answers) about human\ + \ aging.\n\n" +"tag": "mmlu_continuation_other" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_human_aging" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_human_sexuality.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a1473819ab4307f1e02024a0828ad9803710a59b --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_human_sexuality.yaml @@ -0,0 +1,6 @@ +"dataset_name": "human_sexuality" +"description": "The following are questions (with answers) about human\ + \ sexuality.\n\n" +"tag": "mmlu_continuation_social_sciences" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_human_sexuality" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_international_law.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5ea8944bcc109000525b90f26f1d0da914d17437 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_international_law.yaml @@ -0,0 +1,6 @@ +"dataset_name": "international_law" +"description": "The following are questions (with answers) about international\ + \ law.\n\n" +"tag": "mmlu_continuation_humanities" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_international_law" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_jurisprudence.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fca1dda86cc382604ca1bcbc308e0062e08dfa80 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_jurisprudence.yaml @@ -0,0 +1,6 @@ +"dataset_name": "jurisprudence" +"description": "The following are questions (with answers) about jurisprudence.\n\ + \n" +"tag": "mmlu_continuation_humanities" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_jurisprudence" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_logical_fallacies.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1b576f9fb3d0ce1d21e8d7543b56a539300be36a --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_logical_fallacies.yaml @@ -0,0 +1,6 @@ +"dataset_name": "logical_fallacies" +"description": "The following are questions (with answers) about logical\ + \ fallacies.\n\n" +"tag": "mmlu_continuation_humanities" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_logical_fallacies" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_machine_learning.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..15fc3f4bdf0f34e96149ca2f8dddc90d037e8483 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_machine_learning.yaml @@ -0,0 +1,6 @@ +"dataset_name": "machine_learning" +"description": "The following are questions (with answers) about machine\ + \ learning.\n\n" +"tag": "mmlu_continuation_stem" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_machine_learning" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_management.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..575604e0acf52132d9e489a070d28fd761e739eb --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_management.yaml @@ -0,0 +1,6 @@ +"dataset_name": "management" +"description": "The following are questions (with answers) about management.\n\ + \n" +"tag": "mmlu_continuation_other" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_management" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_marketing.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..af715bee02cfe813b5f045670c8e46dda258e77d --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_marketing.yaml @@ -0,0 +1,6 @@ +"dataset_name": "marketing" +"description": "The following are questions (with answers) about marketing.\n\ + \n" +"tag": "mmlu_continuation_other" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_marketing" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_medical_genetics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3bf63614168f648497d046f015472497a2ac7553 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_medical_genetics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "medical_genetics" +"description": "The following are questions (with answers) about medical\ + \ genetics.\n\n" +"tag": "mmlu_continuation_other" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_medical_genetics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_miscellaneous.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f457800932ec2fba831a1d81e6ca4495816f981f --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_miscellaneous.yaml @@ -0,0 +1,6 @@ +"dataset_name": "miscellaneous" +"description": "The following are questions (with answers) about miscellaneous.\n\ + \n" +"tag": "mmlu_continuation_other" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_miscellaneous" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_moral_disputes.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0df1392d5baceb1a3dda1464acbb0b025a8428e8 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_moral_disputes.yaml @@ -0,0 +1,6 @@ +"dataset_name": "moral_disputes" +"description": "The following are questions (with answers) about moral\ + \ disputes.\n\n" +"tag": "mmlu_continuation_humanities" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_moral_disputes" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_moral_scenarios.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bea5e514b85a6ed83026a6fe9d399f92eb59ea99 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_moral_scenarios.yaml @@ -0,0 +1,6 @@ +"dataset_name": "moral_scenarios" +"description": "The following are questions (with answers) about moral\ + \ scenarios.\n\n" +"tag": "mmlu_continuation_humanities" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_moral_scenarios" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_nutrition.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8db80340b2a9984cb8c3e41766e3f0e89af8f252 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_nutrition.yaml @@ -0,0 +1,6 @@ +"dataset_name": "nutrition" +"description": "The following are questions (with answers) about nutrition.\n\ + \n" +"tag": "mmlu_continuation_other" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_nutrition" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_philosophy.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..165de6c90ba1d4756c39e2f5605226dbeb86e314 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_philosophy.yaml @@ -0,0 +1,6 @@ +"dataset_name": "philosophy" +"description": "The following are questions (with answers) about philosophy.\n\ + \n" +"tag": "mmlu_continuation_humanities" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_philosophy" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_prehistory.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..02c4ee7f8af1856f498b7a55c83e085782e36666 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_prehistory.yaml @@ -0,0 +1,6 @@ +"dataset_name": "prehistory" +"description": "The following are questions (with answers) about prehistory.\n\ + \n" +"tag": "mmlu_continuation_humanities" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_prehistory" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_professional_accounting.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bb36a82b9c043b519379626f2d3618efdda9907b --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_professional_accounting.yaml @@ -0,0 +1,6 @@ +"dataset_name": "professional_accounting" +"description": "The following are questions (with answers) about professional\ + \ accounting.\n\n" +"tag": "mmlu_continuation_other" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_professional_accounting" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_professional_law.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac9f2592f41a2bcae43da174d2eb969cf1805251 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_professional_law.yaml @@ -0,0 +1,6 @@ +"dataset_name": "professional_law" +"description": "The following are questions (with answers) about professional\ + \ law.\n\n" +"tag": "mmlu_continuation_humanities" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_professional_law" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_professional_medicine.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..328c128377609327abe0460e2d4ab6af716d02c3 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_professional_medicine.yaml @@ -0,0 +1,6 @@ +"dataset_name": "professional_medicine" +"description": "The following are questions (with answers) about professional\ + \ medicine.\n\n" +"tag": "mmlu_continuation_other" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_professional_medicine" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_professional_psychology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0cca5bde048a23367aa2ccebc893e9fa71996d98 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_professional_psychology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "professional_psychology" +"description": "The following are questions (with answers) about professional\ + \ psychology.\n\n" +"tag": "mmlu_continuation_social_sciences" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_professional_psychology" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_public_relations.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..700c407c2377d8d4d83bbf88d8f7a003a2e2900d --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_public_relations.yaml @@ -0,0 +1,6 @@ +"dataset_name": "public_relations" +"description": "The following are questions (with answers) about public\ + \ relations.\n\n" +"tag": "mmlu_continuation_social_sciences" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_public_relations" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_security_studies.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f5ef99e0f8fe8c98bc9994757d9cc6617e3550e --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_security_studies.yaml @@ -0,0 +1,6 @@ +"dataset_name": "security_studies" +"description": "The following are questions (with answers) about security\ + \ studies.\n\n" +"tag": "mmlu_continuation_social_sciences" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_security_studies" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_sociology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e78621aaa547b419f4133b94ce8dcba00c407f5c --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_sociology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "sociology" +"description": "The following are questions (with answers) about sociology.\n\ + \n" +"tag": "mmlu_continuation_social_sciences" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_sociology" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_us_foreign_policy.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..989bb29aa095e83c2744011775864ef27258ca28 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_us_foreign_policy.yaml @@ -0,0 +1,6 @@ +"dataset_name": "us_foreign_policy" +"description": "The following are questions (with answers) about us\ + \ foreign policy.\n\n" +"tag": "mmlu_continuation_social_sciences" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_us_foreign_policy" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_virology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5c938190bdd755f411914905d5309daa6938f313 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_virology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "virology" +"description": "The following are questions (with answers) about virology.\n\ + \n" +"tag": "mmlu_continuation_other" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_virology" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_world_religions.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f707670066d3f2db4554221a12a3983e2d8febf5 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/continuation/mmlu_world_religions.yaml @@ -0,0 +1,6 @@ +"dataset_name": "world_religions" +"description": "The following are questions (with answers) about world\ + \ religions.\n\n" +"tag": "mmlu_continuation_humanities" +"include": "_continuation_template_yaml" +"task": "mmlu_continuation_world_religions" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/_default_template_yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/_default_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..ed0e70536b94b9d2127c2e02999d34cd6d0c3943 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/_default_template_yaml @@ -0,0 +1,17 @@ +dataset_path: hails/mmlu_no_train # a copy of `cais/mmlu` with no auxiliary_train split +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 +dataset_kwargs: + trust_remote_code: true diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/_mmlu.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/_mmlu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..55099c6f16febd89270ad022abe181bf8ccd708e --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/_mmlu.yaml @@ -0,0 +1,11 @@ +group: mmlu +task: + - mmlu_stem + - mmlu_other + - mmlu_social_sciences + - mmlu_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 2 diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/_mmlu_humanities.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/_mmlu_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7156e2230f09b461b8e783db323b9ee2d8023192 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/_mmlu_humanities.yaml @@ -0,0 +1,9 @@ +group: mmlu_humanities +group_alias: humanities +task: + - mmlu_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 2 diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/_mmlu_other.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/_mmlu_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..79025cec0c639a37872287ecb5ae5c444dce7478 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/_mmlu_other.yaml @@ -0,0 +1,9 @@ +group: mmlu_other +group_alias: other +task: + - mmlu_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 2 diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/_mmlu_social_sciences.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/_mmlu_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fab1ec2c1416bc644c8723bdb18905dff9c00040 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/_mmlu_social_sciences.yaml @@ -0,0 +1,9 @@ +group: mmlu_social_sciences +group_alias: social sciences +task: + - mmlu_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 2 diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/_mmlu_stem.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/_mmlu_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cda82eff10a03afe1a05fd8a1368cf3a7c63dcd8 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/_mmlu_stem.yaml @@ -0,0 +1,9 @@ +group: mmlu_stem +group_alias: stem +task: + - mmlu_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 2 diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_abstract_algebra.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dcde12cb4c5566567482e095c87860f1c6179473 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_abstract_algebra.yaml @@ -0,0 +1,7 @@ +"dataset_name": "abstract_algebra" +"description": "The following are multiple choice questions (with answers) about abstract\ + \ algebra.\n\n" +"tag": "mmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "mmlu_abstract_algebra" +"task_alias": "abstract_algebra" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_anatomy.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5fef7490dd31872f2ed9dcde5c1e817e910b5e39 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_anatomy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "anatomy" +"description": "The following are multiple choice questions (with answers) about anatomy.\n\ + \n" +"tag": "mmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "mmlu_anatomy" +"task_alias": "anatomy" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_astronomy.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..660f07476dfdd115fc0b8d5f04c685b23857cc33 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_astronomy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "astronomy" +"description": "The following are multiple choice questions (with answers) about astronomy.\n\ + \n" +"tag": "mmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "mmlu_astronomy" +"task_alias": "astronomy" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_business_ethics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0f1b1c2dcd802effdf589d4f85b412593dfb622 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_business_ethics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "business_ethics" +"description": "The following are multiple choice questions (with answers) about business\ + \ ethics.\n\n" +"tag": "mmlu_other_tasks" +"include": "_default_template_yaml" +"task": "mmlu_business_ethics" +"task_alias": "business_ethics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_clinical_knowledge.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1c72b71648df5a690963c95180a76f7ad0a495d4 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_clinical_knowledge.yaml @@ -0,0 +1,7 @@ +"dataset_name": "clinical_knowledge" +"description": "The following are multiple choice questions (with answers) about clinical\ + \ knowledge.\n\n" +"tag": "mmlu_other_tasks" +"include": "_default_template_yaml" +"task": "mmlu_clinical_knowledge" +"task_alias": "clinical_knowledge" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_college_biology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ddfd713aa0581b36fdad44da4f80e5b500c47154 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_college_biology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "college_biology" +"description": "The following are multiple choice questions (with answers) about college\ + \ biology.\n\n" +"tag": "mmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "mmlu_college_biology" +"task_alias": "college_biology" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_college_chemistry.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..388c3a91bed8ffb7645e0e7f23fb0a81117503cc --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_college_chemistry.yaml @@ -0,0 +1,7 @@ +"dataset_name": "college_chemistry" +"description": "The following are multiple choice questions (with answers) about college\ + \ chemistry.\n\n" +"tag": "mmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "mmlu_college_chemistry" +"task_alias": "college_chemistry" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_college_computer_science.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a3f692423abfbf036fc0347fdfbb2642a6d16c39 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_college_computer_science.yaml @@ -0,0 +1,7 @@ +"dataset_name": "college_computer_science" +"description": "The following are multiple choice questions (with answers) about college\ + \ computer science.\n\n" +"tag": "mmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "mmlu_college_computer_science" +"task_alias": "college_computer_science" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_college_mathematics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08a9628af175edb897c7f6d88b96d4969fccad29 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_college_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "college_mathematics" +"description": "The following are multiple choice questions (with answers) about college\ + \ mathematics.\n\n" +"tag": "mmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "mmlu_college_mathematics" +"task_alias": "college_mathematics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_college_medicine.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..35197a2a1885f7daf30209d4309dd059243260a8 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_college_medicine.yaml @@ -0,0 +1,7 @@ +"dataset_name": "college_medicine" +"description": "The following are multiple choice questions (with answers) about college\ + \ medicine.\n\n" +"tag": "mmlu_other_tasks" +"include": "_default_template_yaml" +"task": "mmlu_college_medicine" +"task_alias": "college_medicine" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_college_physics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b5017afac65e0acf080a9df84098a1f21681833 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_college_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics.\n\n" +"tag": "mmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "mmlu_college_physics" +"task_alias": "college_physics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_computer_security.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f9b42820f7f7196c6d02922337eaedb7ede5388 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_computer_security.yaml @@ -0,0 +1,7 @@ +"dataset_name": "computer_security" +"description": "The following are multiple choice questions (with answers) about computer\ + \ security.\n\n" +"tag": "mmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "mmlu_computer_security" +"task_alias": "computer_security" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_conceptual_physics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..af61a7e1579ac8613b5535e15a57adc629e2d571 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_conceptual_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "conceptual_physics" +"description": "The following are multiple choice questions (with answers) about conceptual\ + \ physics.\n\n" +"tag": "mmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "mmlu_conceptual_physics" +"task_alias": "conceptual_physics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_econometrics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..609c20af2acbdd7ef36104dc97db97a40bfca6a5 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_econometrics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "econometrics" +"description": "The following are multiple choice questions (with answers) about econometrics.\n\ + \n" +"tag": "mmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "mmlu_econometrics" +"task_alias": "econometrics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_electrical_engineering.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8fa2137ad05a14e32d5d7e8973d6bc9c18d1a555 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_electrical_engineering.yaml @@ -0,0 +1,7 @@ +"dataset_name": "electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering.\n\n" +"tag": "mmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "mmlu_electrical_engineering" +"task_alias": "electrical_engineering" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_elementary_mathematics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d15f6d5ae88b6edf0bba2298ffaacbd4d103aedd --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_elementary_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "elementary_mathematics" +"description": "The following are multiple choice questions (with answers) about elementary\ + \ mathematics.\n\n" +"tag": "mmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "mmlu_elementary_mathematics" +"task_alias": "elementary_mathematics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_formal_logic.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ee2fc2f61073dc11f6f745eaf8927ab70aadad3f --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_formal_logic.yaml @@ -0,0 +1,7 @@ +"dataset_name": "formal_logic" +"description": "The following are multiple choice questions (with answers) about formal\ + \ logic.\n\n" +"tag": "mmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "mmlu_formal_logic" +"task_alias": "formal_logic" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_global_facts.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b27ddefd25be9c6695900ce6d290a811b68356df --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_global_facts.yaml @@ -0,0 +1,7 @@ +"dataset_name": "global_facts" +"description": "The following are multiple choice questions (with answers) about global\ + \ facts.\n\n" +"tag": "mmlu_other_tasks" +"include": "_default_template_yaml" +"task": "mmlu_global_facts" +"task_alias": "global_facts" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_biology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..22bc47943f0f66614f79cd0de5e7614afa1f08d5 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_biology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_biology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school biology.\n\n" +"tag": "mmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "mmlu_high_school_biology" +"task_alias": "high_school_biology" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_chemistry.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5a25617cbd821411e6f0ca9fac853c76b7adb319 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_chemistry.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry.\n\n" +"tag": "mmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "mmlu_high_school_chemistry" +"task_alias": "high_school_chemistry" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_computer_science.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ad4c7d312c7e8f6517d308e6ffeb635a354b843e --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_computer_science.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_computer_science" +"description": "The following are multiple choice questions (with answers) about high\ + \ school computer science.\n\n" +"tag": "mmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "mmlu_high_school_computer_science" +"task_alias": "high_school_computer_science" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_european_history.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c51bbdd7aa87b39da8145f8ea45f6fe13d17623 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_european_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_european_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school european history.\n\n" +"tag": "mmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "mmlu_high_school_european_history" +"task_alias": "high_school_european_history" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_geography.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aad87f1ad57a48102d7807a7a3fd75af86755912 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_geography.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_geography" +"description": "The following are multiple choice questions (with answers) about high\ + \ school geography.\n\n" +"tag": "mmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "mmlu_high_school_geography" +"task_alias": "high_school_geography" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_government_and_politics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2b93b363d658357619eaf907f8d04af339c22a12 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_government_and_politics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_government_and_politics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school government and politics.\n\n" +"tag": "mmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "mmlu_high_school_government_and_politics" +"task_alias": "high_school_government_and_politics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_macroeconomics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a08c579d1480ab592917d5a6673e63cf09198417 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_macroeconomics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_macroeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school macroeconomics.\n\n" +"tag": "mmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "mmlu_high_school_macroeconomics" +"task_alias": "high_school_macroeconomics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_mathematics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3f1b6d70e022414b7d370635daa49e3a9a8649c2 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_mathematics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school mathematics.\n\n" +"tag": "mmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "mmlu_high_school_mathematics" +"task_alias": "high_school_mathematics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_microeconomics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac4f65dad5783bf23c50d9a39e912fe797a047e6 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_microeconomics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_microeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school microeconomics.\n\n" +"tag": "mmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "mmlu_high_school_microeconomics" +"task_alias": "high_school_microeconomics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_physics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b8c449aa1b5bb48d6899c328c82c44ee3ae3ef24 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_physics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school physics.\n\n" +"tag": "mmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "mmlu_high_school_physics" +"task_alias": "high_school_physics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_psychology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..47ba836c71b2be9759bd9fe48dd0cb687ef08636 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_psychology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_psychology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school psychology.\n\n" +"tag": "mmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "mmlu_high_school_psychology" +"task_alias": "high_school_psychology" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_statistics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ef5bdd7cf1577a7ba9f3365643c5e56b21c8a77e --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_statistics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics.\n\n" +"tag": "mmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "mmlu_high_school_statistics" +"task_alias": "high_school_statistics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_us_history.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ececdb0ab921bdc24b8aac41979a93d35670d0c6 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_us_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history.\n\n" +"tag": "mmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "mmlu_high_school_us_history" +"task_alias": "high_school_us_history" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_world_history.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..af34c8ddbe51abc0f44baff2bf8087b4c749825f --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_high_school_world_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_world_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school world history.\n\n" +"tag": "mmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "mmlu_high_school_world_history" +"task_alias": "high_school_world_history" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_human_aging.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3ca720be7c7d757c579e4563cb805dc36a6dcc6d --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_human_aging.yaml @@ -0,0 +1,7 @@ +"dataset_name": "human_aging" +"description": "The following are multiple choice questions (with answers) about human\ + \ aging.\n\n" +"tag": "mmlu_other_tasks" +"include": "_default_template_yaml" +"task": "mmlu_human_aging" +"task_alias": "human_aging" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_human_sexuality.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2acddd1e4ec1d85a7475202d43f5917abb085684 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_human_sexuality.yaml @@ -0,0 +1,7 @@ +"dataset_name": "human_sexuality" +"description": "The following are multiple choice questions (with answers) about human\ + \ sexuality.\n\n" +"tag": "mmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "mmlu_human_sexuality" +"task_alias": "human_sexuality" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_international_law.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9fb2a162aab92931f8b560ce0e76155fbc9bb675 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_international_law.yaml @@ -0,0 +1,7 @@ +"dataset_name": "international_law" +"description": "The following are multiple choice questions (with answers) about international\ + \ law.\n\n" +"tag": "mmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "mmlu_international_law" +"task_alias": "international_law" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_jurisprudence.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c62a911ff5d849651d8c9e09feb34847846d147 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_jurisprudence.yaml @@ -0,0 +1,7 @@ +"dataset_name": "jurisprudence" +"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ + \n" +"tag": "mmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "mmlu_jurisprudence" +"task_alias": "jurisprudence" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_logical_fallacies.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..adf8821e9a8ac9d80f1cfb5c6af5b74a63efda27 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_logical_fallacies.yaml @@ -0,0 +1,7 @@ +"dataset_name": "logical_fallacies" +"description": "The following are multiple choice questions (with answers) about logical\ + \ fallacies.\n\n" +"tag": "mmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "mmlu_logical_fallacies" +"task_alias": "logical_fallacies" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_machine_learning.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d846f96084a8cba059348a90d800a86b92ba09c2 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_machine_learning.yaml @@ -0,0 +1,7 @@ +"dataset_name": "machine_learning" +"description": "The following are multiple choice questions (with answers) about machine\ + \ learning.\n\n" +"tag": "mmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "mmlu_machine_learning" +"task_alias": "machine_learning" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_management.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7dff834ef804039858b6955155a8338dd11b30b3 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_management.yaml @@ -0,0 +1,7 @@ +"dataset_name": "management" +"description": "The following are multiple choice questions (with answers) about management.\n\ + \n" +"tag": "mmlu_other_tasks" +"include": "_default_template_yaml" +"task": "mmlu_management" +"task_alias": "management" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_marketing.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ef004988965c41ff075f2f976b98dca4657ca04 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_marketing.yaml @@ -0,0 +1,7 @@ +"dataset_name": "marketing" +"description": "The following are multiple choice questions (with answers) about marketing.\n\ + \n" +"tag": "mmlu_other_tasks" +"include": "_default_template_yaml" +"task": "mmlu_marketing" +"task_alias": "marketing" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_medical_genetics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..989fb2c1aea91035421e49c7a11293c48ffec0bc --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_medical_genetics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "medical_genetics" +"description": "The following are multiple choice questions (with answers) about medical\ + \ genetics.\n\n" +"tag": "mmlu_other_tasks" +"include": "_default_template_yaml" +"task": "mmlu_medical_genetics" +"task_alias": "medical_genetics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_miscellaneous.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e7bb68bc2eb0f55b784943bd18296aabe3b86a31 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_miscellaneous.yaml @@ -0,0 +1,7 @@ +"dataset_name": "miscellaneous" +"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ + \n" +"tag": "mmlu_other_tasks" +"include": "_default_template_yaml" +"task": "mmlu_miscellaneous" +"task_alias": "miscellaneous" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_moral_disputes.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..348d21403f06669e198146286b83e227fbde5a16 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_moral_disputes.yaml @@ -0,0 +1,7 @@ +"dataset_name": "moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes.\n\n" +"tag": "mmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "mmlu_moral_disputes" +"task_alias": "moral_disputes" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_moral_scenarios.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3762ee1200848439f08a3c69703af4cffb3a9d74 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_moral_scenarios.yaml @@ -0,0 +1,7 @@ +"dataset_name": "moral_scenarios" +"description": "The following are multiple choice questions (with answers) about moral\ + \ scenarios.\n\n" +"tag": "mmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "mmlu_moral_scenarios" +"task_alias": "moral_scenarios" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_nutrition.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..55f8ca01ff42a296c07d8fd2e2ccda373d91775b --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_nutrition.yaml @@ -0,0 +1,7 @@ +"dataset_name": "nutrition" +"description": "The following are multiple choice questions (with answers) about nutrition.\n\ + \n" +"tag": "mmlu_other_tasks" +"include": "_default_template_yaml" +"task": "mmlu_nutrition" +"task_alias": "nutrition" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_philosophy.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5331c812ef70cb0123d754835fabde16ce330245 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_philosophy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "philosophy" +"description": "The following are multiple choice questions (with answers) about philosophy.\n\ + \n" +"tag": "mmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "mmlu_philosophy" +"task_alias": "philosophy" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_prehistory.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0b4ff970a10b7be9ab08527124ea236227b60428 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_prehistory.yaml @@ -0,0 +1,7 @@ +"dataset_name": "prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory.\n\ + \n" +"tag": "mmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "mmlu_prehistory" +"task_alias": "prehistory" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_professional_accounting.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..27b2ec9b9b70e00616d2560c3a8b1259781e8cfb --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_professional_accounting.yaml @@ -0,0 +1,7 @@ +"dataset_name": "professional_accounting" +"description": "The following are multiple choice questions (with answers) about professional\ + \ accounting.\n\n" +"tag": "mmlu_other_tasks" +"include": "_default_template_yaml" +"task": "mmlu_professional_accounting" +"task_alias": "professional_accounting" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_professional_law.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..07c36f1c38d46a513359d80284ead794dd72b7bd --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_professional_law.yaml @@ -0,0 +1,7 @@ +"dataset_name": "professional_law" +"description": "The following are multiple choice questions (with answers) about professional\ + \ law.\n\n" +"tag": "mmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "mmlu_professional_law" +"task_alias": "professional_law" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_professional_medicine.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2c5754bf379cfd884ad837243105a49e3e28d386 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_professional_medicine.yaml @@ -0,0 +1,7 @@ +"dataset_name": "professional_medicine" +"description": "The following are multiple choice questions (with answers) about professional\ + \ medicine.\n\n" +"tag": "mmlu_other_tasks" +"include": "_default_template_yaml" +"task": "mmlu_professional_medicine" +"task_alias": "professional_medicine" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_professional_psychology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e0c0608ef6860edb4b8492402c674a7efda2070f --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_professional_psychology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology.\n\n" +"tag": "mmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "mmlu_professional_psychology" +"task_alias": "professional_psychology" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_public_relations.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..43b675bdfd088bb7e651eece031198b5c0fb8ab3 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_public_relations.yaml @@ -0,0 +1,7 @@ +"dataset_name": "public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations.\n\n" +"tag": "mmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "mmlu_public_relations" +"task_alias": "public_relations" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_security_studies.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b02342d95ede5148ee8b0aeb9e4ad4fb7dd05938 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_security_studies.yaml @@ -0,0 +1,7 @@ +"dataset_name": "security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies.\n\n" +"tag": "mmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "mmlu_security_studies" +"task_alias": "security_studies" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_sociology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..49fa11620fb7147752328a484d56f8ead64c4387 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_sociology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "sociology" +"description": "The following are multiple choice questions (with answers) about sociology.\n\ + \n" +"tag": "mmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "mmlu_sociology" +"task_alias": "sociology" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_us_foreign_policy.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc4335e9eace7816ba112e4f55912223444d4c1f --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_us_foreign_policy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy.\n\n" +"tag": "mmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "mmlu_us_foreign_policy" +"task_alias": "us_foreign_policy" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_virology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f8bc114c3ce7437ad0fb413a69a859f69bcbf99 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_virology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "virology" +"description": "The following are multiple choice questions (with answers) about virology.\n\ + \n" +"tag": "mmlu_other_tasks" +"include": "_default_template_yaml" +"task": "mmlu_virology" +"task_alias": "virology" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_world_religions.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b90972c7031c30d89beea835f70aab7cf45cce81 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/default/mmlu_world_religions.yaml @@ -0,0 +1,7 @@ +"dataset_name": "world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions.\n\n" +"tag": "mmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "mmlu_world_religions" +"task_alias": "world_religions" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/_cot_prompts.json b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/_cot_prompts.json new file mode 100644 index 0000000000000000000000000000000000000000..c374b19d03391e61021af6640558a6de8853d7b0 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/_cot_prompts.json @@ -0,0 +1 @@ +{"abstract_algebra": "The following are multiple choice questions (with answers) about abstract algebra.\n\nQ: Statement 1 | Every element of a group generates a cyclic subgroup of the group. Statement 2 | The symmetric group S_10 has 10 elements.\n(A) True, True (B) False, False (C) True, False (D) False, True\nA: Let's think step by step. A cyclic group is a group that is generated by a single element. Hence a subgroup generated by a single element of a group is cyclic and Statement 1 is True. The answer is (C).\n\nQ: The symmetric group $S_n$ has $\nactorial{n}$ elements, hence it is not true that $S_{10}$ has 10 elements.\nFind the characteristic of the ring 2Z.\n(A) 0 (B) 3 (C) 12 (D) 30\nA: Let's think step by step. A characteristic of a ring is R is $n$ if the statement $ka = 0$ for all $a\\in 2Z$ implies that $k$ is a multiple of $n$. Assume that $ka = 0$ for all $a\\in 2Z$ for some $k$. In particular $2k = 0$. Hence $k=0$ and $n=0$. The answer is (A).\n\nQ: Statement 1| Every function from a finite set onto itself must be one to one. Statement 2 | Every subgroup of an abelian group is abelian.\n(A) True, True (B) False, False (C) True, False (D) False, True\nA: Let's think step by step. Statement 1 is true. Let $S$ be a finite set. If $f:S \nightarrow S$ is a onto function, then $|S| = |f(S)|$. If $f$ was not one to one, then for finite domain $S$ the image would have less than $S$ elements, a contradiction.\nStatement 2 is true. Let $G$ be an abelian group and $H$ be a subgroup of $G$. We need to show that $H$ is abelian. Let $a,b \\in H$. Then $a,b \\in G$ and $ab=ba$. Since $G$ is abelian, $ab=ba$. Since $H$ is a subgroup of $G$, $ab \\in H$. Therefore, $ab=ba$ and $H$ is abelian. The answer is (A).\n\nQ: Statement 1 | If aH is an element of a factor group, then |aH| divides |a|. Statement 2 | If H and K are subgroups of G then HK is a subgroup of G.\n(A) True, True (B) False, False (C) True, False (D) False, True\nA: Let's think step by step. Statement 2 is false. Let $H$ be a subgroup of $S_3$ generated by the cycle $(1,2)$ and $K$ be a subgroup of $S_3$ generated by the cycle $(1,3)$. Both $H$ and $K$ have two elements, the generators and the identity. However $HK$ contains cycles (1,2), (1,3) and (2,3,1), but the inverse of (2,3,1) is (2,1,3) and it does not belong to HK, hence HK is not a subgroup. The answer is (B).\n\nQ: Find all c in Z_3 such that Z_3[x]/(x^2 + c) is a field.\n(A) 0 (B) 1 (C) 2 (D) 3\nA: Let's think step by step. Z_3[x]/(x^2 + c) is a field if and only if x^2 + c does not have roots in Z_3. That is x^2 + c != 0 for every x in Z_3. If c = 0, then x^2 + c = x^2 has root 0. If c = 1 then x^2 + c = x^2 + 1 = 0 + 1 for x = 0, 1 + 1 = 2 for x = 1 and 1 + 1 = 2 for x = 2, hence x^2 + 1 does not have any roots. For c = 2 the polynomial x^2 + 2 has two roots at x = 1 and x = 2. Hence Z_3[x]/(x^2 + c) is a field if and only if c = 1. The answer is (B).\n\n", "anatomy": "The following are multiple choice questions (with answers) about anatomy.\n\nQ: Which of the following is the body cavity that contains the pituitary gland?\n(A) Abdominal (B) Cranial (C) Pleural (D) Spinal\nA: Let's think step by step. We refer to Wikipedia articles on anatomy for help. Let\u2019s solve this problem step by step. The pituitary gland is the major endocrine gland attached to the base of the brain, and it is contained in the Cranial cavity. The answer is (B).\n\nQ: Which of these branches of the trigeminal nerve contain somatic motor processes?\n(A) The supraorbital nerve (B) The infraorbital nerve (C) The mental nerve (D) None of the above\nA: Let's think step by step. We refer to Wikipedia articles on anatomy for help. Let\u2019s solve this problem step by step. \nWe know the following: (A) The supraorbital nerve (also known as the frontal nerve) is the largest branch of the ophthalmic nerve and branch of ophthalmic division of the trigeminal nerve. (B) The infraorbital nerve is a branch of the maxillary division of the trigeminal nerve. (C) The mental nerve is a branch of the mandibular division of the trigeminal nerve. Because all these nerves are purely sensory nerves and do not contain any somatic motor processes. Therefore, the answer should be none of the above, which is (D). The answer is (D).\n\nQ: In Angle's Class II Div 2 occlusion there is\n(A) excess overbite of the upper lateral incisors. (B) negative overjet of the upper central incisors. (C) excess overjet of the upper lateral incisors. (D) excess overjet of the upper central incisors.\nA: Let's think step by step. We refer to Wikipedia articles on anatomy for help. Let\u2019s solve this problem step by step. This is a question related to anatomy and orthodontics. Excess overjet is associated with Class II occlusions; therefore, we can safely eliminate (B) from the list, as negative overjet is often associated with Class III occlusions. Now, we need to determine the location of the excess overjet, and that would be the upper (maxillary) lateral incisors. Only (C) has the correct information. The answer is (C).\n\nQ: The pleura\n(A) have no sensory innervation. (B) are separated by a 2 mm space. (C) extend into the neck. (D) are composed of respiratory epithelium.\nA: Let's think step by step. We refer to Wikipedia articles on anatomy for help. Let\u2019s solve this problem step by step. First, recall that the pleura refers to the thin layer of tissue that covers the lungs and lines the interior wall of the chest cavity. Now, let\u2019s look at each option:\nOption (A): \u201cThe pleura have no sensory innervation.\u201d This information is not correct. The pleura do have a sensory innervation.\nOption (B): \u201cThe pleura are separated by a 2 mm space.\u201d This information is not correct. There is a very thin \u201cpotential\u201d space between the layers of the pleura; however, it is typically filled with serous pleural fluid. \nOption (C): \u201cThe pleura extend into the neck.\u201d This information is actuakky true. The cervical pleura, also known as the dome of the pleuradome of the pleura, lines the extendsiton of the pleural cavity into the neck.\nOption (D): \u201cThe pleura are composed of respiratory epithelium.\u201d This information is not correct. The pleaura are composed of connective tissue (CT).\nBecause (A), (B), and (D) are all incorrect, (D) is the only correct answer. The answer is (C).\n\nQ: What is the embryological origin of the hyoid bone?\n(A) The first pharyngeal arch (B) The first and second pharyngeal arches (C) The second pharyngeal arch (D) The second and third pharyngeal arches\nA: Let's think step by step. We refer to Wikipedia articles on anatomy for help. Let\u2019s solve this problem step by step. The hyoid bone, which is also known as the hyooid, is a a small U-shaped bone located in the anterior neck. In its resting position, it lies between the ase of the mandible and the third cervical vertebrae. We know that the second and the third pharyngeal arches give rise to the horns of the hyoid bone; therefore, the embryological origin of the hyoid bone are the second and the third pharyngeal arches\u2014this information is covered in the last option (D). Therefore, we conclude that (D) must be the correct answer. The answer is (D).\n\n", "astronomy": "The following are multiple choice questions (with answers) about astronomy.\n\nQ: Where do most short-period comets come from and how do we know?\n(A) The Kuiper belt; short period comets tend to be in the plane of the solar system just like the Kuiper belt. (B) The Kuiper belt; short period comets tend to come from random directions indicating a spherical distribution of comets called the Kuiper belt. (C) The asteroid belt; short period comets have orbital periods similar to asteroids like Vesta and are found in the plane of the solar system just like the asteroid belt. (D) The Oort cloud; short period comets tend to be in the plane of the solar system just like the Oort cloud.\nA: Let's think step by step. Most short-period comets come from the Kuiper belt, and we know because short period coments tend to be in the plane of the solar system, just like the Kuiper belt is. The answer is (A).\n\nQ: You are pushing a truck along a road. Would it be easier to accelerate this truck on Mars? Why? (Assume there is no friction)\n(A) It would be harder since the truck is heavier on Mars. (B) It would be easier since the truck is lighter on Mars. (C) It would be harder since the truck is lighter on Mars. (D) It would be the same no matter where you are.\nA: Let's think step by step. If we assume that there is no friction, the force needed to accelerate the truck is by Newton\u2019s second law only dependent on the mass of the truck. Hence (A), (B) and (C) are incorrect since it doesn\u2019t matter that it\u2019s on Mars, and (D) is the correct answer. The answer is (D).\n\nQ: Say the pupil of your eye has a diameter of 5 mm and you have a telescope with an aperture of 50 cm. How much more light can the telescope gather than your eye?\n(A) 10000 times more (B) 100 times more (C) 1000 times more (D) 10 times more\nA: Let's think step by step. The amount of light is proportional to the aperture area $A = \\pi D^2/4$ for a lens with diameter $D$, so the relative amounts of light between the eye with diameter 5mm and the telescope with diameter 50mm is $(50 cm)^2/(5mm)^2 = 10000$. The answer is (A).\n\nQ: Why isn't there a planet where the asteroid belt is located?\n(A) A planet once formed here but it was broken apart by a catastrophic collision. (B) There was not enough material in this part of the solar nebula to form a planet. (C) There was too much rocky material to form a terrestrial planet but not enough gaseous material to form a jovian planet. (D) Resonance with Jupiter prevented material from collecting together to form a planet.\nA: Let's think step by step. The asteroid belt is a stellar disc consisting of a large number of asteroids between Mars and Jupiter's orbits. The asteroids in this belt are affected by the gravitational pull from both other asteroids and nearby planets. Due to the strong gravitational force of Jupiter there are resonances that give rise to low density regions of asteroids known as the Kirkwood gap. So (B) and (C) are not correct since it\u2019s not a lack of material that prevents a planet from being formed, and (A) is incorrect because the Kirkwood gap would have prevented a planet from forming in the first place, and (D) is the correct option. The answer is (D).\n\nQ: Why is Mars red?\n(A) Because the surface is covered with heavily oxidized (\"rusted\") minerals. (B) Because the atmosphere scatters more light at bluer wavelengths transmitting mostly red light. (C) Because Mars is covered with ancient lava flows which are red in color. (D) Because flowing water on Mars's surface altered the surface minerals several billion years ago.\nA: Let's think step by step. Option (B) is not correct because if the red color was caused by the scattering off the atmosphere, then the earth with a much thicker atmosphere would also look red. Options (C) and (D) are not specific enough about why the color of the surface would be red, while (A) is correct because it explains that the surface is red due to the rusted materials on the surface and the red color comes from the rust. So the correct option is (A). The answer is (A).\n\n", "business_ethics": "The following are multiple choice questions (with answers) about business ethics.\n\nQ: In contrast to _______, _______ aim to reward favourable behaviour by companies. The success of such campaigns have been heightened through the use of ___________, which allow campaigns to facilitate the company in achieving _________ .\n(A) Buycotts, Boycotts, Blockchain technology, Charitable donations (B) Buycotts, Boycotts, Digital technology, Increased Sales (C) Boycotts, Buyalls, Blockchain technology, Charitable donations (D) Boycotts, Buycotts, Digital technology, Increased Sales\nA: Let's think step by step. We refer to Wikipedia articles on business ethics for help. The sentence that best uses the possible options above is \u201cIn contrast to *boycotts*, *buycotts* aim to reward favourable behavior by companies. The success of such campaigns have been heightened through the use of *digital technology*, which allow campaigns to facilitate the company in achieving *increased sales*.\u201d The answer is (D).\n\nQ: _______ is the direct attempt to formally or informally manage ethical issues or problems, through specific policies, practices and programmes.\n(A) Corporate social responsibility (B) Business ethics management (C) Sustainability (D) Environmental management\nA: Let's think step by step. We refer to Wikipedia articles on business ethics for help. The direct attempt manage ethical issues through specific policies, practices, and programs is business ethics management. The answer is (B).\n\nQ: Three contrasting tactics that CSO's can engage in to meet their aims are ________ which typically involves research and communication, ________, which may involve physically attacking a company's operations or ________, often involving some form of _______.\n(A) Non-violent direct action, Violent direct action, Indirect action, Boycott (B) Indirect action, Instrumental action, Non-violent direct action, Information campaign (C) Indirect action, Violent direct action, Non-violent direct-action Boycott (D) Non-violent direct action, Instrumental action, Indirect action, Information campaign\nA: Let's think step by step. We refer to Wikipedia articles on business ethics for help. The sentence that best uses the possible options above is \u201cThree contrasting tactics that CSO's can engage in to meet their aims are *indirect action*, which typically involves research and communication, *violent direct action*, which may involve physically attacking a company's operations or *non-violent direct action*, often involving some form of *boycott*.\u201d The answer is (C).\n\nQ: To ensure the independence of the non-executive board members, there are a number of steps which can be taken, which include non-executives being drawn from _______ the company, being appointed for a _________ time period as well as being appointed _________.\n(A) Outside, Limited, Independently (B) Inside, Limited, Intermittently (C) Outside, Unlimited, Intermittently (D) Inside, Unlimited, Independently\nA: Let's think step by step. We refer to Wikipedia articles on business ethics for help. The sentence that best uses the possible options above is \u201cTo ensure the independence of the non-executive board members, there are a number of steps which can be taken, which include non-executives being draw from *outside* the company, being appointed for a *limited* time period as well as being imported *independently*. The answer is (A).\n\nQ: Beyond the business case for engaging in CSR there are a number of moral arguments relating to: negative _______, the _______that corporations possess and the ________ of business and society.\n(A) Externalities, Power, Independence (B) Publicity, Insubstantial resources, Mutual dependence (C) Publicity, Power, Independence (D) Externalities, Power, Mutual dependence\nA: Let's think step by step. We refer to Wikipedia articles on business ethics for help. The sentence that best uses the possible options above is \u201cBeyond the business case for engaging the CSR there are a number of moral arguments relating to: negative *externalities*, the *power* that corporations possess and the *mutual independence* of business and society. The answer is (D).\n\n", "clinical_knowledge": "The following are multiple choice questions (with answers) about clinical knowledge.\n\nQ: Glycolysis is the name given to the pathway involving the conversion of:\n(A) glycogen to glucose-1-phosphate. (B) glycogen or glucose to fructose. (C) glycogen or glucose to pyruvate or lactate. (D) glycogen or glucose to pyruvate or acetyl CoA.\nA: Let's think step by step. We refer to Wikipedia articles on clinical knowledge for help. Glycolysis is the name given to the pathway involving conversion of glycogen or glucose to pyruvate or lactate. The answer is (C).\n\nQ: What is the difference between a male and a female catheter?\n(A) Male and female catheters are different colours. (B) Male catheters are longer than female catheters. (C) Male catheters are bigger than female catheters. (D) Female catheters are longer than male catheters.\nA: Let's think step by step. We refer to Wikipedia articles on clinical knowledge for help. The difference between a male and female catheter is that male catheters tend to be longer than female catheters. The answer is (B).\n\nQ: How many attempts should you make to cannulate a patient before passing the job on to a senior colleague, according to the medical knowledge of 2020?\n(A) 4 (B) 3 (C) 2 (D) 1\nA: Let's think step by step. We refer to Wikipedia articles on clinical knowledge for help. According to the medical protocol as of 2020, you should make two attempts to cannulate a patient before passing the job on to a more-senior practitioner. The answer is (C).\n\nQ: In the assessment of the hand function which of the following is true?\n(A) Abduction of the thumb is supplied by spinal root T2 (B) Opposition of the thumb by opponens policis is supplied by spinal root T1 (C) Finger adduction is supplied by the median nerve (D) Finger abduction is mediated by the palmar interossei\nA: Let's think step by step. We refer to Wikipedia articles on clinical knowledge for help. Of all the options, it is only true that the opposition of the thumb by opponens pollicis is supplied by spinal root T1. The answer is (B).\n\nQ: The energy for all forms of muscle contraction is provided by:\n(A) ATP. (B) ADP. (C) phosphocreatine. (D) oxidative phosphorylation.\nA: Let's think step by step. We refer to Wikipedia articles on clinical knowledge for help. The energy for muscular contraction is provided by ATP (adenosine triphosphate), which is the powerhouse of the cell. The answer is (A).\n\n", "college_biology": "The following are multiple choice questions (with answers) about college biology.\n\nQ: Which of the following represents an accurate statement concerning arthropods?\n(A) They possess an exoskeleton composed primarily of peptidoglycan. (B) They possess an open circulatory system with a dorsal heart. (C) They are members of a biologically unsuccessful phylum incapable of exploiting diverse habitats and nutrition sources. (D) They lack paired, jointed appendages.\nA: Let's think step by step. Peptidoglycan is known to comprise the plasma membrane of most bacteria, rather than the exoskeleton of arthropods, which is made of chitin, which rules out (A). The answer (C) is false because arthropods are a highly successful phylum. Likewise, arthropods have paired, jointed appendages, which rules out (D). The only remaining option is (B), as arthropods have an open circulatory system with a dorsal tubular heart. The answer is (B).\n\nQ: In a given population, 1 out of every 400 people has a cancer caused by a completely recessive allele, b. Assuming the population is in Hardy-Weinberg equilibrium, which of the following is the expected proportion of individuals who carry the b allele but are not expected to develop the cancer?\n(A) 1/400 (B) 19/400 (C) 20/400 (D) 38/400\nA: Let's think step by step. According to the Hardy Weinberg Law, $p^2 + 2 p q + q^2 = 1$, and $p + q = 1$ where $p$ is the frequency of the dominant allele, $q$ is the frequency of the recessive allele, and $p^2$, $q^2$, and $2pq$ are the frequencies of dominant homozygous, recessive homozygous, and heterozygous individuals, respectively. \u200bThe frequency of the recessive allele (q) is $\\sqrt{\frac{1}{400}} = 0.05$. We have $p = 1 - q = 0.95$. The frequency of heterozygous individuals is $2pq = 2 \\cdot 0.05 \\cdot 0.95 = 0.095$. The number of heterozygous individuals is equal to the frequency of heterozygous individuals times the size of the population, or $0.095 * 400 = 38$. So we end up with 38/400. The answer is (D).\n\nQ: According to the pressure-flow model of movement of phloem contents, photosynthate movement from source to sink is driven by\n(A) an ATP-dependent pressure-flow pump (B) a water-pressure potential gradient (C) transpiration (D) apoplastic diffusion\nA: Let's think step by step. It is a gradient in water pressure that induces the movement of phloem content, which refers to answer (B). The mechanism of movement does not rely on metabolism, which rules out (A). Transpiration refers to the exhalation of water vapor through plant stomata, and is also not related, which rules out (C). While the apoplastic pathway is one of two main pathways for water transport in plants, it is not central to the pressure flow model, which rules out (D). The answer is (B).\n\nQ: Which of the following contain DNA sequences required for the segregation of chromosomes in mitosis and meiosis?\n(A) Telomeres (B) Centromeres (C) Nucleosomes (D) Spliceosomes\nA: Let's think step by step. The genetic material in Telomeres is not used, which rules out (A). Nucleosomes are the repeating subunit that comprises chromatin packed in a cell nucleus, and do not specifically refer to DNA sequences necessary for segregating chromosomes in cell division, which rules out (C). A spliceosome is a large ribonucleoprotein that removes introns from transcribed pre-mRNA rather than governing chromosome segregation. Centromeres are directly responsible for segregating chromosomes in cell division. The answer is (B).\n\nQ: The presence of homologous structures in two different organisms, such as the humerus in the front limb of a human and a bird, indicates that\n(A) the human and bird are polyphyletic species (B) a human's and bird's evolution is convergent (C) the human and bird belong to a clade (D) the human and bird developed by analogy\nA: Let's think step by step. Polyphyletic species are organisms that are grouped due to having similar characteristics but which do not have a common ancestor. This is not the case for humans and birds, which rules out (A). Convergent evolution refers to the indepdendent development of similar features in different species at different periods, which is also not the case for humans and birds, which rules out (B). Analogy refers to the superficial resemblance of structures that have different origins, which is not the case for the human and bird forearms, which rules out (D). Humans and birds do belong to the same clade - a group of organisms composed of a common ancestor. The answer is (C).\n\n", "college_chemistry": "The following are multiple choice questions (with answers) about college chemistry.\n\nQ: 3 Cl\u2212(aq) + 4 CrO_4^2\u2212(aq) + 23 H+(aq) \u2192 3 HClO2(aq) + 4 Cr3+(aq) + 10 H2O(l). In the reaction shown above, Cl\u2212(aq) behaves as\n(A) an acid (B) a base (C) a catalyst (D) a reducing agent\nA: Let's think step by step. A molecule that behaves as a base accepts an H+ ion (or proton) from another molecule, whereas a molecule that behaves as an acid donates an H+ ion (or proton) to another molecule. Neither of these is the case for Cl in this reaction, which rules out (A) and (B). A catalyst is a substance that only accelerates a reaction without itself undergoing chemical change, which is not the case here. This rules out (C). Instead, the $Cl^{-} molecules carry a negative charge, which they donate in the reaction to form 3 HClO2. This is the behavior of a reducing agent, or (D). The answer is (D).\n\nQ: Which of the following statements about the lanthanide elements is NOT true?\n(A) The most common oxidation state for the lanthanide elements is +3. (B) Lanthanide complexes often have high coordination numbers (> 6). (C) All of the lanthanide elements react with aqueous acid to liberate hydrogen. (D) The atomic radii of the lanthanide elements increase across the period from La to Lu.\nA: Let's think step by step. The atomic radii of the lanthanide elements in fact decrease across the period from La to Lu. Options (A), (B), and (C) are all true. This means that only (D) is NOT true. The answer is (D).\n\nQ: Which of the following lists the hydrides of group-14 elements in order of thermal stability, from lowest to highest?\n(A) PbH4 < SnH4 < GeH4 < SiH4 < CH4 (B) PbH4 < SnH4 < CH4 < GeH4 < SiH4 (C) CH4 < SiH4 < GeH4 < SnH4 < PbH4 (D) CH4 < PbH4 < GeH4 < SnH4 < SiH4\nA: Let's think step by step. The thermal stability of group-14 hydrides decreases as we move from the top of group 14 to the bottom. The order of elements in the group from top to bottom is C, Si, Ge, Sn, Pb. Therefore in order of increasing thermal stability we have PbH4, SnH4, GeH4, SiH4, and CH4, or answer (A). The answer is (A).\n\nQ: Predict the number of lines in the EPR spectrum of a solution of 13C-labelled methyl radical (13CH3\u2022), assuming the lines do not overlap.\n(A) 4 (B) 3 (C) 6 (D) 24 (E) 8\nA: Let's think step by step. The electron paramagnetic resonance spectrum will be split by two forms of interactions. The first is the hyperfine interaction with the 13C (nuclear spin $I = \nrac{1}{2}$) which will split the spectrum into 2 lines. This will be further split into 4 lines by the interaction with three equivalent 1H nuclei. The total number of lines is therefore $2 \\cdot 4 = 8$. The answer is (E).\n\n", "college_computer_science": "The following are multiple choice questions (with answers) about college computer science.\n\nQ: Which of the following regular expressions is equivalent to (describes the same set of strings as) (a* + b)*(c + d)?\n(A) a*(c + d)+ b(c + d)\n(B) a*(c + d)* + b(c + d)*\n(C) a*(c + d)+ b*(c + d)\n(D) (a + b)*c +(a + b)*d\nA: Let's think step by step. We know that:\n1. (X* + Y)* = (X + Y)*\n2. X(Y + Z)? = XY + XZ\nUsing equation 1 we can rewrite (a* + b)*(c + d)? as:\n3. (a + b)*(c + d)?\nUsing equation 2 we can rewrite equation 3 as:\n(a + b)*c + (a + b)*d The answer is (D).\n\nQ: The Singleton design pattern is used to guarantee that only a single instance of a class may be instantiated. Which of the following is (are) true of this design pattern?\nI. The Singleton class has a static factory method to provide its instance.\nII. The Singleton class can be a subclass of another class.\nIII. The Singleton class has a private constructor.\n(A) I only\n(B) II only\n(C) III only\n(D) I, II, and III\nA: Let's think step by step. Statement I is a correct statement about a Singleton, because a Singleton restricts instantiation to a single, static method. Statement II is also correct, because there is no inherent restriction regarding the inheritance of a Singleton. Statement III is also correct, because a Singletons must be instantiated only once, so its constructor is made private to prevent any construction except via its static factory method.\nGiven these facts, statements I, II, and III are all correct. The answer is (D).\n\nQ: A certain pipelined RISC machine has 8 general-purpose registers R0, R1, . . . , R7 and supports the following operations:\nADD Rs1, Rs2, Rd (Add Rs1 to Rs2 and put the sum in Rd)\nMUL Rs1, Rs2, Rd (Multiply Rs1 by Rs2 and put the product in Rd)\nAn operation normally takes one cycle; however, an operation takes two cycles if it produces a result required by the immediately following operation in an operation sequence.\nConsider the expression AB + ABC + BC, where variables A, B, C are located in registers R0, R1, R2. If the contents of these three registers must not be modified, what is the minimum number of clock cycles required for an operation sequence that computes the value of AB + ABC + BC?\n(A) 5 (B) 6 (C) 7 (D) 8\nA: Let's think step by step. First, we are given that A is in R0, B is in R1, and C is in R2.\nNext, we can see that we must compute three multiplies (AB, BC, and ABC) and two adds (AB + ABC, (AB + ABC) + BC) to compute our final answer, resulting in a minimum of five clock cycles.\nNext, we can see that there is no way to avoid at least one pipeline stall when computing our final answer, because to compute our final sum we must wait at least one cycle for the results from the previous stage to be ready. Thus, our minimum number of cycles must be 6.\nWe can verify that we can create a solution that requires only six cycles as follows:\ncompute AB: MUL R0, R1, R3\ncompute BC: MUL R1, R2, R4\ncompute ABC: MUL R3, R4, R5\ncompute AB + BC: ADD R3, R4, R6\nSTALL\ncompute AB + ABC + BC: ADD R5, R6, R7\nSo there are 6 cycles. The answer is (B).\n\nQ: A compiler generates code for the following assignment statement.\nG := (A + B) * C - (D + E) * F\nThe target machine has a single accumulator and a single-address instruction set consisting of instructions load, store, add, subtract, and multiply. For the arithmetic operations, the left operand is taken from the accumulator and the result appears in the accumulator. The smallest possible number of instructions in the resulting code is\n(A) 5 (B) 6 (C) 7 (D) 9\nA: Let's think step by step. We can compute the final answer with the following sequence of operations:\n1. LOAD D (accumulator = D)\n2. ADD E (accumulator = D+E)\n3. MUL F (accumulator = (D+E)*F)\n4. STORE X (X = (D+E)*F)\n5. LOAD A (accumulator = A)\n6. ADD B (accumulator = A+B)\n7. MUL C (accumulator = (A+B)*C)\n8. SUB X (accumulator = (A+B)*C - (D+E)*F)\n9. STORE G (G = (A+B)*C - (D+E)*F)\nThis sequence takes 9 instructions. The answer is (D).\n\nQ: Consider a computer design in which multiple processors, each with a private cache memory, share global memory using a single bus. This bus is the critical system resource. Each processor can execute one instruction every 500 nanoseconds as long as memory references are satisfied by its local cache. When a cache miss occurs, the processor is delayed for an additional 2,000 nanoseconds. During half of this additional delay, the bus is dedicated to serving the cache miss. During the other half, the processor cannot continue, but the bus is free to service requests from other processors. On average, each instruction requires 2 memory references. On average, cache misses occur on 1 percent of references. What proportion of the capacity of the bus would a single processor consume, ignoring delays due to competition from other processors?\n(A) 1/50 (B) 1/27 (C) 1/25 (D) 2/27\nA: Let's think step by step. We know that each instruction requires two memory references per instruction, and that there is an average cache miss rate of one percent.\nThus a given processor has:\n(1 cache miss / 100 references) * (2 references / instruction) =\n(2 cache misses / 100 instructions), so:\nmisses_per_instruction = 1 cache miss / 50 instructions.\nNext, we know that each instruction requires 500 nanoseconds when there is no cache miss, and 500 + 2000 = 2500 nanoseconds when there is a cache miss. Thus:\n50 instructions / (49 * 500) + (1 * 2500) nanoseconds, so:\ninstructions_per_ns = 50 instructions / 27000 nanoseconds.\nNow, we know that each cache miss locks the bus for half of the 2000 nanosecond cache miss delay, or 1000 nanoseconds, so:\nlock_ns_per_miss = 1000 nanoseconds / cache miss.\nThus we can see that on average a single processor will lock the bus for:\nlock_ns_per_miss * misses_per_instruction * instructions_per_ns =\n(1000 nanoseconds / cache miss) * (1 cache miss / 50 instructions) * (50 instructions / 27000 nanoseconds) = 1000 * (1/50) * (50/27000) = 1000/27000 = 1/27. The answer is (B).\n\n", "college_mathematics": "The following are multiple choice questions (with answers) about college mathematics.\n\nQ: Let V be the set of all real polynomials p(x). Let transformations T, S be defined on V by T:p(x) -> xp(x) and S:p(x) -> p'(x) = d/dx p(x), and interpret (ST)(p(x)) as S(T(p(x))). Which of the following is true?\n(A) ST = 0 (B) ST = T (C) ST = TS (D) ST - TS is the identity map of V onto itself.\nA: Let's think step by step. For a given polynomial $p$ we have\n\\[ST(p) = (xp(x))\u2019 = p(x) + xp\u2019(x)\\]\nand\n\\[TS(p) = xp\u2019(x).\\]\nHence \\[ST(p) - TS(p) = p(x) + xp\u2019(x) - xp\u2019(x).\\] The answer is (D).\n\nQ: Suppose that f(1 + x) = f(x) for all real x. If f is a polynomial and f(5) = 11, then f(15/2)\n(A) -11 (B) 0 (C) 11 (D) 33/2\nA: Let's think step by step. The only polynomial so that $f(1 + x) = f(x)$ is a constant polynomial. Hence $f(5) = 11 = f(15/2)$. The answer is (C).\n\nQ: Let A be a real 2x2 matrix. Which of the following statements must be true?\nI. All of the entries of A^2 are nonnegative.\nII. The determinant of A^2 is nonnegative.\nIII. If A has two distinct eigenvalues, then A^2 has two distinct eigenvalues.\n(A) I only (B) II only (C) III only (D) II and III only\nA: Let's think step by step. We have \\[ det(A^2) = (det(A))^2 \\geq 0,\\] hence II holds.\nIII is false: as a counterexample take a diagonal matrix with -1 and 1 on the diagonal. Then $A^2$ is the identity matrix. The answer is (B).\n\nQ: Let A be the set of all ordered pairs of integers (m, n) such that 7m + 12n = 22. What is the greatest negative number in the set B = {m + n : (m, n) \\in A}?\n(A) -5 (B) -4 (C) -3 (D) -2\nA: Let's think step by step. We have 12n = 22 - 7m and one of the solutions is $m = -2$, $n = 3$. Then $m + n = 1$, hence we need to look for smaller $m$ in order to make $m + n$ negative. The next solution is $m = -14$ and $n = 10$. For smaller $m$ we have $m + n$ smaller than $-4$. The answer is (B).\n\nQ: A tank initially contains a salt solution of 3 grams of salt dissolved in 100 liters of water. A salt solution containing 0.02 grams of salt per liter of water is sprayed into the tank at a rate of 4 liters per minute. The sprayed solution is continually mixed with the salt solution in the tank, and the mixture flows out of the tank at a rate of 4 liters per minute. If the mixing is instantaneous, how many grams of salt are in the tank after 100 minutes have elapsed?\n(A) 2 (B) 2 - e^-2 (C) 2 + e^-2 (D) 2 + e^-4\nA: Let's think step by step. For all $t \\in \\mathbb{R}$, let $s(t)$ denote the number grams of salt in the tank at the $t$ minute mark. Then $s(0) = 3$.\nWe use $s$ and $s(t)$ interchangeably. We also use $s^{\\prime}$ and $s^{\\prime}(t)$ interchangeably. The solution sprayed into the tank adds $(0.02) 4=2 / 25$ grams of salt per minute. There are always 100 liters of liquid in the tank, containing $s$ grams of salt. So the density of salt in the tank is $s / 100$ grams per liter. The flow of water out of the tank therefore subtracts $4(s / 100)=s / 25$ grams of salt per minute. Then, for all $t \\in \\mathbb{R}$, we have $s^{\\prime}(t)=(2 / 25)-(s / 25)=(2-s) / 25$, and so $[s(t)=2] \\Rightarrow\\left[s^{\\prime}(t)=0\right]$. For all $t \\in \\mathbb{R}$,\n$$\n\frac{d}{d t}[\\ln (s-2)]=\frac{s^{\\prime}}{s-2}=\frac{-1}{25}=\frac{d}{d t}\\left[-\frac{t}{25}\right] .\n$$\nChoose $C \\in \\mathbb{R}$ such that, for all $t \\in \\mathbb{R}, \\ln ((s(t)-2))=-[t / 25]+C$. Let $K:=e^{C}$. Then, for all $t \\in \\mathbb{R}$, we have $(s(t))-2=K e^{-t / 25}$, and so $s(t)=2+K e^{-t / 25}$. Then $3=s(0)=2+K e^{0}=2+K$, so $K=1$. Then $s(100)=2+K e^{-100 / 25}=2+1 \\cdot e^{-4}=2+e^{-4}$. The answer is (D).\n\n", "college_medicine": "The following are multiple choice questions (with answers) about college medicine.\n\nQ: An expected side effect of creatine supplementation is:\n(A) muscle weakness. (B) gain in body mass. (C) muscle cramps. (D) loss of electrolytes.\nA: Let's think step by step. We refer to Wikipedia articles on medicine for help. Creatine supplementation is a dietary supplement that results in body mass gain. The answer is (B).\n\nQ: Which of the following is not a true statement?\n(A) Muscle glycogen is broken down enzymatically to glucose-1-phosphate (B) Elite endurance runners have a high proportion of Type I fibres in their leg muscles (C) Liver glycogen is important in the maintenance of the blood glucose concentration (D) Insulin promotes glucose uptake by all tissues in the body\nA: Let's think step by step. We refer to Wikipedia articles on medicine for help. Let\u2019s solve this step by step and go over each choice: \n(A) \u201cMuscle glycogen is broken down enzymatically to glucose-1-phosphate\u201d: This is a correct statement.\n(B) \u201cElite endurance runners have a high proportion of Type I fibres in their leg muscles\u201d: This is a correct statement.\n(C) \u201cLiver glycogen is important in the maintenance of the blood glucose concentration\u201d: This is a correct statement. \n(D) \u201cInsulin promotes glucose uptake by all tissues in the body\u201d: This is not a correct statement, because insulin promotes glucose uptake by the liver, adipose tissue, and muscle, but not all tissues. For instance, the tissues in the brain and red blood cells are not affected by insulin. The answer is (D).\n\nQ: A high school science teacher fills a 1 liter bottle with pure nitrogen and seals the lid. The pressure is 1.70 atm, and the room temperature is 25\u00b0C. Which two variables will both increase the pressure of the system, if all other variables are held constant?\n(A) Increasing temperature, increasing moles of gas (B) Increasing temperature, increasing volume (C) Decreasing volume, decreasing temperature (D) Decreasing moles of gas, increasing volume\nA: Let's think step by step. We refer to Wikipedia articles on medicine for help. The relevant equation for this is the ideal gas law: PV=nRT. To increase the pressure of the system (P), then either n (number of moles of the gas) or T (temperature) have to increase. The answer is (A).\n\nQ: In a genetic test of a newborn, a rare genetic disorder is found that has X-linked recessive transmission. Which of the following statements is likely true regarding the pedigree of this disorder?\n(A) All descendants on the maternal side will have the disorder. (B) Females will be approximately twice as affected as males in this family. (C) All daughters of an affected male will be affected. (D) There will be equal distribution of males and females affected.\nA: Let's think step by step. We refer to Wikipedia articles on medicine for help. Let\u2019s solve this step by step. Let's recall first that females have two X chromosomes, while males have one X and one Y chromosome. This is an important fact we need to know before answering this question. \nBecause a male can only pass his only one X chromosome to a daughter, if he is affected by this rare genetic disorder, then we know for sure that he will pass this rare genetic disorder to all his future-born daughters. Therefore, \u201c(C): All daughters of an affected male will be affected\u201d is a correct statement. The answer is (C).\n\nQ: Glucose is transported into the muscle cell:\n(A) via protein transporters called GLUT4. (B) only in the presence of insulin. (C) via hexokinase. (D) via monocarbylic acid transporters.\nA: Let's think step by step. We refer to Wikipedia articles on medicine for help. Glucose (also known as the blood sugar) is the main sugar found in the human body. It is transported into the muscle cell via diffusion through protein transporters called GLUT4. The answer is (A).\n\n", "college_physics": "The following are multiple choice questions (with answers) about college physics.\n\nQ: A refracting telescope consists of two converging lenses separated by 100 cm. The eye-piece lens has a focal length of 20 cm. The angular magnification of the telescope is\n(A) 4 (B) 5 (C) 6 (D) 20\nA: Let's think step by step. In a refracting telescope, if both lenses are converging, the focus of both lenses must be between the two lenses, and thus the focal lengths of the two lenses must add up to their separation. Since the focal length of one lens is 20 cm, the focal length of the other must be 80 cm. The magnification is the ratio of these two focal lengths, or 4. The answer is (A).\n\nQ: The muon decays with a characteristic lifetime of about 10^-6 second into an electron, a muon neutrino, and an electron antineutrino. The muon is forbidden from decaying into an electron and just a single neutrino by the law of conservation of\n(A) charge (B) mass (C) energy and momentum (D) lepton number\nA: Let's think step by step. Lepton number must be conserved, meaning the total number of leptons minus the number of antileptons. If a muon decays into an electron and a single neutrino, the total lepton number would go from one to two, violating lepton number conservation. The answer is (D).\n\nQ: One end of a Nichrome wire of length 2L and cross-sectional area A is attached to an end of another Nichrome wire of length L and cross- sectional area 2A. If the free end of the longer wire is at an electric potential of 8.0 volts, and the free end of the shorter wire is at an electric potential of 1.0 volt, the potential at the junction of the two wires is most nearly equal to\n(A) 2.4 V (B) 3.3 V (C) 4.5 V (D) 5.7 V\nA: Let's think step by step. This is a simple voltage divider problem, where the longer wire has a resistance four times that of the shorter end. So the voltage divider ratio is 1 / 5, meaning that the potential in the middle is 1.0 V + (8.0 V - 1.0 V) * 1/5 = 2.4 V. The answer is (A).\n\nQ: A refracting telescope consists of two converging lenses separated by 100 cm. The eye-piece lens has a focal length of 20 cm. The angular magnification of the telescope is\n(A) 4 (B) 5 (C) 6 (D) 20\nA: Let's think step by step. In a refracting telescope, if both lenses are converging, the focus of both lenses must be between the two lenses, and thus the focal lengths of the two lenses must add up to their separation. Since the focal length of one lens is 20 cm, the focal length of the other must be 80 cm. The magnification is the ratio of these two focal lengths, or 4. The answer is (A).\n\nQ: For which of the following thermodynamic processes is the increase in the internal energy of an ideal gas equal to the heat added to the gas?\n(A) Constant temperature (B) Constant volume (C) Constant pressure (D) Adiabatic\nA: Let's think step by step. Heat added to the gas can go into the gases internal energy or work done against an external force. However, if the volume of the gas container is constant, no work will be done (since work is pressure times change in volume). So, at constant volume, all of the heat goes into the internal energy. The answer is (B).\n\n", "computer_security": "The following are multiple choice questions (with answers) about computer security.\n\nQ: SHA-1 has a message digest of\n(A) 160 bits (B) 512 bits (C) 628 bits (D) 820 bits\nA: Let's think step by step. Since SHA-1 is a hash function which takes an input and produces a 160-bit (20-byte) hash value, its message digest is 160 bits. The answer is (A).\n\nQ: _____________ can modify data on your system \u2013 so that your system doesn\u2019t run correctly or you can no longer access specific data, or it may even ask for ransom in order to give your access.\n(A) IM \u2013 Trojans (B) Backdoor Trojans (C) Trojan-Downloader (D) Ransom Trojan\nA: Let's think step by step. The system is asking for trojans, which are for ransom, which means ransom trojan. The answer is (D).\n\nQ: What is ethical hacking?\n(A) \"Hacking\" ethics so they justify unintended selfish behavior (B) Hacking systems (e.g., during penetration testing) to expose vulnerabilities so they can be fixed, rather than exploited (C) Hacking into systems run by those whose ethics you disagree with (D) A slang term for rapid software development, e.g., as part of hackathons\nA: Let's think step by step. Ethical hacking is a process of detecting vulnerabilities in an application, system, or organization's infrastructure that an attacker can use to exploit an individual or organization. They use this process to prevent cyberattacks and security breaches by lawfully hacking into the systems and looking for weak points. The answer is (B).\n\nQ: The ____________ is anything which your search engine cannot search.\n(A) Haunted web (B) World Wide Web (C) Surface web (D) Deep Web\nA: Let's think step by step. The search engine searches on the Surface Web, which is the portion of the world wide web which is visible so (B,C) are wrong. The Haunted Web doesn\u2019t correspond to an internet concept. The Deep Web is the part of the World Wide Web which is not indexed. The answer is (D).\n\nQ: Exploitation of the Heartbleed bug permits\n(A) overwriting cryptographic keys in memory (B) a kind of code injection (C) a read outside bounds of a buffer (D) a format string attack\nA: Let's think step by step. The Heartbleed Bug is a serious vulnerability in the popular OpenSSL cryptographic software library. Heartbleed resulted from improper input validation (due to a missing bounds check) in the implementation of the TLS heartbeat extension. The vulnerability was classified as a buffer over-read, a situation where more data can be read than should be allowed. The answer is (C).\n\n", "conceptual_physics": "\nThe following are multiple choice questions (with answers) about conceptual physics.\n\nQ: Colors in a soap bubble result from light\n(A) converted to a different frequency (B) deflection (C) interference (D) polarization\nA: Let's think step by step. In a soap bubble film, the light bounces between the two soap-air interfaces many times, interfering with itself constructively or destructively depending on the width of the film. This results in different colors being visible. The answer is (C).\n\nQ: Compared with the mass of a uranium atom undergoing fission, the combined masses of the products after fission are\n(A) less (B) more (C) the same (D) zero\nA: Let's think step by step. Fission releases energy, which comes from the rest mass of its initial nucleus. Thus the mass of the products is less than the mass of the reactant uranium nucleus. The answer is (A).\n\nQ: Things that are equivalent according to the equivalence principle are\n(A) space and time. (B) a traveling twin and a stay-at-home twin. (C) gravity and acceleration. (D) mass and energy.\nA: Let's think step by step. Einstein\u2019s famous equivalence principle states that gravity and acceleration are equivalent. The answer is (C).\n\nQ: Which of these three elements has the most mass per nucleon?\n(A) Hydrogen (B) Iron (C) Uranium (D) Same in each\nA: Let's think step by step. Due to nuclear binding energy, the mass of an atomic nucleus is less than the sum of individual masses of the free constituent protons and neutrons; this is known as the mass defect. Hydrogen has no mass defect because it has only a single nucleon, so it will have the most mass per nucleon. The answer is (A).\n\nQ: A model airplane flies slower when flying into the wind and faster with wind at its back. When launched at right angles to the wind a cross wind its groundspeed compared with flying in still air is\n(A) the same (B) greater (C) less (D) either greater or less depending on wind speed\nA: Let's think step by step. The plane\u2019s speed in the direction of the wind is greater than it would be in the absence of wind, and its direction orthogonal to the wind is the same as it would be in the absence of the wind. The total speed, which is these two components added in quadrature, is thus greater than the speed in still air. The answer is (B).\n\n", "econometrics": "The following are multiple choice questions (with answers) about econometrics.\n\nQ: Suppose now that a researcher wishes to use information criteria to determine the optimal lag length for a VAR. 500 observations are available for the bi-variate VAR, and the values of the determinant of the variance-covariance matrix of residuals are 0.0336, 0.0169, 0.0084, and 0.0062 for 1, 2, 3, and 4 lags respectively. What is the optimal model order according to Akaike's information criterion?\n(A) 1 lag (B) 2 lags (C) 3 lags (D) 4 lags\nA: Let's think step by step. We refer to Wikipedia articles on econometrics for help. Let\u2019s solve this problem step by step. First of all, let\u2019s recall that for a given set of data, Akaike's information criterion (AIC) allows us to measure how well a statistical model fits the data; it is an estimator of prediction error. Here in this problem we will need to use the formula ln(det(sigma_hat)) + (2 * k / T) to determine the values of Akaike\u2019s criterion, where ln denotes the natural log function, det the determinant function, k the total number of parameters in total (across both equations), and T the number of observations (which, in this case, is equal to 500). For 1 lag, the number of parameters in total is equal to 6; for 2 lags, it is 10; for 3 lags, it is 14; and for 4 lags, it is 18. Now, let\u2019s calculate the values of the criterion for each lag:\n(A) 1 lag: ln(0.0336) + (2 * 6 / 500) = ln(0.0336) + (12 / 500) = -3.369\n(B) 2 lags: ln(0.0169) + (2 * 10 / 500) = ln(0.0169) + (20 / 500) = -4.040\n(C) 3 lags: ln(0.0084) + (2 * 14 / 500) = ln(0.0084) + (28 / 500) =-4.724\n(D) 4 lags: ln(0.0062) + (2 * 18 / 500) = ln(0.0062) + (36 / 500) =-5.011\nBecause the optimal model order according to AIC minimizes the information criterion, the answer should be the one with the lowest value. In this case, (D) has the lowest value. The answer is (C).\n\nQ: Consider the following AR(1) model with the disturbances having zero mean and unit variance\nyt = 0.2 + 0.4 yt-1 + ut\nThe (unconditional) mean of y will be given by\n(A) 0.2 (B) 0.4 (C) 0.5 (D) 0.33\nA: Let's think step by step. We refer to Wikipedia articles on econometrics for help. Let\u2019s solve this problem step by step. If we have a an AR(1) model with the disturbances having zero mean and unit variance, then the unconditional mean of y is equal to the following:\nunconditional mean of y = (the intercept term) / (1 - autoregressive coefficient)\nWe know that the intercept term is 0.2 and the autoregressive coefficient is 0.4; thus, we have:\nunconditional mean of y = (0.2) / (1 - 0.4) = (0.2) / (0.6) = 2 / 6 = 1 / 3, which is approximately 0.33. That means that the answer should be (D) 0.33. The answer is (D).\n\nQ: What would be then consequences for the OLS estimator if heteroscedasticity is present in a regression model but ignored?\n(A) It will be biased (B) It will be inconsistent (C) It will be inefficient (D) All of (a), (b) and (c) will be true.\nA: Let's think step by step. We refer to Wikipedia articles on econometrics for help. Heteroscedasticity refers to the condition where the variance of the error terms is not constant across multiple observations. If heteroscedasticity is present in a regression model, then the coefficient estimates in the OLS estimator will be not only unbiased and consistent but also inefficient. Because (A) and (B) are incorrect choices and (C) is a correct choice, (D) cannot be the right answer. Ultimately, (C) is the only true choice. The answer is (C).\n\nQ: Suppose that a test statistic has associated with it a p-value of 0.08. Which one of the following statements is true?\n(i) If the size of the test were exactly 8%, we would be indifferent between rejecting and not rejecting the null hypothesis\n(ii) The null would be rejected if a 10% size of test were used\n(iii) The null would not be rejected if a 1% size of test were used\n(iv) The null would be rejected if a 5% size of test were used.\n(A) (ii) and (iv) only (B) (i) and (iii) only (C) (i), (ii), and (iii) only (D) (i), (ii), (iii), and (iv).\nA: Let's think step by step. We refer to Wikipedia articles on econometrics for help. Let\u2019s reason about each of the options.\n(i) is a true statement.\n(ii) is a true statement.\n(iii) is a true statement.\n(iv) is not a true statement. Thus, (i), (ii), and (iii) are true. The answer is (C).\n\nQ: For a stationary autoregressive process, shocks will\n(A) Eventually die away (B) Persist indefinitely (C) Grow exponentially (D) Never occur\nA: Let's think step by step. We refer to Wikipedia articles on econometrics for help. This is a formal logic problem about stationally process. For a stationary autoregressive process, shocks will eventually die away. The answer is (A).\n\n", "electrical_engineering": "\nThe following are multiple choice questions (with answers) about electrical engineering.\n\nQ: A point pole has a strength of 4\u03c0 * 10^-4 weber. The force in newtons on a point pole of 4\u03c0 * 1.5 * 10^-4 weber placed at a distance of 10 cm from it will be\n(A) 15 N. (B) 20 N. (C) 7.5 N. (D) 3.75 N.\nA: Let's think step by step. The force between two point poles is given by m_1m_2/(mu_0 4 \\pi r^2), in analogy to Coulomb\u2019s law. Plugging in the values given in the question, we calculate that the force is approximately 15 N. The answer is (A).\n\nQ: The coil of a moving coil meter has 100 turns, is 40 mm long and 30 mm wide. The control torque is 240*10-6 N-m on full scale. If magnetic flux density is 1Wb/m2 range of meter is\n(A) 1 mA. (B) 2 mA. (C) 3 mA. (D) 4 mA.\nA: Let's think step by step. The torque on a coil in a uniform magnetic field is given by BANI, where B is the magnetic flux density, A is the area of the coil, N is the number of turns, and I is the current. So we have that I = (Torque)/(BAN), or 240e-6/(1200e-6 * 100 * 1) = 2e-3. The answer is (B).\n\nQ: In an SR latch built from NOR gates, which condition is not allowed\n(A) S=0, R=0 (B) S=0, R=1 (C) S=1, R=0 (D) S=1, R=1\nA: Let's think step by step. An SR latch is a set-reset latch; in the case where S=1 and R=1, the circuit has no stable state; instead a race condition will be produced within the circuit, so the device will be in an undefined state. So S=1, R=1 is an illegal input. The answer is (D).\n\nQ: Two long parallel conductors carry 100 A. If the conductors are separated by 20 mm, the force per meter of length of each conductor will be\n(A) 100 N. (B) 0.1 N. (C) 1 N. (D) 0.01 N.\nA: Let's think step by step. The magnetic force-per-length between two current-carrying conductors is given by \\mu_0 I_1 I_2 / (2 \\pi r), where $r$ is the separation distance and I_1 and I_2 are the currents. Plugging in 100 A for I_1 and I_2, and 20 mm for r, gives 0.1 N. The answer is (B).\n\nQ: In a 2 pole lap winding dc machine , the resistance of one conductor is 2\u03a9 and total number of conductors is 100. Find the total resistance\n(A) 200\u03a9 (B) 100\u03a9 (C) 50\u03a9 (D) 10\u03a9\nA: Let's think step by step. In lap winding, effectively two resistors are connected in parallel, so the actual resistance of each pair is 1 Ohm. Since we have 50 pairs, we get a total resistance of 50 Ohms. The answer is (C).\n\n", "elementary_mathematics": "The following are multiple choice questions (with answers) about elementary mathematics.\n\nQ: Olivia used the rule \"Add 11\" to create the number pattern shown below. 10, 21, 32, 43, 54. Which statement about the number pattern is true?\n(A) The 10th number in the pattern will be an even number.\n(B) The number pattern will never have two even numbers next to each other.\n(C) The next two numbers in the pattern will be an even number then an odd number.\n(D) If the number pattern started with an odd number then the pattern would have only odd numbers in it.\nA: Let's think step by step. Choice A is incorrect because every even-numbered term in the pattern is odd, and 10 is an even number. Choice B is correct, because adding an odd number (in this case 11) to an odd number produces an even number, and adding an odd number to an even number produces an odd number. Thus the terms in the pattern will alternate between odd and even, so there will never be two even numbers next to each other. Choice C is incorrect because the last term in the example is even (54), and we know that the terms will alternate between even and odd. Choice D is incorrect because the terms in the pattern will alternate between odd and even, regardless of the value of the first term. The answer is (B).\n\nQ: The population of the city where Michelle was born is 145,826. What is the value of the 5 in the number 145,826?\n(A) 5 thousands\n(B) 5 hundreds\n(C) 5 tens\n(D) 5 ones\nA: Let's think step by step. Choice A is correct, because there are three digits following the 5, so\nthe 5 is in the thousands place. Thus the other choices are incorrect. The answer is (A).\n\nQ: A store sells 107 different colors of paint. They have 25 cans of each color in storage. The number of cans of paint the store has in storage can be found using the expression below. 107 \u00d7 25. How many cans of paint does the store have in storage?\n(A) 749\n(B) 2,675\n(C) 2,945\n(D) 4,250\nA: Let's think step by step. We can calculate 107 x 25 = (100 x 25) + (7 x 25) = 2500 + 175 = 2675. The answer is (B).\n\nQ: A total of 30 players will play basketball at a park. There will be exactly 5 players on each team. Which statement correctly explains how to find the number of teams needed?\n(A) Add 5 to 30 to find 35 teams.\n(B) Divide 30 by 5 to find 6 teams.\n(C) Multiply 30 and 5 to find 150 teams.\n(D) Subtract 5 from 30 to find 25 teams.\nA: Let's think step by step. We want to find the number of teams. We know that there are 5 players/team, and 30 players. Thus to get the number of teams we divide players by players/team, so 30 players / 5 players/team = 6 teams. The answer is (B).\n\nQ: Which expression is equivalent to 5 x 9?\n(A) (5 x 4) x (6 x 5)\n(B) (5 x 5) + (5 x 4)\n(C) (5 x 5) + (5 x 9)\n(D) (5 x 9) x (6 x 9)\nA: Let's think step by step. We know that 9 = (5 + 4), so 5 x 9 = 5 x (5 + 4) = (5 x 5) + (5 x 4). The answer is (B).\n\n", "formal_logic": "The following are multiple choice questions (with answers) about formal logic.\n\nQ: Which of the given formulas of PL is the best symbolization of the following sentence?\nTurtles live long lives and are happy creatures, unless they are injured.\n(A) (L \u2022 H) \u2261 I (B) (L \u2022 H) \u2228 I (C) L \u2022 (H \u2228 I) (D) L \u2022 (H \u2283 R).\nA: Let's think step by step. We refer to Wikipedia articles on formal logic for help. Let\u2019s solve this step by step. Let \u201cL\u201d denote \u201cliving long\u201d, H \u201cbeing happy\u201d, and \u201cI\u201d \u201cbeing injured\u201d. Now, consider each choice:\n(A) means (living long AND being happy) is equivalent to (being injured). \n(B) means (living long AND being happy) OR (being injured). \n(C) means (living long) AND (being happy OR being injured). \n(D) means (living long) AND (being happy implies being R), but what R denotes is not clear.\nObviously, (B) is the best symbolization of the original sentence. The answer is (B).\n\nQ: Select the best translation into predicate logic.George borrows Hector's lawnmower. (g: George; h: Hector; l: Hector's lawnmower; Bxyx: x borrows y from z).\n(A) Blgh (B) Bhlg (C) Bglh (D) Bghl\nA: Let's think step by step. We refer to Wikipedia articles on formal logic for help. Let\u2019s solve this step by step. We are told that \u201cBxyx\u201d means \u201cx borrows y from z\u201d. We can rewrite \u201cGeorge borrows Hector's lawnmower\u201d as \u201cGeorge borrows a lawnmower from Hector\u201d, which can then be translated into predicate logic as \u201cBglh\u201d. The answer \u201cBglh\u201d appears in (C); therefore, (C) must be the correct answer. The answer is (C).\n\nQ: \nSelect the best English interpretation of the given arguments in predicate logic.\nDm\n(\u2200x)(Wx \u2283 ~Dx). \n(\u2200x)Wx \u2228 Ag\t/ (\u2203x)Ax\n(A) Marina is a dancer. Some weaklings are not dancers. Either everything is a weakling or Georgia plays volleyball. So something plays volleyball. (B) Marina is a dancer. No weakling is a dancer. Everything is either a weakling or plays volleyball. So something plays volleyball. (C) Marina is a dancer. Some weaklings are not dancers. Everything is either a weakling or plays volleyball. So something plays volleyball. (D) Marina is a dancer. No weakling is a dancer. Either everything is a weakling or Georgia plays volleyball. So something plays volleyball.\nA: Let's think step by step. We refer to Wikipedia articles on formal logic for help. Let\u2019s solve this step by step. Let \u201cD\u201d denote \u201cbeing a dancer\u201d, \u201cm\u201d denote \u201cMaria\u201d, \u201cg\u201d denote \u201cGeorgia\u201d, \u201cW\u201d denote \u201cweakling\u201d, \u201cA\u201d denote \u201cplaying volleyball\u201d. Then, we have the following:\n1. Dm \u2192 Maria is a dance.\n2. (\u2200x)(Wx \u2283 ~Dx). \u2192 For all x, if x is a weakling, then x is not a dancer. In other words, no weakling is a dancer.\n3. (\u2200x)Wx \u2228 Ag\t/ (\u2203x)Ax \u2192 For all x, x is a weakling or Georgia plays volleyball. So there exists an x that plays volleyball. \nOptions (A) and (C) do claim that some weaklings are not dancers, but the second argument strongly states that no weakling is a dancer. Thus, we can eliminate them. Option (B) omits the important detail about Georgia playing volleyball. Option (D) has all the details presented in the arguments and is the best English interpretation of the arguments. The answer is (D).\n\nQ: Select the best translation into predicate logic: No people drive on Mars.\n(A) ~Pd (B) (\u2200x)(Px \u2228 ~Dx) (C) (\u2200x)(Px \u2283 ~Dx) (D) ~Dp\nA: Let's think step by step. We refer to Wikipedia articles on formal logic for help. Let\u2019s solve this step by step. Let \u201cP\u201d denote \u201cbeing on Mars\u201d and \u201cD\u201d denote \u201cdriving on Mars\u201d. Then let\u2019s consider each option:\nOption (A): ~Pd \u2192 d is not on Mars.\nOption (B): (\u2200x)(Px \u2228 ~Dx) \u2192 For all x, x is on Mars and x do not drive on Mars.\nOption (C): (\u2200x)(Px \u2283 ~Dx) \u2192 For all x, x is on Mars implies that x do not drive on Mars.\nOption (D): ~Dp: \u2192 p do not drive on Mars.\nOf all these options, Option (C) appears to be the best and most meaningful interpretation of the argument \u201cNo people drive on Mars.\u201d The answer is (C).\n\n", "global_facts": "The following are multiple choice questions (with answers) about global facts.\n\nQ: As of 2017, how many of the world\u2019s 1-year-old children today have been vaccinated against some disease? *\n(A) 80% (B) 60% (C) 40% (D) 20%\nA: Let's think step by step. We refer to Wikipedia articles on global facts for help. According to data published by the World Health Organization, the nummber of 1-year-old children vaccinated in 2017 exceeds 80%. The answer is (A).\n\nQ: As of 2019, about what percentage of Americans agree that the state is run for the benefit of all the people?\n(A) 31% (B) 46% (C) 61% (D) 76%\nA: Let's think step by step. We refer to Wikipedia articles on global facts for help. In 2019, about 46% percentage of Americans agree that the state is run for the benefit of all the people. The answer is (B).\n\nQ: As of 2019, about what percentage of Russians say it is very important to have free media in our country without government/state censorship?\n(A) 38% (B) 53% (C) 68% (D) 83%\nA: Let's think step by step. We refer to Wikipedia articles on global facts for help. As of 2019, about 38% of Russians say it is very important to have free media in our country. The answer is (A).\n\nQ: As of 2015, since 1990 forests have ____ in Europe and have ____ in Africa and the Americas.\n(A) increased, increased (B) increased, decreased (C) decreased, increased (D) decreased, decreased\nA: Let's think step by step. We refer to Wikipedia articles on global facts for help. As of 2015, since 1990 forests have increased in Europe and have decreased in Africa and the Americas. The answer is (B).\n\nQ: Which of the following pairs of statements are both true (as of 2019)?\n(A) People tend to be optimistic about their own future and the future of their nation or the world. (B) People tend to be optimistic about their own future but pessimistic about the future of their nation or the world. (C) People tend to be pessimistic about their own future but optimistic about the future of their nation or the world. (D) People tend to be pessimistic about their own future and the future of their nation or the world.\nA: Let's think step by step. We refer to Wikipedia articles on global facts for help. As of 2019, most people tend to be optimistic about their own future but pessimistic about the future of their nation or the world. The answer is (B).\n\n", "high_school_biology": "The following are multiple choice questions (with answers) about high school biology.\n\nQ: In animal cells, which of the following represents the most likely pathway that a secretory protein takes as it is synthesized in a cell?\n(A) Plasma membrane\u2013Golgi apparatus\u2013ribosome\u2013secretory vesicle\u2013rough ER (B) Ribosome\u2013Golgi apparatus\u2013rough ER\u2013secretory vesicle\u2013plasma membrane (C) Plasma membrane\u2013Golgi apparatus\u2013ribosome\u2013secretory vesicle\u2013rough ER (D) Ribosome\u2013rough ER\u2013Golgi apparatus\u2013secretory vesicle\u2013plasma membrane\nA: Let's think step by step. Protein synthesis starts at the ribosome, so we can eliminate (A) and (C). The ribosome is often in the endoplasmic reticulum and moves from there to the Golgi apparatus, where it is modified and packaged into a vesicle. The vesicle then floats to the plasma membrane and is secreted. The answer is (D).\n\nQ: A mutation in a bacterial enzyme changed a previously polar amino acid into a nonpolar amino acid. This amino acid was located at a site distant from the enzyme\u2019s active site. How might this mutation alter the enzyme\u2019s substrate specificity?\n(A) By changing the enzyme\u2019s pH optimum (B) By changing the enzyme\u2019s location in the cell (C) By changing the shape of the protein (D) An amino acid change away from the active site cannot alter the enzyme\u2019s substrate specificity.\nA: Let's think step by step. A change in an amino acid leads to a change in the primary structure of the protein. A change in the primary structure may lead to a change in the secondary and the tertiary structure of the protein. A change in the tertiary structure means a change in the shape of the protein, so (C) has to be correct. Since the change does not affect the active site of the enzyme, we do not expect the activity of the enzyme to be affected. The answer is (C).\n\nQ: Which of the following is not a way to form recombinant DNA?\n(A) Translation (B) Conjugation (C) Specialized transduction (D) Transformation\nA: Let's think step by step. The introduction of foreign DNA or RNA into bacteria or eukaryotic cells is a common technique in molecular biology and scientific research. There are multiple ways foreign DNA can be introduced into cells including transformation, transduction, conjugation, and transfection. In contrast, (A) is not a way to form DNA: during translation the ribosomes synthesize proteins from RNA. The answer is (A).\n\nQ: Homologous structures are often cited as evidence for the process of natural selection. All of the following are examples of homologous structures EXCEPT\n(A) the wings of a bird and the wings of a bat (B) the flippers of a whale and the arms of a man (C) the pectoral fins of a porpoise and the flippers of a seal (D) the forelegs of an insect and the forelimbs of a dog\nA: Let's think step by step. \u200b\u200bHomologous structures are similar physical features in organisms that share a common ancestor \u200b\u200bbut different functions. Comparisons (B) and (C) are clearly homologous because they share a common ancestor and the structures serve different purposes. Bat wings and birg wings are also homologous, while they are both wings, the forelimbs serve different purposes. Insects and dogs are very far ancestors since one is vertebrate while the other is invertebrate and the forelimbs serve the same purpose, so they are not homologous. The answer is (D).\n\nQ: Which of the following is not known to be involved in the control of cell division?\n(A) Cyclins (B) Protein kinases (C) Checkpoints (D) Fibroblast cells\nA: Let's think step by step. Normal cells move through the cell cycle in a regulated way. At the checkpoint stage, they use information about their own internal state and cues from the environment around them to decide whether to proceed with cell division. Cues like these act by changing the activity of core cell cycle regulators inside the cell. The most common regulators are cyclins and cyclin-dependent kinases. Fibroblast cells do not play any role in cell division. The answer is (D).\n\n", "high_school_chemistry": "The following are multiple choice questions (with answers) about high school chemistry.\n\nQ: Which of the following is considered an acid anhydride?\n(A) HCl (B) H2SO3 (C) SO2 (D) Al(NO3)3\nA: Let's think step by step. An acid anhydride is a compound that is derived by removing water from an acid. The chemical formula for water is H2O, which means that we need to determine which of these options, when combined with H2O, forms an acid. SO2, or Sulfur dioxide, when combined with H2O, makes H2SO4, or sulfuric acid. The answer is (C).\n\nQ: Which of the following is expected to be a polar molecule?\n(A) PCl4F (B) BF3 (C) CO2 (D) Si(CH3)4\nA: Let's think step by step. A polar molecule is one that has a slightly positive charge on one end of the molecule and a slightly negative charge on the other end. Boron trifluoride (BF3) has Boron as the center atom and three fluorine atoms attached to it; it is trigonal planar and symmetric, so it is nonpolar. Carbon Dioxide (CO2) has Carbon as the central atom with double bonds to two Oxygen atoms - this is also symmetrical and therefore nonpolar. The same is the case for tetramethyl silane (SI(CH3)4), which is a Silicon atom surrounded by four methyl groups. The structure of PCL4F is that Phosphorus is the central atom, attached to four chlorines and one fluorine atom. This is asymmetrical, and therefore has a net dipole and is expected to be a polar molecule. The answer is (A).\n\nQ: From the solubility rules, which of the following is true?\n(A) All chlorides, bromides, and iodides are soluble (B) All sulfates are soluble (C) All hydroxides are soluble (D) All ammonium-containing compounds are soluble\nA: Let's think step by step. The chlorides, bromides, and iodides of lead, silver, and mercury are not soluble in water. This rules out (A). The sulfates of lead, barium, and calcium are not soluble in water, which rules out (B). The hydroxides of any metal besides sodium, potassium, ammonium, calcium, and barium are insoluble. This rules out (C). Typically ammonium ions indicate a soluble ionic substance. The answer is (D).\n\nQ: A new compound is synthesized and found to be a monoprotic acid with a molar mass of 248 g/mol. When 0.0050 mol of this acid are dissolved in 0.500 L of water, the pH is measured as 3.89. What is the pKa of this acid?\n(A) 3.89 (B) 7.78 (C) 5.78 (D) 2.33\nA: Let's think step by step. Recall that $[A] = [H^{+}]$. Here, this is equal to $$10^{-3.89}$. Then we have $K_{a} = $\nrac{[H^{+}][A^{-}]}{[HA]} = \nrac{10^{-3.89} \\cdot 10^{-3.89}}{10^{-2}}. The resulting exponent is $-3.89 + (-3.89) - (-2) = 5.78$, therefore $K_a = 10^{-5.78}$. The $pK_a$ is the negative log of $K_a$, which is equal to $5.78$. The answer is (C).\n\nQ: A solution contains 2.00 mole of acetic acid, CH3COOH, and 1.00 mole of calcium acetate, Ca(CH3COO)2. The solution is able to resist the addition of a small amount of strong acid or strong base with only minor changes in the pH of the solution. Larger quantities of strong acid or strong base can cause a significant change in pH. How many moles of nitric acid, HNO3, may be added before the pH begins to change significantly?\n(A) 0.500 mole (B) 1.00 mole (C) 2.00 mole (D) 3.00 mole\nA: Let's think step by step. We would like to compute the buffer capacity of this solution. First we write the equation for the ionization of the weak acid, in this case of acetic acid. $CH_{3}COOH (aq) + H_{2}O \nightarrow H_{3}O^{+} + CH3COO^{-}$. The conjugate base is therefore the acetate ion. The added strong acid, Nitric acid, will react with the conjugate base. Therefore the maximum amount of acid that can be added will be equal to the amount of acetate ion, or 2 moles. The answer is (C).\n\n", "high_school_computer_science": "The following are multiple choice questions (with answers) about high school computer science.\n\nQ: Which of the following is an example of the use of a device on the Internet of Things (IoT) ?\n(A) A car alerts a driver that it is about to hit an object. (B) A hiker uses a G P S watch to keep track of her position. (C) A refrigerator orders milk from an online delivery service when the milk in the refrigerator is almost gone. (D) A runner uses a watch with optical sensors to monitor his heart rate.\nA: Let's think step by step. The term Internet of Things (IoT) refers to common devices which are connected to the internet, enabling new functionality. Choice A is incorrect because it does not describe an internet connected device. In choice B, the watch is only described as having GPS functionality but no internet connectivity. Choice C describes a common device (a refrigerator) which has internet connectivity enabling new functionality (online ordering). Choice D does not mention internet connectivity for the watch, only optical sensors. The answer is (C).\n\nQ: Many Web browsers allow users to open anonymous windows. During a browsing session in an anonymous window, the browser does not record a browsing history or a list of downloaded files. When the anonymous window is exited, cookies created during the session are deleted. Which of the following statements about browsing sessions in an anonymous window is true?\n(A) The activities of a user browsing in an anonymous window will not be visible to people who monitor the user's network, such as the system administrator. (B) Items placed in a Web store's shopping cart for future purchase during the anonymous browsing session will not be saved on the user's computer. (C) A user will not be able to log in to e-mail or social media accounts during the anonymous browsing session. (D) A user browsing in an anonymous window will be protected from viruses launched from any web sites visited or files downloaded.\nA: Let's think step by step. Choice A is incorrect as it only describes network traffic, which an anonymous browser does not change. Choice B is correct as it correctly describes how an anonymous browser will prevent saving data on the user\u2019s computer after the session is ended. Choice C is incorrect because an anonymous browser will not prevent logging in to email or social media accounts. Choice D is incorrect because an anonymous browser in itself performs no virus protection. The answer is (B).\n\nQ: In the program below, the initial value of X is 5 and the initial value of Y is 10.\nIF (X < 0){\n DISPLAY (\"Foxtrot\")\n} ELSE {\n IF (X > Y){\n DISPLAY (\"Hotel\")\n } ELSE {\n IF (Y > 0){\n DISPLAY (\"November\")\n } ELSE {\n DISPLAY (\"Yankee\")\n }\n }\n}\nWhat is displayed as a result of running the program?\n(A) Foxtrot (B) Hotel (C) November (D) Yankee\nA: Let's think step by step. Because X has the value 5, the first conditional IF (X < 0) is false, so we move to the first ELSE clause. Because X is 5 and Y is 10, the second conditional IF (X > Y) is false, so we move to the following ELSE clause. Since Y is 10, the conditional IF (Y > 0) is true, so the command DISPLAY (\"November\") is executed. The answer is (C).\n\nQ: What is the output of \"abc\"[::-1] in Python 3?\n(A) Error (B) abc (C) cba (D) c\nA: Let's think step by step. We know that the slicing operator [::-1] takes all of the elements in the string in reverse order, so we reverse the order of the string \"abc\", resulting in \"cba\". The answer is (C).\n\nQ: A list of numbers has n elements, indexed from 1 to n. The following algorithm is intended to display the number of elements in the list that have a value greater than 100. The algorithm uses the variables count and position. Steps 3 and 4 are missing.\n Step 1: Set count to 0 and position to 1.\n Step 2: If the value of the element at index position is greater than 100, increase the value of count by 1.\n Step 3: (missing step)\n Step 4: (missing step)\n Step 5: Display the value of count.\nWhich of the following could be used to replace steps 3 and 4 so that the algorithm works as intended?\n(A) Step 3: Increase the value of position by 1.\n Step 4: Repeat steps 2 and 3 until the value of count is greater than 100.\n(B) Step 3: Increase the value of position by 1.\n Step 4: Repeat steps 2 and 3 until the value of position is greater than n.\n(C) Step 3: Repeat step 2 until the value of count is greater than 100.\n Step 4: Increase the value of position by 1.\n(D) Step 3: Repeat step 2 until the value of position is greater than n.\n Step 4: Increase the value of count by 1.\nA: Let's think step by step. Choice A is incorrect, because its Step 4 has an incorrect termination condition, stopping when count is greater than 100. We need to stop after inspecting all elements in the list. Choice B is correct because it correctly increments both count and position, and correctly repeats these steps and terminates when all elements in the list have been inspected. Choice C is incorrect because it incorrectly increments the variable count until its value is greater than 100, regardless of the elements in the list. Choice D is incorrect because its step 3 does not increment the value of position, so it will repeat forever. The answer is (B).\n\n", "high_school_european_history": "The following are multiple choice questions (with answers) about high school european history.\n\nQ: This question refers to the following information.\nAlbeit the king's Majesty justly and rightfully is and ought to be the supreme head of the Church of England, and so is recognized by the clergy of this realm in their convocations, yet nevertheless, for corroboration and confirmation thereof, and for increase of virtue in Christ's religion within this realm of England, and to repress and extirpate all errors, heresies, and other enormities and abuses heretofore used in the same, be it enacted, by authority of this present Parliament, that the king, our sovereign lord, his heirs and successors, kings of this realm, shall be taken, accepted, and reputed the only supreme head in earth of the Church of England, called Anglicans Ecclesia; and shall have and enjoy, annexed and united to the imperial crown of this realm, as well the title and style thereof, as all honors, dignities, preeminences, jurisdictions, privileges, authorities, immunities, profits, and commodities to the said dignity of the supreme head of the same Church belonging and appertaining; and that our said sovereign lord, his heirs and successors, kings of this realm, shall have full power and authority from time to time to visit, repress, redress, record, order, correct, restrain, and amend all such errors, heresies, abuses, offenses, contempts, and enormities, whatsoever they be, which by any manner of spiritual authority or jurisdiction ought or may lawfully be reformed, repressed, ordered, redressed, corrected, restrained, or amended, most to the pleasure of Almighty God, the increase of virtue in Christ's religion, and for the conservation of the peace, unity, and tranquility of this realm; any usage, foreign land, foreign authority, prescription, or any other thing or things to the contrary hereof notwithstanding.\nEnglish Parliament, Act of Supremacy, 1534\nFrom the passage, one may infer that the English Parliament wished to argue that the Act of Supremacy would\n(A) give the English king a new position of authority (B) give the position of head of the Church of England to Henry VIII alone and exclude his heirs (C) establish Calvinism as the one true theology in England (D) end various forms of corruption plaguing the Church in England\nA: Let's think step by step. We refer to Wikipedia articles on european history for help. The Act of Supremacy states that it grants authority to the king \"to repress and extirpate all errors, heresies, and other enormities and abuses\", referring to the corruption in the Church of England. The answer is (D).\n\nQ: This question refers to the following information.\nRead the following excerpt.\nThe revolutionary seed had penetrated into every country and spread more or less. It was greatly developed under the r\u00e9gime of the military despotism of Bonaparte. His conquests displaced a number of laws, institutions, and customs; broke through bonds sacred among all nations, strong enough to resist time itself; which is more than can be said of certain benefits conferred by these innovators.\nThe monarchs will fulfil the duties imposed upon them by Him who, by entrusting them with power, has charged them to watch over the maintenance of justice, and the rights of all, to avoid the paths of error, and tread firmly in the way of truth. Placed beyond the passions which agitate society, it is in days of trial chiefly that they are called upon to despoil realities of their false appearances, and to show themselves as they are, fathers invested with the authority belonging by right to the heads of families, to prove that, in days of mourning, they know how to be just, wise, and therefore strong, and that they will not abandon the people whom they ought to govern to be the sport of factions, to error and its consequences, which must involve the loss of society.\nUnion between the monarchs is the basis of the policy which must now be followed to save society from total ruin. . . .\nLet them not confound concessions made to parties with the good they ought to do for their people, in modifying, according to their recognized needs, such branches of the administration as require it.\nLet them be just, but strong; beneficent, but strict.\nLet them maintain religious principles in all their purity, and not allow the faith to be attacked and morality interpreted according to the social contract or the visions of foolish sectarians.\nLet them suppress Secret Societies; that gangrene of society.\n\u2014Klemens von Metternich, Political Confession of Faith, 1820\nWhich of the following was the greatest cause of the fears expressed by Metternich in the document above?\n(A) The ideas of personal liberty and nationalism conceived during the Enlightenment resulted in radical revolutions that could spread throughout Europe. (B) The conquest of Europe by Napoleon led to the creation of new factions and shifted the European balance of power. (C) The power of monarchs had grown to the point where it needed to be checked by other powers within each nation or domination of civilians would occur. (D) The rising and falling economic cycle of the newly emerging capitalist economy could lead to civilian unrest that must be suppressed.\nA: Let's think step by step. We refer to Wikipedia articles on european history for help. The fears of revolution in early 19th century Europe expressed by Klemens von Metternich, a conservative Austrian statesman, were a direct result of the age of Enlightenment, a period of European history where the absolute power of the monarchy was challenged with ideas of individual liberty and nationalism, leading to the French revolution and its effects all over Europe. The answer is (A).\n\nQ: This question refers to the following information.\nThe excerpts below are from the Navigation Acts of 1651.\n[A]fter the first day of December, one thousand six hundred fifty and one, and from thence forwards, no goods or commodities whatsoever of the growth, production or manufacture of Asia, Africa or America, or of any part thereof; or of any islands belonging to them, or which are described or laid down in the usual maps or cards of those places, as well of the English plantations as others, shall be imported or brought into this Commonwealth of England, or into Ireland, or any other lands, islands, plantations, or territories to this Commonwealth belonging, or in their possession, in any other ship or ships, vessel or vessels whatsoever, but only in such as do truly and without fraud belong only to the people of this Commonwealth, or the plantations thereof, as the proprietors or right owners thereof; and whereof the master and mariners are also of the people of this Commonwealth, under the penalty of the forfeiture and loss of all the goods that shall be imported contrary to this act, , , ,\n[N]o goods or commodities of the growth, production, or manufacture of Europe, or of any part thereof, shall after the first day of December, one thousand six hundred fifty and one, be imported or brought into this Commonwealth of England, or any other lands or territories to this Commonwealth belonging, or in their possession, in any ship or ships, vessel or vessels whatsoever, but in such as do truly and without fraud belong only to the people of this Commonwealth, and in no other, except only such foreign ships and vessels as do truly and properly belong to the people of that country or place, of which the said goods are the growth, production or manufacture.\nWhich of the following best describes the outcome of the Navigation Acts of 1651?\n(A) They served as a catalyst for the growth of English shipping and overseas trade, but did little to limit the prospects of the Dutch in the seventeenth century. (B) They brought about almost immediate hardships for the Dutch economy as their dominance of overseas trade quickly ended. (C) They were rescinded during the restoration of the Stuarts as they sought normal diplomatic relations with the Dutch so not as to need Parliament's financial support for war. (D) They led to nearly a century of recurrent war between England and the Netherlands, which would not end until after American independence.\nA: Let's think step by step. We refer to Wikipedia articles on european history for help. The Navigation Acts of 1651 helped English shipping by restricting the ability of ships from other European countries, especially the Dutch, to transport goods from colonies in Asia and Africa into England. The answer is (A).\n\nQ: This question refers to the following information.\nIn Russia there was nothing going on well, and [Souvarine] was in despair over the news he had received. His old companions were all turning to the politicians; the famous Nihilists who made Europe tremble-sons of village priests, of the lower middle class, of tradesmen-could not rise above the idea of national liberation, and seemed to believe that the world would be delivered-when they had killed their despot&\u2026\n\"Foolery! They'll never get out of it with their foolery.\"\nThen, lowering his voice still more, in a few bitter words he described his old dream of fraternity. He had renounced his rank and his fortune; he had gone among workmen, only in the hope of seeing at last the foundation of a new society of labour in common. All the sous in his pockets had long gone to the urchins of the settlement; he had been as tender as a brother with the colliers, smiling at their suspicion, winning them over by his quiet workmanlike ways and his dislike of chattering. But decidedly the fusion had not taken place.\nHis voice changed, his eyes grew bright, he fixed them on \u00e9tienne, directly addressing him:\n\"Now, do you understand that? These hatworkers at Marseilles who have won the great lottery prize of a hundred thousand francs have gone off at once and invested it, declaring that they are going to live without doing anything! Yes, that is your idea, all of you French workmen; you want to unearth a treasure in order to devour it alone afterwards in some lazy, selfish corner. You may cry out as much as you like against the rich, you haven't got courage enough to give back to the poor the money that luck brings you. You will never be worthy of happiness as long as you own anything, and your hatred of the bourgeois proceeds solely from an angry desire to be bourgeois yourselves in their place.\"\n\u00e9mile Zola, French writer, Germinal, 1885\nThe passage displays the direct concern for the welfare of the working classes that was typically a part of which movement?\n(A) Capitalist (B) Scientific (C) Communist (D) Existentialist\nA: Let's think step by step. We refer to Wikipedia articles on european history for help. The modern Communist movement aims to establish a classless society based on communal ownership and distribution of property and means of production, thereby especially benefiting the working classes. The answer is (C).\n\nQ: This question refers to the following information.\nThe following excerpt is from a pamphlet.\nYou will do me the justice to remember, that I have always strenuously supported the Right of every man to his own opinion, however different that opinion might be to mine. He who denies to another this right, makes a slave of himself to his present opinion, because he precludes himself the right of changing it.\nThe most formidable weapon against errors of every kind is Reason. I have never used any other, and I trust I never shall.\nThe circumstance that has now taken place in France of the total abolition of the whole national order of priesthood, and of everything appertaining to compulsive systems of religion, and compulsive articles of faith, has not only precipitated my intention, but rendered a work of this kind exceedingly necessary, lest in the general wreck of superstition, of false systems of government, and false theology, we lose sight of morality, of humanity, and of the theology that is true.\nI believe in one God, and no more; and I hope for happiness beyond this life.\nI believe in the equality of man; and I believe that religious duties consist in doing justice, loving mercy, and endeavoring to make our fellow-creatures happy.\nI do not believe in the creed professed by the Jewish church, by the Roman church, by the Greek church, by the Turkish church, by the Protestant church, nor by any church that I know of. My own mind is my own church.\nAll national institutions of churches, whether Jewish, Christian or Turkish, appear to me no other than human inventions, set up to terrify and enslave mankind, and monopolize power and profit.\nI do not mean by this declaration to condemn those who believe otherwise; they have the same right to their belief as I have to mine.\n\u2014Thomas Paine, The Age of Reason, 1794\u20131795\nWhich of the following Enlightenment philosophes designed a system of checks and balances for government to avoid abuses of power?\n(A) Jean Jacques Rousseau (B) Baron Montesquieu (C) Mary Wollstonecraft (D) Adam Smith\nA: Let's think step by step. We refer to Wikipedia articles on european history for help. Baron Montesquieu was a 18th centrury French philsopher who wrote extensively against the monoplization of power and advocated for a system of checks and balances in government to prevent the rise of despotism. The answer is (B).\n\n", "high_school_geography": "The following are multiple choice questions (with answers) about high school geography.\n\nQ: Which one of the following items is an example of nonmaterial culture?\n(A) Dove soap (B) Dove candy bar (C) Dove symbol (D) A dove (bird).\nA: Let's think step by step. We refer to Wikipedia articles on geography for help. Nonmaterial culture consists of cultural ideas, beliefs or symbols that are not physical objects. The answer is (C).\n\nQ: During the third stage of the demographic transition model, which of the following is true?\n(A) Birth rates increase and population growth rate is less rapid. (B) Birth rates decline and population growth rate is less rapid. (C) Birth rates increase and population growth rate increases. (D) Birth rates decrease and population growth rate increases.\nA: Let's think step by step. We refer to Wikipedia articles on geography for help. The demographic transition model models the five different stages of population growth as a country goes through economic development, where the third stage refers to a period of declining birth rates and lower population growth. The answer is (B).\n\nQ: The practice of hiring a foreign third-party service provider to run an operation is called\n(A) outsourcing. (B) offshoring. (C) maquiladoras. (D) locational interdependence.\nA: Let's think step by step. We refer to Wikipedia articles on geography for help. \"Offshoring\" literally means to move or base some of the activities or processes of a company to a foreign country. The answer is (B).\n\nQ: Which of the following statements is NOT accurate regarding the services provided by local governments in the United States?\n(A) Duplication of efforts occurs often. (B) Social problems of the central city spill over into the surrounding residential suburbs. (C) Inefficiency in providing services occurs often. (D) One neighborhood's efforts to reduce pollution are always supported by neighboring communities.\nA: Let's think step by step. We refer to Wikipedia articles on geography for help. There may be economic, social or political reasons for two neighboring communities and their local governments not agreeing to pollution reduction efforts initiated by one of them. The answer is (D).\n\nQ: The rate of natural increase of a population is found by subtracting the\n(A) crude death rate from the crude birth date. (B) crude birth rate from the crude death rate. (C) doubling time from the crude birth rate. (D) fertility rate from the crude death rate.\nA: Let's think step by step. We refer to Wikipedia articles on geography for help. The difference between number of births and deaths gives the population increase at any given time. The answer is (A).\n\n", "high_school_government_and_politics": "The following are multiple choice questions (with answers) about high school government and politics.\n\nQ: Which of the following best states an argument made by James Madison in The Federalist number 10?\n(A) Honest politicians can prevent factions from developing. (B) Factions are more likely to occur in large republics than in small ones. (C) The negative effects of factionalism can be reduced by a republican government. (D) Free elections are the people's best defense against factionalism.\nA: Let's think step by step. We refer to Wikipedia articles on government and politics for help. In the Federalist number 10, James Madison advocated for a representative republican form of government to guard against factionalism. The answer is (C).\n\nQ: The term \"budget deficit\" refers to the\n(A) annual increase in federal spending on the military (B) amount of interest on the national debt (C) difference between the initial budget proposals made by the president and Congress (D) amount the government spends in excess of its revenues\nA: Let's think step by step. We refer to Wikipedia articles on government and politics for help. When the goverment spends more than it earns, their difference is the budget deficit. The answer is (D).\n\nQ: Which of the following statements about cabinet departments is FALSE?\n(A) They are established by the legislative branch. (B) Their members often don't have much influence over presidential decisions. (C) They cannot all be run by leaders who belong to the same political party the president does. (D) Not every federal agency is a cabinet department.\nA: Let's think step by step. We refer to Wikipedia articles on government and politics for help. There is no law stipulating that some cabinet department leaders have to belong to a political party different from that of the president. The answer is (C).\n\nQ: Which of the following cases established the precedent that a defendant must be informed of the right to remain silent, the right to a lawyer, and protection from self-incrimination?\n(A) Weeks v. United States (B) Betts v. Brady (C) Mapp v. Ohio (D) Miranda v. Arizona\nA: Let's think step by step. We refer to Wikipedia articles on government and politics for help. In the landmark Miranda v. Arizona in 1966, the US Supreme Court, based on the Fifth and Sixth Amendment of the US Constitution, guaranteed a defendant's right to an attorney and protection from self-incrimination. The answer is (D).\n\nQ: Uncertainty over the limits to presidential power is caused primarily by the fact that\n(A) the constitutional definition of those powers is broad and unspecific (B) most people agree that the Constitution places too many limits on presidential power (C) the Supreme Court consistently refuses to rule on cases concerning presidential powers (D) constitutional amendments have greatly increased presidential powers\nA: Let's think step by step. We refer to Wikipedia articles on government and politics for help. The US Constitution is not very specific about the powers of the president, leading to uncertainty over its limits. The answer is (A).\n\n", "high_school_macroeconomics": "The following are multiple choice questions (with answers) about high school macroeconomics.\n\nQ: Which of the following policies best describes supply-side fiscal policy?\n(A) An increase in the money supply (B) Increased government spending (C) Lower taxes on research and development of new technology (D) Higher taxes on household income\nA: Let's think step by step. We refer to Wikipedia articles on macroeconomics for help. Supply-side fiscal policy stimulates the economy by encouraging more production of goods and services through reduction in taxes and deregulation. The answer is (C).\n\nQ: The short-run Phillips curve indicates a\n(A) direct relation between unemployment and inflation (B) direct relation between price and quantity demanded (C) inverse relation between price and quantity demanded (D) inverse relation between unemployment and inflation\nA: Let's think step by step. We refer to Wikipedia articles on macroeconomics for help. The short-run Phillips curve shows that whenever unemployment decreases below a natural level, the inflation starts increasing, and vice-versa. The answer is (D).\n\nQ: Holding all else equal which of the following monetary policies would be used to boost U.S. exports?\n(A) Increasing the discount rate (B) Increasing the reserve ratio (C) Buying government securities (D) Lowering tariffs\nA: Let's think step by step. We refer to Wikipedia articles on macroeconomics for help. Buying government securities leads to reduction in demand for US dollars from foreign buyers, thereby making it cheaper and hence making US exports more attractive. The answer is (C).\n\nQ: A federal deficit occurs when\n(A) exports exceed imports. (B) imports exceed exports. (C) federal tax collections exceed spending. (D) federal spending exceeds federal tax revenues.\nA: Let's think step by step. We refer to Wikipedia articles on macroeconomics for help. A federal deficit occurs when federal spending exceeds federal income which is primarily from tax revenues. The answer is (D).\n\nQ: Which of the following is not included in the U.S. GDP?\n(A) The U.S. military opens a new base in a foreign country with 1000 U.S. personnel. (B) Japanese consumers buy thousands of CDs produced in the United States. (C) An American pop singer performs a sold-out concert in Paris. (D) A French theatrical production tours dozens of American cities.\nA: Let's think step by step. We refer to Wikipedia articles on macroeconomics for help. The economic transactions related to the performance of the American pop-singer in Paris happens entirely outside the U.S. and hence is not included in the GDP numbers. The answer is (C).\n\n", "high_school_mathematics": "The following are multiple choice questions (with answers) about high school mathematics.\n\nQ: Simplify and write the result with a rational denominator: $$\\sqrt{\\sqrt[3]{\\sqrt{\\frac{1}{729}}}}$$\n(A) \\frac{3\\sqrt{3}}{3} (B) \\frac{1}{3} (C) \\sqrt{3} (D) \\frac{\\sqrt{3}}{3}\nA: Let's think step by step. Factoring $729=3^6$ and combining the roots $\\frac{1}{2}\\frac{1}{3}\\frac{1}{2}=\\frac{1}{12}$, we get that $\\sqrt{\\sqrt[3]{\\sqrt{\\frac{1}{729}}}}=\\left(\\frac{1}{3^6}\\right)^{\\frac{1}{12}}=\\frac{1}{3^{\\frac{1}{2}}}=\\frac{3}{\\sqrt{3}}$ The answer is (D).\n\nQ: Five thousand dollars compounded annually at an $x\\%$ interest rate takes six years to double. At the same interest rate, how many years will it take $\\$300$ to grow to $\\$9600$?\n(A) 12 (B) 1 (C) 30 (D) 5\nA: Let's think step by step. To go from $\\$300$ to $\\$9600$, the value must go up by a factor of $9600/300=32=2^5$. Since at this interest rate it takes six years for it to double, it will take $5*6=30$ years to grow to $\\$9600$. The answer is (C).\n\nQ: Ten students take a biology test and receive the following scores: 45, 55, 50, 70, 65, 80, 40, 90, 70, 85. What is the mean of the students\u2019 test scores?\n(A) 55 (B) 60 (C) 62 (D) 65\nA: Let's think step by step. There are 10 students and the sum of their scores is $45 + 55 + 50 + 70 + 65 + 80 + 40 + 90 + 70 + 85 = 650$, the mean is $650/10=65$. The answer is (D).\n\nQ: The variable $x$ varies directly as the square of $y$, and $y$ varies directly as the cube of $z$. If $x$ equals $-16$ when $z$ equals 2, what is the value of $x$ when $z$ equals $\\frac{1}{2}$?\n(A) -1 (B) 16 (C) -\\frac{1}{256} (D) \\frac{1}{16}\nA: Let's think step by step. We know that $x \\propto y^2$ and $y \\propto z^3$, so $x = k z^6$ for some constant $k$. Plugging in for $x=-16$ and $z=2$, the constant value is $k=\\frac{x}{z^6}=\\frac{-16}{64}=-\\frac{1}{4}$. So, when $z=\\frac{1}{2}$, the value of $x$ is $x=kz^6=-\\frac{1}{4}\\frac{1}{2^6}=-\\frac{1}{256}$. The answer is (C).\n\nQ: Joe was in charge of lights for a dance. The red light blinks every two seconds, the yellow light every three seconds, and the blue light every five seconds. If we include the very beginning and very end of the dance, how many times during a seven minute dance will all the lights come on at the same time? (Assume that all three lights blink simultaneously at the very beginning of the dance.)\n(A) 3 (B) 15 (C) 6 (D) 5\nA: Let's think step by step. The least common multiple of 2, 3 and 5 is 30, so during a 7 minute dance, all the three lights will come on at the same time $2*7+1=15$ times. The answer is (B).\n\n", "high_school_microeconomics": "The following are multiple choice questions (with answers) about high school microeconomics.\n\nQ: Which of the following is necessarily a characteristic of oligopoly?\n(A) Free entry into and exit from the market (B) A few large producers (C) One producer of a good with no close substitutes (D) A homogenous product\nA: Let's think step by step. We refer to Wikipedia articles on microeconomics for help. An oligopoly is when a market is dominated by just one or a few number of sellers or producers. To get oligopoly, the market should have high barriers to new entry, and the product has differentiation. The answer is (B).\n\nQ: If the government subsidizes producers in a perfectly competitive market, then\n(A) the demand for the product will increase (B) the demand for the product will decrease (C) the consumer surplus will increase (D) the consumer surplus will decrease\nA: Let's think step by step. We refer to Wikipedia articles on microeconomics for help. (A) and (B) are wrong because the demand curve does not change at all. If the government subsidizes producers, the supply will increase, and thus the consumer surplus also increases. The answer is (C).\n\nQ: Which of the following is true of a price floor?\n(A) The price floor shifts the demand curve to the left. (B) An effective floor creates a shortage of the good. (C) The price floor shifts the supply curve of the good to the right. (D) To be an effective floor, it must be set above the equilibrium price.\nA: Let's think step by step. We refer to Wikipedia articles on microeconomics for help. Price floor does not shift the demand or shift curve. An effective price floor should be set above the equilibrium price, otherwise the market bears and the floor does not have effective effect. The answer is (D).\n\nQ: The concentration ratio for a monopoly is\n(A) 0 (B) 5 (C) 10 (D) 100\nA: Let's think step by step. We refer to Wikipedia articles on microeconomics for help. The concentration ratio is calculated as the sum of market share of a specific number of largest companies. Monopoly means one company or entity controls the entire market, therefore, the concentration ratio is 100 percent. The answer is (D).\n\nQ: In a competitive labor market for housepainters, which of the following would increase the demand for housepainters?\n(A) An effective minimum wage imposed on this labor market. (B) An increase in the price of gallons of paint. (C) An increase in the construction of new houses. (D) An increase in the price of mechanical painters so long as the output effect exceeds the substitution effect.\nA: Let's think step by step. We refer to Wikipedia articles on microeconomics for help. An increase in the construction of new houses means an increase demand of in-house painting, thus increases the demand for housepainters. The answer is (C).\n\n", "high_school_physics": "The following are multiple choice questions (with answers) about high school physics.\n\nQ: A microwave oven is connected to an outlet, 120 V, and draws a current of 2 amps. At what rate is energy being used by the microwave oven?\n(A) 10 W (B) 30 W (C) 60 W (D) 240 W\nA: Let's think step by step. Rate of energy usage is known as power; in an dissipative electrical circuit, power is given by voltage times current. So in our case, the power is 120 V times 2 amps, or 240 W. The answer is (D).\n\nQ: A point charge, Q = +1 mC, is fixed at the origin. How much work is required to move a charge, Q = +8 \u00b5C, from the point (0, 4 meters) to the point (3 meters, 0)?\n(A) 3.5 J (B) 6.0 J (C) 22.5 J (D) 40 J\nA: Let's think step by step. To calculate the work required to move a charge from one location to another in a fixed electric field, it is enough to calculate the potential difference between the two locations. Here, the potential only depends on the distance between the charges; it\u2019s $k q_1 q_2 / r$, where $k$ is Coulomb\u2019s constant. Plugging in values $q_1 = $ 1 mC, $q_2 = 8 \\mu$ C, gives the answer as 5.992 J, which rounds to 6 J. The answer is (B).\n\nQ: Which of the following conditions will ensure that angular momentum is conserved? I. Conservation of linear momentum II. Zero net external force III. Zero net external torque\n(A) I and II only (B) I and III only (C) II and III only (D) III only\nA: Let's think step by step. Torque is defined as the change in angular momentum; if there is zero external torque, angular momentum is conserved. The answer is (D).\n\nQ: A photocell of work function \u03d5 = 2eV is connected to a resistor in series. Light of frequency f = 1 \u00d7 10^15 Hz hits a metal plate of the photocell. If the power of the light is P = 100 W, what is the current through the resistor?\n(A) 2:00 AM (B) 6:00 AM (C) 12:00 AM (D) 24 A\nA: Let's think step by step. The only answer above which has units of current is D, 24 A. The answer is (D).\n\nQ: A pipe full of air is closed at one end. A standing wave is produced in the pipe, causing the pipe to sound a note. Which of the following is a correct statement about the wave\u2019s properties at the closed end of the pipe?\n(A) The pressure is at a node, but the particle displacement is at an antinode. (B) The pressure is at an antinode, but the particle displacement is at a node. (C) The pressure and the particle displacement are both at nodes. (D) The pressure and the particle displacement are both at antinodes.\nA: Let's think step by step. At the closed end of the pipe, the particles cannot have any net displacement because the pipe closure stops them. So the particle displacement is at a node. This closure also causes the pressure to be maximal, i.e. an antinode. The answer is (B).\n\n", "high_school_psychology": "The following are multiple choice questions (with answers) about high school psychology.\n\nQ: Pascale is interested in the processing strategies children use to learn new information. Pascale would best be classified as what type of psychologist?\n(A) sociocultural (B) clinical (C) cognitive (D) behaviorist\nA: Let's think step by step. We refer to Wikipedia articles on psychology for help. Sociocultural psychologist focuses on the effect of societal factors on people. Clinical psychologist focuses on people with mental issues. Cognitive psychologist focuses on how people think and learn, including the processing strategies. Behaviorist focuses more on the environment and experience effect on people. The answer is (C).\n\nQ: According to Caplan's model of consultee-centered case consultation, the consultant is primarily interested in\n(A) identifying the causes and solutions of the client's presenting problems (B) identifying and eliminating the causes of the consultee's difficulties in handling a problem (C) establishing a hierarchy of authority to enable effective decision making (D) presenting a single, well-defined and unambiguous course of action for the consultant to overcome skills deficits\nA: Let's think step by step. We refer to Wikipedia articles on psychology for help. Caplan defines two type of consultation. Client-centered case consultation aims to handle client's problems, while consultee-centered case consultation aims to identify the reason of client's difficulty to solve problems. The answer is (B).\n\nQ: According to the Individuals with Disabilities Education Improvement Act, which of the following must an educational agency do before it changes the educational placement of a student with a disability?\n(A) Give the child a trial period in the new environment (B) Notify the parents in writing (C) Obtain school board approval (D) Obtain parental consent\nA: Let's think step by step. We refer to Wikipedia articles on psychology for help. When the decision to change the educational placement of a student with a disability is made, the educational agency must notify the parents in writing on that date. The answer is (B).\n\nQ: While swimming in the ocean, Ivan is frightened by a dark shadow in the water even before he has the chance to identify what the shadow is. The synaptic connections taking place during this incident of fright are best described by which of the following?\n(A) Messages are sent from the thalamus directly to the amygdala. (B) Messages are sent from the thalamus to the \"what\" and \"where\" pathways. (C) Messages are sent from the parasympathetic nervous system to the cerebral cortex. (D) Messages are sent from the frontal lobes to the pituitary gland.\nA: Let's think step by step. We refer to Wikipedia articles on psychology for help. Our neural system has a mechanism that can respond immediate emotional signal before going to the thought center. In the Ivan's case, messages travel directly from thalamus to amygdala. The answer is (A).\n\nQ: Ani believes that her attitudes and behavior play a central role in what happens to her. Such a belief is likely to be associated with\n(A) a strong superego. (B) low self-esteem. (C) low self-efficacy. (D) an internal locus of control.\nA: Let's think step by step. We refer to Wikipedia articles on psychology for help. People with an external locus of control believes fate and luck play an important role in their lives, while people with an internal locus of control believes they control their lives. The answer is (D).\n\n", "high_school_statistics": "The following are multiple choice questions (with answers) about high school statistics.\n\nQ: A new smartwatch is manufactured in one part of a factory, then secured for shipping in another, independent part of the factory. The weight of the smartwatch has a mean of 62 grams and a standard deviation of 1.0 grams. The weight of the packaging (box, user's guide, bubble wrap, etc.) has a mean of 456 grams and a standard deviation of 6 grams. Together, the distribution of the weight of the smartwatch and its packaging would have the following mean and standard deviation:\n(A) Mean 518 grams; standard deviation 7.0 grams (B) Mean 518 grams; standard deviation 3.5 grams (C) Mean 518 grams; standard deviation 6.1 grams (D) Mean 394 grams; standard deviation 6.1 grams\nA: Let's think step by step. Since the weight of the watch and the weight of the packaging are independent random variables, the mean and variance of their sum is equal to the sum of their individual means and variances. So the mean is 62 + 456 = 518 grams, and the variances is 1.0^2 + 6.0^2 = 37, leading to a standard deviation of 6.1 grams. The answer is (C).\n\nQ: After a frost warning was issued, the owner of a large orange grove asked his workers to spray all his trees with water. The water was supposed to freeze and form a protective covering of ice around the orange blossom. Nevertheless, the owner suspected that some trees suffered considerable damage due to the frost. To estimate the proportion of trees that suffered more than 50 percent damage due to the frost, he took a random sample of 100 trees from his grove. What is the response variable in this experiment?\n(A) The proportion of trees that suffered more than 50 percent damage due to frost. (B) The number of trees affected by the frost. (C) The number of trees sampled from the grove. (D) For each sampled tree, whether it suffered more than 50 percent damage or at most 50 percent damage.\nA: Let's think step by step. In this experiment, the response variable is what is measured. For each tree, what is measured is whether or not it suffered more than 50 percent damage due to the frost. The answer is (D).\n\nQ: Suppose X and Y are random variables with E(X) = 37, var(X) = 5, E(Y) = 62, and var(Y) = 12. What are the expected value and variance of the random variable X + Y?\n(A) E(X + Y) = 99, var(X + Y) = 8.5 (B) E(X + Y) = 99, var(X + Y) = 13 (C) E(X + Y) = 99, var(X + Y) = 17 (D) There is insufficient information to answer this question.\nA: Let's think step by step. While means of sums of random variables add (regardless of whether the variables are independent) in order to determine the variance of a sum of random variables, we need to know not just their individual variances but the covariance of the two variables, which is not given in this problem. The answer is (D).\n\nQ: Which of the following sets has the smallest standard deviation? Which has the largest?\nI: {1,2,3}\nII: {-10,10}\nIII: {100}\n(A) I, II (B) II, III (C) III, I (D) III, II\nA: Let's think step by step. The variance of distribution I is the expected squared deviation from its mean (which is 2), so the variance is 2/3 . The variance of distribution II is 10^2 (because both elements are 10 away from the mean of zero). The variance of distribution III is 0, since it has a single entry. So distribution III has the smallest standard deviation and distribution II has the largest. The answer is (D).\n\nQ: Which of the following is a correct statement about correlation?\n(A) If the slope of the regression line is exactly 1, then the correlation is exactly 1. (B) If the correlation is 0, then the slope of the regression line is undefined. (C) Switching which variable is called x and which is called y changes the sign of the correlation. (D) The correlation r is equal to the slope of the regression line when z-scores for the y-variable are plotted against z-scores for the x-variable.\nA: Let's think step by step. Statement A is false because the slope of the regression line being exactly 1 can occur even when the two variables are not perfectly correlated. Statement B is false because uncorrelated variables regression lines can have slope zero. Statement C is false because correlation is symmetric in the two random variables. The answer is (D).\n\n", "high_school_us_history": "The following are multiple choice questions (with answers) about high school us history.\n\nQ: This question refers to the following information.\nI come not to urge personal claims, nor to seek individual benefits; I appear as the advocate of those who cannot plead their own cause; I come as the friend of those who are deserted, oppressed, and desolate. In the Providence of God, I am the voice of the maniac whose piercing cries from the dreary dungeons of your jails penetrate not your Halls of Legislation. I am the Hope of the poor crazed beings who pine in the cells, and stalls, and cages, and waste rooms of your poor-houses. I am the Revelation of hundreds of wailing, suffering creatures, hidden in your private dwellings, and in pens and cabins\u2014shut out, cut off from all healing influences, from all mind-restoring cares.\u2026 Could their melancholy histories be spread before you as revealed to my grieved spirit during the last three months, how promptly, how earnestly would you search out the most approved means of relief; how trifling, how insignificant, by comparison, would appear the sacrifices you are asked to make; how would a few dimes and dollars, gathered from each citizen, diminish in value as a possession, compared with the certain benefits and vast good to be secured for the suffering insane...by the consecration and application of a sufficient fund to the construction of a suitable hospital.\u2026\n\u2014Dorothea Dix, Memorial Soliciting a State Hospital for the Protection and Cure of the Insane,\nSubmitted to the General Assembly of North Carolina, November 1848\nDorothea Dix can best be compared to whom?\n(A) Abigail Adams (B) Clara Barton (C) Shirley Temple (D) Hillary Clinton\nA: Let's think step by step. We refer to Wikipedia articles on us history for help. Both Dorothea Dix and Clara barton are American nurses. The answer is (B).\n\nQ: This question refers to the following information.\n\"As our late Conduct at the Conestoga Manor and Lancaster have occasioned much Speculation & a great diversity of Sentiments in this and neighboring Governments; some vindicating & others condemning it; some charitably alleviating the Crime, & others maliciously painting it in the most odious & detestable Colours, we think it our duty to lay before the Publick, the whole Matter as it appeared, & still appears, to us. . . .\n\"If these things are not sufficient to prove an unjustifiable Attachment in the Quakers to the Indians Savages, a fixed Resolution to befriend them & an utter insensibility to human Distresses, let us consider a few more recent Facts. When we found the last Summer that we were likely to get no Assistance from the Government, some Volunteers went out at our own Expense, determined to drive our Enemies from our Borders; & when we came near to the great Island, we understood that a Number of their Warriors had gone out against our Frontiers. Upon this we returned and came up with them and fought with them at the Munfey Hill where we lost some of our Men & killed some of their Warriors & thereby saved our Frontiers from this Story in another Expedition. But no sooner had we destroyed their Provisions on the great Island, & ruined their trade with the good People at Bethlehem, but these very Indians, who were justly suspected of having murdered our Friends in Northampton County, were by the Influence of some Quakers taken under the Protection of the Government to screen them from the Resentments of the Friends and Relations of the Murdered, & to support them thro the Winter.\"\n\u2014\"Apology of the Paxton Boys\" (pamphlet), 1764 (Note: \"apology\" in this context should be read as an explanation, not an admission of guilt or regret.\nThe sentiments expressed in the explanation above reflect which of the ongoing tensions during the colonial period of American history?\n(A) Tensions between British policies and the aspirations of North American colonists. (B) Tensions between American Indians allied with the French and those allied with the British. (C) Tensions between freed African Americans and white planters. (D) Tensions between backcountry settlers and elites within colonial America.\nA: Let's think step by step. We refer to Wikipedia articles on us history for help. After the French and Indian War, the Scotch-Irish settlers attacked American Indians. After the attacks on the Conestoga, about 250 Paxton Boys present their grievances to the Pennsylvania legislature. As mentioned in the information, the Paxton Boys cited resentiment at local elites. The answer is (D).\n\nQ: This question refers to the following information.\nOur leaders talk about stopping aggression from the north, but this was a struggle among groups of Vietnamese until we intervened. We seem bent upon saving the Vietnamese from Ho Chi Minh even if we have to kill them and demolish their country to do it. As the native people survey bombed-out villages, women and children burned by napalm, rice crops destroyed and cities overrun with our military personnel, they are doubtless saying secretly of the Vietcong guerillas and of the American forces, \"A plague on both your houses.\" \u2026 Stop the bombing, north and south, end search and destroy offensive sweeps, and confine our military action to holding operations on the ground. Bombing the north has failed to halt or seriously check the flow of troops to the south and may, in fact, have prompted a much greater war effort by Hanoi.\n\u2014Senator George McGovern, \"The Lessons of Vietnam,\" April 25, 1967\nWhich of the following opinions from the 1960s most directly reflects the perspective of George McGovern's speech?\n(A) Americans must maximize their technological edge in Vietnam. (B) American bombing in Vietnam is step by step leading to progress in the war. (C) American bombing in Vietnam is a failure. (D) America must not give in to defeatism about the war in Vietnam.\nA: Let's think step by step. We refer to Wikipedia articles on us history for help. \"Stop the bombing\" and \"Bombing the north has failed to halt or seriously check the flow of troops to the south\" indicate that the perspective of George McGovern's speech is that Amerian bombing in Vietnam is a failure. The answer is (C).\n\nQ: This question refers to the following information.\n\"In the new Code of Laws which I suppose it will be necessary for you to make I desire you would Remember the Ladies, and be more generous and favorable to them than your ancestors. Do not put such unlimited power into the hands of the Husbands. Remember all Men would be tyrants if they could. If particular care and attention is not paid to the Ladies we are determined to foment a Rebellion, and will not hold ourselves bound by any Laws in which we have no voice, or Representation.\"\nAbigail Adams, in a letter to John Adams, 1776\n\"Special legislation for woman has placed us in a most anomalous position. Women invested with the rights of citizens in one section\u2014voters, jurors, office-holders\u2014crossing an imaginary line, are subjects in the next. In some States, a married woman may hold property and transact business in her own name; in others, her earnings belong to her husband. In some States, a woman may testify against her husband, sue and be sued in the courts; in others, she has no redress in case of damage to person, property, or character. In case of divorce on account of adultery in the husband, the innocent wife is held to possess no right to children or property, unless by special decree of the court. But in no State of the Union has the wife the right to her own person, or to any part of the joint earnings of the co-partnership during the life of her husband. In some States women may enter the law schools and practice in the courts; in others they are forbidden. In some universities girls enjoy equal educational advantages with boys, while many of the proudest institutions in the land deny them admittance, though the sons of China, Japan and Africa are welcomed there. But the privileges already granted in the several States are by no means secure.\"\nSusan B. Anthony, \"Declaration of Rights for Women,\" July 4, 1876\nThe sentiments expressed in the second excerpt by Susan B. Anthony are most likely in support of\n(A) the Equal Rights Amendment (B) universal suffrage (C) states' rights (D) prohibition\nA: Let's think step by step. We refer to Wikipedia articles on us history for help. The above information mentioned that women are in an anomalous position in terms of legislation. Women's earnings do not belong to themselves, or they cannot testify against her husbands. Susan believes women should have equal legal rights as men. The answer is (B).\n\nQ: This question refers to the following information.\n\"Society in every state is a blessing, but government even in its best state is but a necessary evil; in its worst state an intolerable one; for when we suffer, or are exposed to the same miseries by a government, which we might expect in a country without government, our calamity is heightened by reflecting that we furnish the means by which we suffer. Government, like dress, is the badge of lost innocence; the palaces of kings are built on the ruins of the bowers of paradise. For were the impulses of conscience clear, uniform, and irresistibly obeyed, man would need no other lawgiver; but that not being the case, he finds it necessary to surrender up a part of his property to furnish means for the protection of the rest; and this he is induced to do by the same prudence which in every other case advises him out of two evils to choose the least. Wherefore, security being the true design and end of government, it unanswerably follows that whatever form thereof appears most likely to ensure it to us, with the least expense and greatest benefit, is preferable to all others.\"\nThomas Paine, Common Sense, 1776\nWhich of the following \"miseries\" alluded to above were most condemned by Anti-Federalists of the post-Revolutionary era?\n(A) Organized response to Bacon's Rebellion (B) Federal response to Shays's Rebellion (C) Federal response to the Whiskey Rebellion (D) Federal response to Pontiac's Rebellion\nA: Let's think step by step. We refer to Wikipedia articles on us history for help. Anti-Federalists do not believe centralized government power, and suspect Washington's military response to Whiskey Rebellion. Bacon's Rebellion and Pontiac's Rebellion happen before the Revolution and they can be ruled out. The answer is (C).\n\n", "high_school_world_history": "The following are multiple choice questions (with answers) about high school world history.\n\nQ: This question refers to the following information.\n\"At least one of the [world's] societies would have to somehow enormously increase its productivity [in order to achieve global hegemony]. That quantum jump would have to be made before the various scientific, technological, agricultural, and industrial revolutions on which our post-quantum-leap world rests. It could only be accomplished by exploiting the ecosystems, mineral resources, and human assets of whole continents outside the lands of the society making the jump. Western Europe did just that by means of its brutality and guns and, more important, by geographical and ecological luck.\"\nCopyright \u00a9 2015 Cambridge University Press.\nAlfred Crosby, historian, Ecological Imperialism, 2004\nThe \"quantum jump\" mentioned in the passage most directly contributed to which of the following developments in the period 1450\u20131750 C.E.?\n(A) A breakdown in trade routes through the collapse of the established state structure (B) An increase in the population of the world through more plentiful supplies of food (C) The spread of Chinese and Indian belief systems across the world (D) An increase in social unrest\nA: Let's think step by step. We refer to Wikipedia articles on world history for help. The \"quantum jump\" mentioned in the passage refers to the conquest of the New World and the Columbian Exchange. Choice (A) and (C) did not happen in history. Choice (C) refers to the human assets. The answer is (B).\n\nQ: This question refers to the following information.\n\"The struggle against neo-colonialism is not aimed at excluding the capital of the developed world from operating in less developed countries. It is aimed at preventing the financial power of the developed countries being used in such a way as to impoverish the less developed.\nNon-alignment, as practiced by Ghana and many other countries, is based on co-operation with all States whether they be capitalist, socialist or have a mixed economy. Such a policy, therefore, involves foreign investment from capitalist countries, but it must be invested in accordance with a national plan drawn up by the government of the non-aligned State with its own interests in mind. The issue is not what return the foreign investor receives on his investments\u2026The question is one of power. A State in the grip of neo-colonialism is not master of its own destiny.\"\nKwame Nkrumah, Neo-Colonialism, 1965\nWhich of the following provides the best context for Nkrumah's writings?\n(A) The Industrial Revolution (B) Decolonization (C) Regional Free Trade Associations (D) Autarky\nA: Let's think step by step. We refer to Wikipedia articles on world history for help. The passage expresses a point that the successful fight against neo-colonialism were in danger and the newly independent nations like Ghana may be re-colonized via financial power of the developed countries. The answer is (B).\n\nQ: This question refers to the following information.\n\"Indeed, as both the fatwas of distinguished [scholars] who base their opinion on reason and tradition alike and the consensus of the Sunni community agree that the ancient obligation of extirpation, extermination, and expulsion of evil innovation must be the aim of our exalted aspiration, for \"Religious zeal is a victory for the Faith of God the Beneficent\"; then, in accordance with the words of the Prophet (Peace upon him!) \"Whosoever introduces evil innovation into our order must be expelled\" and \"Whosoever does aught against our order must be expelled,\" action has become necessary and exigent\u2026\"\nLetter from Ottoman Sultan Selim I to Safavid Shah Ismail I, 1514\nThe letter from Selim I is most clearly an example of which of the following?\n(A) The maintenance of military supremacy at all costs (B) Expanding tensions between religious sects (C) Factors that brought about the collapse of the Ottoman Empire (D) Peacemaking efforts among the Islamic empires\nA: Let's think step by step. We refer to Wikipedia articles on world history for help. The passage is an example of expanding tensions between Selim and Ismail. In the passage the Selim references the fatwa and the consensus of the Sunni community to against whosoever introduces evil. The answer is (B).\n\nQ: This question refers to the following information.\n\"The real grievance of the worker is the insecurity of his existence; he is not sure that he will always have work, he is not sure that he will always be healthy, and he foresees that he will one day be old and unfit to work. If he falls into poverty, even if only through a prolonged illness, he is then completely helpless, exam_ins to his own devices, and society does not currently recognize any real obligation towards him beyond the usual help for the poor, even if he has been working all the time ever so faithfully and diligently. The usual help for the poor, however, leaves a lot to be desired, especially in large cities, where it is very much worse than in the country.\"\nOtto von Bismarck, 1884\nOtto von Bismarck likely made this speech in reaction to which of the following issues?\n(A) Social acceptance of child labor (B) Declining life expectancy in Germany (C) Criticisms of German trade tariffs (D) Negative effects attributed to industrial capitalism\nA: Let's think step by step. We refer to Wikipedia articles on world history for help. The passage talks about the grievance of the work under the industrial capitalism. The answer is (D).\n\nQ: This question refers to the following information.\nHe contains all works and desires and all perfumes and all tastes. He enfolds the whole universe and in silence is loving to all. This is the Spirit that is in my heart, this is Brahman. To him I shall come when I go beyond this life, and to him will come he who has faith and doubts not.\n\u2014The Upanishads, India, c. 1000 BCE\nTo which religion does the speaker most likely belong?\n(A) Hinduism (B) Buddhism (C) Shintoism (D) Zoroastrianism\nA: Let's think step by step. We refer to Wikipedia articles on world history for help. Brahman refers to the ultimate reality of all things in the Hindu religion. In contrast, Buddhism does not have a concept of supreme God. The answer is (A).\n\n", "human_aging": "The following are multiple choice questions (with answers) about human aging.\n\nQ: All other things being equal, which of the following persons is more likely to show osteoporosis?\n(A) An older Hispanic American woman (B) An older African American woman (C) An older Asian American woman (D) An older Native American woman\nA: Let's think step by step. We refer to Wikipedia articles on human aging for help. Although osteoporosis can occur at any age, the risk is higher for older people. It is most common in Asian and non-Hispanic white women. The answer is (C).\n\nQ: The finding that adults tend to remember events from their adolescence better than from other periods in their lives is referred to as the\n(A) Adolescence advantage (B) Reminiscence bump (C) Memorial memorial (D) Quadratic retrieval spike\nA: Let's think step by step. We refer to Wikipedia articles on human aging for help. Reminiscence bump is a phenomenon that older adults tend to recollect events during their young ages. People usually have a period of childhood amnesia from birth to around age 5, and a reminiscence bump between 10 and 30. The answer is (B).\n\nQ: Which element in tobacco smoke is responsible for cancers?\n(A) Nicotine (B) Tar (C) Carbon monoxide (D) Smoke particles\nA: Let's think step by step. We refer to Wikipedia articles on human aging for help. The benzene, acrylamide and acrylonitrile in tar interact with the lungs and cause DNA mutations in cells of the lungs, and lead to cancer. The answer is (B).\n\nQ: When older adults move to a new state after retirement, which of the following is the more likely destination?\n(A) Texas (B) California (C) Hawaii (D) Vermont\nA: Let's think step by step. We refer to Wikipedia articles on human aging for help. Texas does not have state tax, and has low cost of living compared with the other three options. The answer is (A).\n\n", "human_sexuality": "The following are multiple choice questions (with answers) about human sexuality.\n\nQ: The most common disorder among men who seek sexual therapy is:\n(A) premature ejaculation (B) inhibited ejaculation (C) erectile disorder (D) ejaculatory disorder\nA: Let's think step by step. We refer to Wikipedia articles on human sexuality for help. The most common type of arousal disorder for men is erectile dysfunction, meaning that a person is not able to get any physical satisfaction from sexual activity although he may be interested in it. The answer is (C).\n\nQ: A woman who knows she has active herpes and untreated syphilis but continues to have sex without informing her partners of her condition has, in psychoanalytic terms:\n(A) a strong ego (B) a weak superego (C) a weak id (D) a strong superego\nA: Let's think step by step. We refer to Wikipedia articles on human sexuality for help. A person with weak superego tends to be delinquent, criminal or have antisocial personality. The action of the woman who knows she has active venereal disease but still have sex with her partners indicate she may has antisocial personality. The answer is (B).\n\nQ: The nature of homosexual activities that occur during preadolescence include all but which of the following?\n(A) sexual intercourse (B) circle jerks (C) exhibitionism (D) touching each other's genitals\nA: Let's think step by step. We refer to Wikipedia articles on human sexuality for help. Sexual intercourse is prohibited during preadolescence. The answer is (A).\n\nQ: Women's ability to have multiple orgasms is primarily due to:\n(A) the fact that they do not have a refractory period. (B) the response of the inner layers of the vagina. (C) having alternating orgasms in different locations. (D) the G-Spot.\nA: Let's think step by step. We refer to Wikipedia articles on human sexuality for help. The refractory period is the time when a person is not able to be erect or is not interested in sex. The answer is (A).\n\nQ: Morning sickness is typically a problem:\n(A) during the first trimester (B) during the second trimester (C) during the third trimester (D) all through the pregnancy\nA: Let's think step by step. We refer to Wikipedia articles on human sexuality for help. Morning sickness usually begins by nine weeks after conception, corresponding to the first trimester. The answer is (A).\n\n", "international_law": "The following are multiple choice questions (with answers) about international law.\n\nQ: How the consent to be bound of a State may be expressed?\n(A) The consent of a State to be bound is expressed only by ratification (B) The consent of a state to be bound by a treaty may be expressed by signature, ratification, acceptance, approval or accession (C) The consent of a State to be bound is expressed by signature (D) The consent of a State to be bound is expressed by whatever means they choose\nA: Let's think step by step. We refer to Wikipedia articles on international law for help. Article 11 of Vienna Convention on the Law of Treaties signed in 1969 states that \"the consent of a State to be bound by a treaty may be expressed by signature, exchange of instruments constituting a treaty, ratification, acceptance, approval or accession, or by any other means if so agreed.\" (B) is the most precise and accurate answer. The answer is (B).\n\nQ: What is the judge ad hoc?\n(A) If a party to a contentious case before the ICJ does not have a national sitting as judge, it is entitled to nominate someone as a judge solely for that case, with the title of judge ad hoc (B) Judge ad hoc is the member of the bench of the ICJ with a casting vote (C) Judge ad hoc is a surrogate judge, in case a judge is disqualified or passes away (D) Judge ad hoc is the judge that each party will always nominate in every contentious case\nA: Let's think step by step. We refer to Wikipedia articles on international law for help. As \"ad hoc\" implies, a judge ad hoc is appointed only for a specific case or period, when a party to a contentious case before the International Court of Justice does not have a regular national sitting as judge. The answer is (A).\n\nQ: When 'consent' can serve as a circumstance precluding the wrongfulness of a State conduct?\n(A) Consent can serve as a circumstance precluding the wrongfulness whenever it is given (B) Consent can never serve as a circumstance precluding wrongfulness (C) Consent can serve as a circumstance precluding wrongfulness, provided the consent is valid and to the extent that the conduct remains within the limits of the consent given (D) Consent can always serve as a circumstance precluding wrongfulness, no matter which organ of the State gives it\nA: Let's think step by step. We refer to Wikipedia articles on international law for help. Valid consent can serve as a circumstance precluding the wrongfulness of a State conduct if the conduct remains within the limits of that consent, according to Chapter V of the Responsibility of States for Internationally Wrongful Acts, 2001, United Nations. The answer is (C).\n\nQ: Would a reservation to the definition of torture in the ICCPR be acceptable in contemporary practice?\n(A) This is an acceptable reservation if the reserving country's legislation employs a different definition (B) This is an unacceptable reservation because it contravenes the object and purpose of the ICCPR (C) This is an unacceptable reservation because the definition of torture in the ICCPR is consistent with customary international law (D) This is an acceptable reservation because under general international law States have the right to enter reservations to treaties\nA: Let's think step by step. We refer to Wikipedia articles on international law for help. For it contravenes the object and purpose of the ICCPR, this is an unacceptable reservation in contemporary practice. The answer is (B).\n\nQ: What types of force does Article 2(4) of the UN Charter prohibit?\n(A) Article 2(4) encompasses only armed force (B) Article 2(4) encompasses all types of force, including sanctions (C) Article 2(4) encompasses all interference in the domestic affairs of States (D) Article 2(4) encompasses force directed only against a State's territorial integrity\nA: Let's think step by step. We refer to Wikipedia articles on international law for help. Article 2(4) of the UN Charter prohibits states from using armed forces in their international relations. The answer is (A).\n\n", "jurisprudence": "The following are multiple choice questions (with answers) about jurisprudence.\n\nQ: Iverson Jewelers wrote a letter to Miller, 'We have received an exceptionally fine self winding Rolox watch which we will sell to you at a very favorable price.'\n(A) The letter is an offer to sell (B) A valid offer cannot be made by letter. (C) The letter contains a valid offer which will terminate within a reasonable time. (D) The letter lacks one of the essential elements of an offer.\nA: Let's think step by step. We refer to Wikipedia articles on jurisprudence for help. An offer shows the intent to enter into a mutually-beneficial contract with specific terms. An offer can be made by a letter. While this letter indicates the willingness to sell, the lack of specific terms, such as transaction price and offer expiration date, makes it an incomplete offer. The answer is (D).\n\nQ: Functions of the law include all but which of the following?\n(A) maximizing individual freedom (B) providing a basis for compromise (C) keeping the peace (D) promoting the principles of the free enterprise system\nA: Let's think step by step. We refer to Wikipedia articles on jurisprudence for help. Laws are fundamentally about helping resolve disputes between individuals, and therefore essential for maximizing individual freedom, providing a basis for compromise, and keeping the peace. The answer is (D).\n\nQ: The ________ School of jurisprudence postulates that the law is based on what is \"correct.\"\n(A) Natural Law (B) Analytical (C) Historical (D) Sociological\nA: Let's think step by step. We refer to Wikipedia articles on jurisprudence for help. Natural Law School of jurisprudence focuses on the laws of nature, and states that the law should be based on ethics, morals, and what is \"correct\". Analytical deals with the law as it already exists, Historical postulates that the law was found and not made, and Sociological studies how the law and society impact each other. The answer is (A).\n\nQ: Which word best summarizes Weber's explanation of the development of formally rational law?\n(A) Authority. (B) Charisma. (C) Co-operation. (D) Capitalism.\nA: Let's think step by step. We refer to Wikipedia articles on jurisprudence for help. Weber explained the development of formal rationality in laws as how the modern society moved from tradition to rationality, where people decide actions based less on how they were culturally done and more on expected utilities. How rational individuals optimize efficiency of accomplishing tasks for higher rewards is a core principle of Capitalism. The answer is (D).\n\nQ: Which position does Rawls claim is the least likely to be adopted by the POP (people in the original position)?\n(A) The POP would choose equality above liberty. (B) The POP would opt for the 'maximin' strategy. (C) The POP would opt for the 'difference principle'. (D) The POP would reject the 'system of natural liberty.'\nA: Let's think step by step. We refer to Wikipedia articles on jurisprudence for help. The POP would opt for the 'maximin' strategy, opt for the 'difference principle', and reject the 'system of natural liberty', but the POP would not choose equality above liberty, since the POP assume both equal and free citizens. The answer is (A).\n\n", "logical_fallacies": "The following are multiple choice questions (with answers) about logical fallacies.\n\nQ: When an arguer causes confusion during refutation because of real or feigned lack of an ability to engage in refutation, that arguer may have committed the fallacy of\n(A) poor sportsmanship (B) appeal to compassion (C) argument against the person (D) ignorance of refutation\nA: Let's think step by step. We refer to Wikipedia articles on logical fallacies for help. Ignorance of refutation, one of Aristotle's original list of logical fallacies in his Organon, is when someone causes confusion in an argument through real or feigned inability to engage in refutation, in order to win the argument. The answer is (D).\n\nQ: The complex question fallacy consists of\n(A) arguing something is inferior just because it doesn't do something it was never intended to do. (B) including more than one claim in the proposition and treating proof for one claim as proof for all the claims. (C) drawing a conclusion before examining the evidence, and only considering evidence that supports that conclusion. (D) asking a question that includes either an unproven assumption or more than one question, thus making a straightforward yes or no answer meaningless.\nA: Let's think step by step. We refer to Wikipedia articles on logical fallacies for help. The complex question fallacy is when someone makes a single yes or no answer to a question meaningless, by including either an unproven assumption or many questions. The latter is also known as the many questions fallacy. The answer is (D).\n\nQ: Arguing that what is true of the parts must be true of the whole is the fallacy of...\n(A) Division (B) Composition (C) Appeal to the person (D) Appeal to ignorance\nA: Let's think step by step. We refer to Wikipedia articles on logical fallacies for help. Fallacy of composition occurs when someone argues what is true of the parts must be true of the whole. The answer is (B).\n\nQ: Which of the following is true of a valid categorical syllogism?\n(A) The minor premise must deny the antecedent (B) The major premise must affirm the consequent (C) The middle term must be used in at least one premise in a universal or unqualified sense (D) All of the above\nA: Let's think step by step. We refer to Wikipedia articles on logical fallacies for help. A valid categorical syllogism must satisfy several conditions: (1) the syllogism must have exactly three terms (2) every term of the syllogism must be used twice exactly, (3) a term may be used only once in any premise, and (4) the middle term must be used in at least one premise in a universal or unqualified sense, etc. Only (C) is true. The answer is (C).\n\nQ: If someone attacks the character of an opposing arguer, instead of responding to that opponent's arguments, the first person has probably committed which of the following fallacies?\n(A) tu quoque (B) horse laugh (C) argument against the person (D) ignoratio elenchi\nA: Let's think step by step. We refer to Wikipedia articles on logical fallacies for help. The argument against the person fallacy occurs when someone irrelevantly attacks the character of an opposing arguer, instead of addressing that opponent's arguments. The answer is (C).\n\n", "machine_learning": "The following are multiple choice questions (with answers) about machine learning.\n\nQ: Which image data augmentation is most common for natural images?\n(A) random crop and horizontal flip (B) random crop and vertical flip (C) posterization (D) dithering\nA: Let's think step by step. Data augmentation is used to increase the diversity of images in the training dataset. It is important that natural images are kept natural after being augmented. Vertical flips of images are not natural, so (B) is false. Posterization makes the image look like a poster and and dithering increases color depth. None of these two preserve the natural property. The only natural data augmentation technique is (A). The answer is (A).\n\nQ: Traditionally, when we have a real-valued input attribute during decision-tree learning we consider a binary split according to whether the attribute is above or below some threshold. Pat suggests that instead we should just have a multiway split with one branch for each of the distinct values of the attribute. From the list below choose the single biggest problem with Pat\u2019s suggestion:\n(A) It is too computationally expensive. (B) It would probably result in a decision tree that scores badly on the training set and a testset. (C) It would probably result in a decision tree that scores well on the training set but badly on a testset. (D) It would probably result in a decision tree that scores well on a testset but badly on a training set.\nA: Let's think step by step. Because the input is real valued, it is unlikely that the same values appear both at training and test time. This means that while such a decision tree could yield good performance on the training data, when evaluated on the test data it will perform badly because the decision tree won\u2019t know what to do with numbers that did not appear in the training data. The answer is (C).\n\nQ: You are reviewing papers for the World\u2019s Fanciest Machine Learning Conference, and you see submissions with the following claims. Which ones would you consider accepting?\n(A) My method achieves a training error lower than all previous methods! (B) My method achieves a test error lower than all previous methods! (Footnote: When regularisation parameter \u03bb is chosen so as to minimise test error.) (C) My method achieves a test error lower than all previous methods! (Footnote: When regularisation parameter \u03bb is chosen so as to minimise cross-validaton error.) (D) My method achieves a cross-validation error lower than all previous methods! (Footnote: When regularisation parameter \u03bb is chosen so as to minimise cross-validaton error.)\nA: Let's think step by step. In machine learning, we train with some data and fixed hyperparameters and the training error can be arbitrarily low, so (A) can\u2019t be right. Then, one compares different hyperparameters by selecting the model with the lowest cross-validation error, this means that (B) and (D) are not the right procedure. The only relevant number after these is the test error and thus (C) is the right answer. The answer is (C).\n\nQ: A 6-sided die is rolled 15 times and the results are: side 1 comes up 0 times; side 2: 1 time; side 3: 2 times; side 4: 3 times; side 5: 4 times; side 6: 5 times. Based on these results, what is the probability of side 3 coming up when using Add-1 Smoothing?\n(A) 2.0/15 (B) 1.0/7 (C) 3.0/16 (D) 1.0/5\nA: Let's think step by step. Add-1 smoothing adds the value of one to the different counts and then normalizes the probabilities accordingly. The counts after adding one will be: side 1 comes up 1 time; side 2: 2 times; side 3: 3 times; side 4: 4 times; side 5: 5 times; side 6: 6 times. The number of sum one die rolls will be 21, so the probability of drawing a three is 3/21 = 1/7. The answer is (B).\n\nQ: To achieve an 0/1 loss estimate that is less than 1 percent of the true 0/1 loss (with probability 95%), according to Hoeffding's inequality the IID test set must have how many examples?\n(A) around 10 examples (B) around 100 examples (C) between 100 and 500 examples (D) more than 1000 examples\nA: Let's think step by step. By the Hoeffding\u2019s inequality, we expect that with 95% probability the in-sample and out-of-sample errors differ by epsilon when we have N samples if 2 exp(-2 epsilon^2 N)<0.05, this implies that N > -1/(2*epsilon**2) log ( 0.05/2 )= log (40)*5000. Since log(40)>1, we have that one needs more than 1000 examples. The answer is (D).\n\n", "management": "The following are multiple choice questions (with answers) about management.\n\nQ: How can organisational structures that are characterised by democratic and inclusive styles of management be described?\n(A) Hierarchical (B) Bureaucratic (C) Flat (D) Functional\nA: Let's think step by step. We refer to Wikipedia articles on management for help. Flat organizational structures are characterized by democratic and inclusive styles of management, and have few (if any) levels of management between the workers and managers. The answer is (C).\n\nQ: Hygiene factors are associated with which writer?\n(A) Frederick Hertzberg (B) D.C. McClelland (C) Abraham Maslow (D) Douglas McGregor\nA: Let's think step by step. We refer to Wikipedia articles on management for help. Hygiene factors include compensation, company policies, supervision, interpersonal relations, and work environments. Hertzberg lists them as factors that cannot motivate employees but can minimize job dissatisfaction. The answer is (A).\n\nQ: What characteristic is not a key feature of the 'open systems' model of management?\n(A) Morale (B) Innovation (C) Growth resource (D) Adaptation\nA: Let's think step by step. We refer to Wikipedia articles on management for help. The key characteristics of an open system in management include innovation, growth resource, and adaption, but do not include morale. The answer is (A).\n\nQ: Which element of the cultural web forms regalia?\n(A) Symbols (B) Rituals and routines (C) Power structures (D) Control systems\nA: Let's think step by step. We refer to Wikipedia articles on management for help. The cultural web is a tool for mapping an organization's culture, where symbols form the regalia that visually expresses the values that the organization holds as important. The answer is (A).\n\nQ: What are the two main dimensions of the Ohio Studies into leadership?\n(A) Starting position and end position (B) Initial environment and changed environment (C) Organisational structure and conditioning (D) Initiating structure and considerations\nA: Let's think step by step. We refer to Wikipedia articles on management for help. The Ohio State Leadership Studies conducted in the 1940s identified initiating structure and consideration as the two main dimensions of leader behavior. The answer is (D).\n\n", "marketing": "The following are multiple choice questions (with answers) about marketing.\n\nQ: Although the content and quality can be as controlled as direct mail, response rates of this medium are lower because of the lack of a personal address mechanism. This media format is known as:\n(A) Care lines. (B) Direct mail. (C) Inserts. (D) Door to door.\nA: Let's think step by step. We refer to Wikipedia articles on marketing for help. Door to door marketing delivers non-addressed items within all buildings within a geographic area. While it can control the content and quality as well as direct mail marketing, its response rate is lower because of the lack of a personal address mechanism. The answer is (D).\n\nQ: In an organization, the group of people tasked with buying decisions is referred to as the _______________.\n(A) Outsourcing unit. (B) Procurement centre. (C) Chief executive unit. (D) Decision-making unit.\nA: Let's think step by step. We refer to Wikipedia articles on marketing for help. In an organization, the group of the people tasked with buying decision is referred to as the decision-making unit. The answer is (D).\n\nQ: The single group within society that is most vulnerable to reference group influence is:\n(A) The older consumer who feels somewhat left out of things. (B) The married women, many of whom feel a need for stability in their lives. (C) New immigrants who really want to assimilate into their new culture. (D) Children, who base most of their buying decisions on outside influences.\nA: Let's think step by step. We refer to Wikipedia articles on marketing for help. Children, who mostly based their buying decisions on outside influences, are the single group within society that is more vulnerable to reference group influence. The answer is (D).\n\nQ: Which of the following is an assumption in Maslow's hierarchy of needs?\n(A) Needs are dependent on culture and also on social class. (B) Lower-level needs must be at least partially satisfied before higher needs can affect behaviour. (C) Needs are not prioritized or arranged in any particular order. (D) Satisfied needs are motivators, and new needs emerge when current needs remain unmet.\nA: Let's think step by step. We refer to Wikipedia articles on marketing for help. Maslow's hierarchy of needs, from the bottom upwards, are physiological (food and clothing), safety, love and belonging needs, esteem, and self-actualization. Lower-level needs must be at least partially satisfied before higher ones can affect behavior. The answer is (B).\n\nQ: _____________ is a natural outcome when combining demographic and geographic variables.\n(A) Geodemographics (B) Product differentiation. (C) ANSOFF matrix. (D) Brand management.\nA: Let's think step by step. We refer to Wikipedia articles on marketing for help. Geodemographics is a natural outcome when combining demographic and geographic variables. The answer is (A).\n\n", "medical_genetics": "The following are multiple choice questions (with answers) about medical genetics.\n\nQ: The stage of meiosis in which chromosomes pair and cross over is:\n(A) prophase I (B) metaphase I (C) prophase II (D) metaphase II\nA: Let's think step by step. We refer to Wikipedia articles on medical genetics for help. Prophase I is the stage of meiosis where homologous chromosomes pair with each other and exchange genetic material. The answer is (A).\n\nQ: DNA ligase is\n(A) an enzyme that joins fragments in normal DNA replication (B) an enzyme of bacterial origin which cuts DNA at defined base sequences (C) an enzyme that facilitates transcription of specific genes (D) an enzyme which limits the level to which a particular nutrient reaches\nA: Let's think step by step. We refer to Wikipedia articles on medical genetics for help. DNA ligase is a type of enzyme (EC 6.5.1.1) responsible for joining DNA strands together by catalyzing a phosphodiester bond. The answer is (A).\n\nQ: Which of the following conditions does not show multifactorial inheritance?\n(A) Pyloric stenosis (B) Schizophrenia (C) Spina bifida (neural tube defects) (D) Marfan syndrome\nA: Let's think step by step. We refer to Wikipedia articles on medical genetics for help. Multifactorial inheritance is when more than a single factor is responsible for causing a given trait or health problem. Genes cannot be the only factor. Marfan syndrome, on the other hand, requires only one abnormal copy of the of the Marfan gene, from one parent, to inherit the trait. The answer is (D).\n\nQ: A gene showing codominance\n(A) has both alleles independently expressed in the heterozygote (B) has one allele dominant to the other (C) has alleles tightly linked on the same chromosome (D) has alleles expressed at the same time in development\nA: Let's think step by step. We refer to Wikipedia articles on medical genetics for help. Codominance, as it relates to genetics, refers to a type of genetic inheritance where the phenotype of both the parents is easily observed in the offspring. A heterozygote is an individual having two different alleles of a gene. The answer is (A).\n\nQ: Large triplet repeat expansions can be detected by:\n(A) polymerase chain reaction. (B) single strand conformational polymorphism analysis. (C) Southern blotting. (D) Western blotting.\nA: Let's think step by step. We refer to Wikipedia articles on medical genetics for help. A Southern blot is a method in molecular biology for detecting specific DNA sequences in a sample. Large triplet repeat expansions are usually detected with this method. The answer is (C).\n\n", "miscellaneous": "The following are multiple choice questions (with answers) about miscellaneous.\n\nQ: Which of these songs was a Top 10 hit for the rock band The Police?\n(A) 'Radio Ga-Ga' (B) 'Ob-la-di Ob-la-da' (C) 'De Do Do Do De Da Da Da' (D) 'In-a-Gadda-Da-Vida'\nA: Let's think step by step. We refer to Wikipedia for help. Radio Ga-Ga is by Queen. Ob-la-di Ob-la-da is by The Beatles. And In-a-Gadda-Da-Vida is by Iron Butterfly. Leaving 'De Do Do Do De Da Da Da' as the only song by The Police, and also a Top 10 hit. The answer is (C).\n\nQ: What place is named in the title of the 1979 live album by rock legends Cheap Trick?\n(A) Budapest (B) Budokan (C) Bhutan (D) Britain\nA: Let's think step by step. We refer to Wikipedia for help. Nippon Budokan is an indoor arena in Tokyo, Japan renowned for hosting rock music concerts including Cheap Trick in 1978. 'Cheap Trick at Budokan' became the name of their album. The answer is (B).\n\nQ: What is produced during photosynthesis?\n(A) hydrogen (B) nylon (C) oxygen (D) light\nA: Let's think step by step. We refer to Wikipedia for help. Photosynthesis is the process in which green plants use the green pigment chlorophyll to synthesize foods with water and carbon dioxide. Oxygen is the byproduct of this process. The answer is (C).\n\nQ: Who is the shortest man to ever win an NBA slam dunk competition?\n(A) Anthony 'Spud' Webb (B) Michael 'Air' Jordan (C) Tyrone 'Muggsy' Bogues (D) Julius 'Dr J' Erving\nA: Let's think step by step. We refer to Wikipedia for help. In 1986, Spud Webb, standing only 5'7\" became the shortest NBA player in history to win an official slam dunk contest. The answer is (A).\n\nQ: How many axles does a standard automobile have?\n(A) one (B) two (C) four (D) eight\nA: Let's think step by step. We refer to Wikipedia for help. Most cars have two axles to rotate the wheels.. The answer is (B).\n\n", "moral_disputes": "The following are multiple choice questions (with answers) about moral disputes.\n\nQ: Baron admits that the versions of the ticking bomb hypothetical she discusses are \"stunningly stupid,\" but she claims this is actually evidence of\n(A) the stupidity of most traditional philosophical examples. (B) a general lack of intelligence among people with advanced degrees. (C) the wrongness of torture. (D) the readiness on the part of many intelligent people to see torture as the best solution to deal with terrorism.\nA: Let's think step by step. We refer to Wikipedia articles on moral disputes for help. The ticking bomb hypothetical poses a problem where many people will die to an exploding bomb, if the hypothetical terrorist does not disclose how to defuse it. Baron sees this hypothetical as silly, but its prevalence does suggest intelligent people, particularly utilitarians, see torture as justifiable to save the lives in this scenario. The answer is (D).\n\nQ: A fertilized ovum is also known as\n(A) a zygote. (B) an embryo. (C) a viability. (D) a blastocyst.\nA: Let's think step by step. We refer to Wikipedia articles on moral disputes for help. Once a single sperm penetrates the layers of an egg to form a new cell, that cell is called a zygote. The answer is (A).\n\nQ: Pence compares six different cases of reproduction, from natural twinning to SCNT. What conclusion does he draw from this comparison?\n(A) SCNT is not a different kind of reproduction because there are no morally relevant differences between it and other permissible means of reproduction. (B) Because there is a low risk of harm for natural twinning, there will be a low risk of harm for SCNT. (C) both A and B (D) neither A nor B\nA: Let's think step by step. We refer to Wikipedia articles on moral disputes for help. SCNT, known as 'somatic cell nuclear transfer', is a technique in which the nucleus of a somatic (body) cell is transferred to the cytoplasm of an enucleated egg. This is used in farm animal cloning. Pence may compare this to other processes of reproduction, and see little moral separation. The answer is (A).\n\nQ: Which of the following is an intrinsic good, according to Singer?\n(A) being in the upper class (B) the ability to learn and solve problems (C) being more beautiful than most of the population (D) being shorter than average\nA: Let's think step by step. We refer to Wikipedia articles on moral disputes for help. Bioethicist Peter Singer sees intrinsic value as innate values conferred by oneself, for oneself. Innanimate objects can be beautiful, short, or have some valuable criteria, but capabilities are intrinsically good. The answer is (B).\n\nQ: According to Metz, what is wrong with consequentialist arguments against capital punishment based on African values?\n(A) It is unclear as of yet whether or not capital punishment deters harm to the community. (B) It is unclear as of yet whether or not capital punishment deters harm to any individuals. (C) Consequentialism is not supported by African values. (D) Even though consequentialism is supported by African values, no consequentialist arguments framed in terms of African values have been offered.\nA: Let's think step by step. We refer to Wikipedia articles on moral disputes for help. Thaddeus Metz is a humanities research professor focusing on ethical philosophy in South Africa. Metz has written the death penalty is unjustified as it treats individuals as incapable of communal relations. It is unclear that capital punishment is to the benefit of, or a deterrent of harm to the community. The answer is (A).\n\n", "moral_scenarios": "The following are multiple choice questions (with answers) about moral scenarios.\n\nQ: For which of these two scenarios does the main character (who uses I/me/my) do something clearly morally wrong, according to ordinary moral standards in the US as of 2020? Scenario 1 | I took his wallet after he harassed me at the bar. Scenario 2 | I made love to my newborn baby boy.\n(A) Wrong, Wrong (B) Wrong, Not wrong (C) Not wrong, Wrong (D) Not wrong, Not wrong\nA: Let's think step by step. We refer to Wikipedia articles on moral scenarios for help. Stealing from someone is not considered an ethical response to harrassment. And having sex with a minor is egregiously unethical. The answer is (A).\n\nQ: For which of these two scenarios does the main character (who uses I/me/my) do something clearly morally wrong, according to ordinary moral standards in the US as of 2020? Scenario 1 | I asked for a cigarette from the guy outside the store. Scenario 2 | When the security guard left his post, I pocketed a rare coin.\n(A) Wrong, Wrong (B) Wrong, Not wrong (C) Not wrong, Wrong (D) Not wrong, Not wrong\nA: Let's think step by step. We refer to Wikipedia articles on moral scenarios for help. Asking for something is not morally wrong. Stealing is morally wrong. The answer is (C).\n\nQ: For which of these two scenarios does the main character (who uses I/me/my) do something clearly morally wrong, according to ordinary moral standards in the US as of 2020? Scenario 1 | During the house's demolition, I pushed him into a wall. Scenario 2 | When I choose a babysitter for my twin nieces, I don't care about her racial background.\n(A) Wrong, Wrong (B) Wrong, Not wrong (C) Not wrong, Wrong (D) Not wrong, Not wrong\nA: Let's think step by step. We refer to Wikipedia articles on moral scenarios for help. Physical violence is morally wrong. Being open to hiring people from different racial backgrounds is an expected moral stance. The answer is (B).\n\nQ: For which of these two scenarios does the main character (who uses I/me/my) do something clearly morally wrong, according to ordinary moral standards in the US as of 2020? Scenario 1 | I bought an old pick-up truck for work. Scenario 2 | I looked up at the gorgeous waterfall and threw a rock at it.\n(A) Wrong, Wrong (B) Wrong, Not wrong (C) Not wrong, Wrong (D) Not wrong, Not wrong\nA: Let's think step by step. We refer to Wikipedia articles on moral scenarios for help. Neither of these are wrong as no one is wronged by buing a pick-up truck or throwing a rock into a stream. The answer is (D).\n\nQ: For which of these two scenarios does the main character (who uses I/me/my) do something clearly morally wrong, according to ordinary moral standards in the US as of 2020? Scenario 1 | I really loved my newborn baby boy. Scenario 2 | The new girl is embarrassed to live in a trailer park, so I invite the class to her house.\n(A) Wrong, Wrong (B) Wrong, Not wrong (C) Not wrong, Wrong (D) Not wrong, Not wrong\nA: Let's think step by step. We refer to Wikipedia articles on moral scenarios for help. Loving someone is not wrong. However, exposing something that someone is embarrassed about could be considered quite mean. The answer is (C).\n\n", "nutrition": "The following are multiple choice questions (with answers) about nutrition.\n\nQ: What is the first-line drug for patients with type 2 diabetes and obesity, as of 2020?\n(A) Acarbose (B) Metformin (C) Sulphonylureas (D) Insulin\nA: Let's think step by step. We refer to Wikipedia articles on nutrition for help. Metformin (Fortamet, Glumetza, or others) is usually the first medication prescribed for type 2 diabetes, as well as obesity. It works by lowering glucose production in the liver and improving the body's sensitivity to insulin. The answer is (B).\n\nQ: Which of the following statements is correct (according to knowledge in 2020)?\n(A) Consumers with phenylketonuria must avoid the consumption of the sweetener aspartame (B) Consumers with phenylketonuria must avoid the consumption of the sweetener saccharin (C) Consumers with phenylketonuria must avoid the consumption of the sweetener sucralose (D) Consumers with phenylketonuria must avoid the consumption of the sweetener acesulfame K\nA: Let's think step by step. We refer to Wikipedia articles on nutrition for help. People with phenylketonuria (PKU) cannot break down the amino acid phenylalanine. As it builds up in the blood and brain it can lead to brain damage. People with PKU should avoid foods that are converted to phenylalanine in the body, such as aspartame. The answer is (A).\n\nQ: Which of the following statements about iodine is correct, as of 2020?\n(A) 50% of adults consume iodine at levels below the RNI (B) Dairy products are a poor source of iodine (C) The iodine content of organic milk is generally lower that the level in non-organic milk (D) UK dietary reference values recommend an increase in iodine intake in pregnancy\nA: Let's think step by step. We refer to Wikipedia articles on nutrition for help. Organic milk usually has less iodine content than non-organic milk. The answer is (C).\n\nQ: Which of the following is the most plausible explanation for the protective effect of dietary fibre against cancer of the colon, as of 2020?\n(A) Propionic acid, formed during colonic fibre fermentation inhibits liver fatty acid synthesis (B) Butyric acid, formed during colonic fibre fermentation stimulates \"silencing\" of the SLC5A8 tumour suppressor gene (C) None of these options are correct (D) Butyric acid, formed during colonic fibre fermentation stimulates anti-oxidant defences in the colon\nA: Let's think step by step. We refer to Wikipedia articles on nutrition for help. Dietary fibre is inversely proportional to the risk of colorectal cancer. This is presumed because butyric acid (BA) stimulates antioxidants which help protect the colon from cancerous tumors. The answer is (D).\n\nQ: In a cohort study, the risk ratio of developing diabetes was 0.86 when comparing consumers of tea (the exposed) to those who did not drink tea (the unexposed). Which one statement is correct (according to knowledge in 2020)?\n(A) The tea drinkers have lower risk of developing diabetes. (B) The tea drinkers have higher risk of developing diabetes. (C) Based on the information given we cannot tell if the observed difference in disease risk is the result of chance. (D) The risk ratio is close to the value one, so there is no difference in disease risk between the two groups.\nA: Let's think step by step. We refer to Wikipedia articles on nutrition for help. The risk ratio is not sufficiently reduced that it could not be explained by random chance given the studies sample size. The answer is (C).\n\n", "philosophy": "The following are multiple choice questions (with answers) about philosophy.\n\nQ: The study of reality in the broadest sense, an inquiry into the elemental nature of the universe and the things in it, is known as _____.\n(A) metaphysics (B) epistemology (C) quantum physics (D) axiology\nA: Let's think step by step. We refer to Wikipedia articles on philosophy for help. Among the options, only metaphysics studies the nature of reality and existence. The answer is (A).\n\nQ: According to Moore\u2019s \u201cideal utilitarianism,\u201d the right action is the one that brings about the greatest amount of:\n(A) pleasure. (B) happiness. (C) good. (D) virtue.\nA: Let's think step by step. We refer to Wikipedia articles on philosophy for help. Moore's \"ideal utilitarianism\" states that one's actions should maximize intrinsic goods. The answer is (C).\n\nQ: Before Tolstoy's Christian conversion, what was his perspective on the meaning of life?\n(A) optimist (B) satisfied (C) nominally religious (D) pessimist\nA: Let's think step by step. We refer to Wikipedia articles on philosophy for help. Before his conversion, Tolstoy feels that life was uncertain, which is a pessimist's point of view. The answer is (D).\n\nQ: According to d'Holbach, people always act according to _____.\n(A) free choices (B) dictates of the soul (C) necessary natural laws (D) undetermined will\nA: Let's think step by step. We refer to Wikipedia articles on philosophy for help. d'Holbach believes that people act according to necessary laws, and it proves nothing about people's free will. The answer is (C).\n\nQ: Psychological egoism is:\n(A) an ethical theory about how we ought to behave. (B) a generalization concerning the way people tend to behave. (C) a claim about human nature and the ways people are capable of behaving. (D) none of the above.\nA: Let's think step by step. We refer to Wikipedia articles on philosophy for help. Psychological egoism suggests that one behaves based on what makes one feels good, hence it is a claim about human nature and how humans are capable of behaving. The answer is (C).\n\n", "prehistory": "The following are multiple choice questions (with answers) about prehistory.\n\nQ: What is the approximate mean cranial capacity of Homo erectus?\n(A) under 650 cc (B) about 800 cc (C) just under 1000 cc (D) 1200 cc\nA: Let's think step by step. We refer to Wikipedia articles on prehistory for help. The average cranium capacity of Homo erectus is less than 1000 cubic cm. The answer is (C).\n\nQ: According to Timothy Pauketat, the evidence for social stratification and political power at Cahokia suggests:\n(A) a center of Mississippian civilization with conditions similar to the rise of early states. (B) the limitations of authority in a Native American society of egalitarian foragers. (C) a simple chiefdom or perhaps a complex chiefdom had evolved by A.D. 1500. (D) a center of Mississippian civilization with conditions similar to societies on the Northwest Coast of North America.\nA: Let's think step by step. We refer to Wikipedia articles on prehistory for help. Timothy Pauketat is known for his research on Cahokia, the center of the Mississippian culture, where he found similar conditions to the rise of early states. The answer is (A).\n\nQ: Recent research on hominid species dating from the Middle Pliocene indicates there was (as of 2020):\n(A) a great amount of species diversity, or a single species that exhibited a lot of diversity. (B) very little species diversity during this period and very few hominids. (C) decreased species diversity due to a prolonged ice age followed by a severe drought. (D) decreased species diversity but increased numbers of hammerstones and flakes, indicating stone tool manufacture.\nA: Let's think step by step. We refer to Wikipedia articles on prehistory for help. Recent research has recognized multiple hominid species from the Middle Pliocene, meaning that there is a great amount of species diversity or diversity in a single species. The answer is (A).\n\nQ: Researchers now believe that the decline of the Maya was caused chiefly by:\n(A) a cataclysm of some kind, such as an earthquake, volcano, or tsunami. (B) ecological degradation resulting from slash-and-burn farming techniques. (C) endless wars between neighboring Mayan city-states. (D) practices of interbreeding that led to a steep rise in congenital disorders.\nA: Let's think step by step. We refer to Wikipedia articles on prehistory for help. Researchers believe that the Maya collapse was mainly caused by over-exploitation of natural resources like the slash-and-burn farming techniques. The answer is (B).\n\nQ: The great Mayan king Pacal built temples in the city of Palenque in order to:\n(A) satisfy the powerful Mayan astronomer priests. (B) display his generosity to the common people, since they were allowed to live in the temples. (C) frighten away enemies, in particular the Spaniards. (D) legitimize his kingship, since his father was not royal.\nA: Let's think step by step. We refer to Wikipedia articles on prehistory for help. Pacal built the temples as the funerary monument to legitimize his kingship. The answer is (D).\n\n", "professional_accounting": "The following are multiple choice questions (with answers) about professional accounting.\n\nQ: An auditor traces the serial numbers on equipment to a nonissuer\u2019s subledger. Which of the following management assertions is supported by this test?\n(A) Valuation and allocation (B) Completeness (C) Rights and obligations (D) Presentation and disclosure\nA: Let's think step by step. We refer to Wikipedia articles on accounting for help. The completeness assertion is tested by tracing supporting documents to the record entries. The answer is (B).\n\nQ: One hundred years ago, your great-great-grandmother invested $100 at 5% yearly interest. What is the investment worth today?\n(A) $13,000 (B) $600 (C) $15,000 (D) $28,000\nA: Let's think step by step. We refer to Wikipedia articles on accounting for help. A $100 investment at 5% yearly interest is worth 100*(1.05)^100=13150 after 100 years, which is around $13,000. The answer is (A).\n\nQ: On January 1, year 1, Alpha Co. signed an annual maintenance agreement with a software provider for $15,000 and the maintenance period begins on March 1, year 1. Alpha also incurred $5,000 of costs on January 1, year 1, related to software modification requests that will increase the functionality of the software. Alpha depreciates and amortizes its computer and software assets over five years using the straight-line method. What amount is the total expense that Alpha should recognize related to the maintenance agreement and the software modifications for the year ended December 31, year 1?\n(A) $5,000 (B) $13,500 (C) $16,000 (D) $20,000\nA: Let's think step by step. We refer to Wikipedia articles on accounting for help. The maintenance period begins on March 1, so only 10 months of expenses should be recognized, which is $15,000/12*10=$12,500. The software modification cost is amortized over 5 years, so each year is $5,000/5=$1,000. So the total expense is $12,500+$1,000=$13,500. The answer is (B).\n\nQ: Krete is an unmarried taxpayer with income exclusively from wages. By December 31, year 1, Krete's employer has withheld $16,000 in federal income taxes and Krete has made no estimated tax payments. On April 15, year 2, Krete timely filed for an extension request to file her individual tax return, and paid $300 of additional taxes. Krete's year 1 tax liability was $16,500 when she timely filed her return on April 30, year 2, and paid the remaining tax liability balance. What amount would be subject to the penalty for underpayment of estimated taxes?\n(A) $0 (B) $500 (C) $1,650 (D) $16,500\nA: Let's think step by step. We refer to Wikipedia articles on accounting for help. The tax due after withholding is $16,500-$16,000=$500, which is less than $1000, hence there is no underpayment penalty of estimated taxes. The answer is (A).\n\nQ: Box a nongovernmental not-for-profit organization had the following transactions during the year: Proceeds from sale of investments $80000 Purchase of property plant and equipment $10000 Proceeds from long-term debt $100000 Loss on sale of investment $5000 What amount should be reported as net cash provided by financing activities in Box's statement of cash flows?\n(A) $70,000 (B) $75,000 (C) $80,000 (D) 100000\nA: Let's think step by step. We refer to Wikipedia articles on accounting for help. Among the four transactions, only Proceeds from long-term debt belongs to the financing activities section of cashflow, hence the amount reported should be $100000. The answer is (D).\n\n", "professional_law": "The following are multiple choice questions (with answers) about professional law.\n\nQ: A son owed a creditor $5,000. The son's father contacted the creditor and told him that he wanted to pay the son's debt. The father signed a document that stated the father would pay the son's debt at a rate of $500 a month for 10 months. The creditor made no written or oral commitment to forbear to sue the son to collect the $5,000 debt, and the father made no oral or written request for any such forbearance. For the next five months, the father made and the creditor accepted the $500 monthly payments as agreed. During that period, the creditor, in fact, did forbear to take any legal action against the son. However, the father then informed the creditor that he would make no further payments on the debt. Which of the following is the most persuasive argument that the father is liable to the creditor under the terms of their agreement?\n(A) The father's promise and the creditor's reliance thereon, if proved, gave rise to a valid claim by the creditor against the father based on the doctrine of promissory estoppel. (B) Because it was foreseeable that the father's promise would induce the creditor to forbear taking any action against the son, such forbearance was, as a matter of law, a bargained-for consideration for the father's promise. (C) The father's five payments to the creditor totaling $2,500 manifested a serious intent on the father's part to be contractually bound, and such manifestation is generally recognized as an effective substitute for consideration. (D) By assuming the antecedent debt obligation that the son owed to the creditor, the father became a surety whose promise to the creditor was enforceable, since it was in writing and supported by adequate consideration. \nA: Let's think step by step. We refer to Wikipedia articles on law for help. The doctrine of promissory estoppel stops a person from going back on a promise in contract law, hence option (A) should be the most persuasive argument. The answer is (A).\n\nQ: A state has recently enacted a statute prohibiting the disposal of any nuclear wastes within the state. This law does not contravene or conflict with any federal statutes. A man operates a company in the state that is engaged in the disposal of nuclear wastes. Subsequent to the passage of the state statute, the man, not yet aware of the new law, entered into contracts with many out-of-state firms to dispose of their nuclear wastes in the state. On account of this new law, however, the man will be unable to perform these contracts. Assume that the man has standing to challenge this state law. Which of the following presents his strongest constitutional grounds to challenge the state law prohibiting the disposal of nuclear wastes within the state?\n(A) The commerce clause. (B) The equal protection clause of the Fourteenth Amendment. (C) The privileges and immunities clause of Article IV, Section 2. (D) The contract clause.\nA: Let's think step by step. We refer to Wikipedia articles on law for help. The commerce clause states that Congress shall have the power to regulate commerce with foreign Nations, and among the several States, and with the Indian Tribes. The statute affects inter-state commerce which puts it into question. Hence the man's strongest argument should be the commerce clause. The answer is (A).\n\nQ: On October 1, 1980, a developer, owner of several hundred acres in a rural county, drafted a general development plan for the area. The duly recorded plan imposed elaborate limitations and restrictions upon the land in the plan, which was to be developed as a residential district. The restrictions were to extend to all persons acquiring any of the lots and to their heirs, assigns, and lessees. It was further provided that all subsequent owners would be charged with due notice of the restrictions. Among those restrictions in the general plan were the following:(22) A franchise right is created in a strip of land 10 feet in width along the rear of each lot for the use of public utility companies with right of ingress and egress. (23) No house or structure of any kind shall be built on the aforementioned strip of land running through the said blocks. In 2000, a retiree purchased one of the lots, built a house, and erected a fence in the rear of his property within the restricted area. In 2004, a teacher purchased a lot adjacent to the retiree's property and built a new house. Two years later, a librarian purchased the lot that adjoined the teacher's property. The three deeds to those properties each contained references to the deed book where the general plan was recorded. In 2008, the librarian began the construction of a seven-foot post-and-rail fence along the line dividing his lot with the teacher's, and along the center of the area subject to the franchise right. Although the teacher objected to its construction, the fence was completed. If the teacher seeks a mandatory injunction to compel removal of the librarian's fence, the court will most likely\n(A) grant relief, because the fence was in violation of the easement restriction. (B) grant relief, because the encroachment of the fence violated the restriction in the original plan. (C) deny relief, because the teacher failed to enforce the restriction against the retiree. (D) deny relief, because the fence would not be construed as \"a structure\" within the terms of the restriction. \nA: Let's think step by step. We refer to Wikipedia articles on law for help. The restrictions in the original plan say no house or structure of any kind shall be built on the aforementioned strip of land running through the said blocks. Hence the court will most likely grant relief because the fence violated the restriction in the original plan. The answer is (B).\n\nQ: Judge took judicial notice of some facts at the beginning of the trial. Which of the following is not an appropriate kind of fact for judicial notice?\n(A) Indisputable facts. (B) Facts that have been asserted by individual political organizations. (C) Facts recognized to be true by common knowledge. (D) Facts capable of scientific verification.\nA: Let's think step by step. We refer to Wikipedia articles on law for help. Among the options, facts that have been asserted by individual political organizations is not an appropriate kind of fact for judicial notice. The answer is (B).\n\nQ: A state legislature has recently enacted a statute making it a misdemeanor to curse or revile or use obscene or opprobrious language toward or in reference to a police officer perfonning his duties. A student at a state university organized a demonstration on campus to protest the war. The rally was attended by a group of 50 students who shouted anti-war messages at cars passing by. To show his contempt for the United States, the student sewed the American flag to the rear of his jeans. When a police officer saw the flag sown on the student's jeans, he approached and told him to remove the flag or he would be placed under arrest. The student became angered and shouted at the police officer, \"Listen, you bastard, I'll wear this rag anywhere I please. \" The student was subsequently placed under arrest and charged with violating the state statute. The student subsequently brings suit in state court challenging the constitutionality of the statute. The strongest constitutional argument for the student is that\n(A) the statute is void for vagueness under the Fourteenth Amendment's due process clause. (B) the statute is invalid because it violates the petitioner's freedom of speech under the First Amendment. (C) the statute is an abridgment of freedom of speech under the First Amendment because less restrictive means are available for achieving the same purpose. (D) the statute is overbroad and consequently invalid under the First and Fourteenth Amendments.\nA: Let's think step by step. We refer to Wikipedia articles on law for help. The Fourteenth Amendment further supports the First Amendment by establishing a due process clause. Hence the strongest argument should be the statute is overbroad and consequently invalid under the First and Fourteenth Amendments. The answer is (D).\n\n", "professional_medicine": "The following are multiple choice questions (with answers) about professional medicine.\n\nQ: A 22-year-old male marathon runner presents to the office with the complaint of right-sided rib pain when he runs long distances. Physical examination reveals normal heart and lung findings and an exhalation dysfunction at ribs\u00a04-5 on the right. Which of the following muscles or muscle groups will be most useful in correcting this dysfunction utilizing a direct method?\n(A) anterior scalene (B) latissimus dorsi (C) pectoralis minor (D) quadratus lumborum\nA: Let's think step by step. We refer to Wikipedia articles on medicine for help. Among the options, only pectoralis minor muscle origins from the outer surfaces of the 3rd to 5th ribs. The answer is (C).\n\nQ: A 36-year-old male presents to the office with a\u00a03-week\u00a0history of low back pain. He denies any recent trauma but says that he climbs in and out of his truck numerous times a day for his job. Examination of the patient in the prone position reveals a deep sacral sulcus on the left, a posterior inferior lateral angle on the right, and a lumbosacral junction that springs freely on compression. The most likely diagnosis is\n(A) left-on-left sacral torsion (B) left-on-right sacral torsion (C) right unilateral sacral flexion (D) right-on-right sacral torsion\nA: Let's think step by step. We refer to Wikipedia articles on medicine for help. The deep sulcus on the left, a posterior ILA on the right, with a negative spring test suggests a right-on-right sacral torsion. All other options have a deep sulcus on the right. The answer is (D).\n\nQ: A 44-year-old man comes to the office because of a 3-day history of sore throat, nonproductive cough, runny nose, and frontal headache. He says the headache is worse in the morning and ibuprofen does provide some relief. He has not had shortness of breath. Medical history is unremarkable. He takes no medications other than the ibuprofen for pain. Vital signs are temperature 37.4\u00b0C (99.4\u00b0F), pulse 88/min, respirations 18/min, and blood pressure 120/84 mm Hg. Examination of the nares shows erythematous mucous membranes. Examination of the throat shows erythema and follicular lymphoid hyperplasia on the posterior oropharynx. There is no palpable cervical adenopathy. Lungs are clear to auscultation. Which of the following is the most likely cause of this patient's symptoms?\n(A) Allergic rhinitis (B) Epstein-Barr virus (C) Mycoplasma pneumonia (D) Rhinovirus\nA: Let's think step by step. We refer to Wikipedia articles on medicine for help. The symptoms, especially the headache, suggest that the most likely cause is Rhinovirus. Epstein-Barr virus will cause swollen lymph nodes but there is no palpable cervical adenopathy. Lungs are clear to auscultation suggests it's not Mycoplasma pneumonia. The answer is (D).\n\nQ: A previously healthy 32-year-old woman comes to the physician 8 months after her husband was killed in a car crash. Since that time, she has had a decreased appetite and difficulty falling asleep. She states that she is often sad and cries frequently. She has been rechecking the door lock five times before leaving her house and has to count exactly five pieces of toilet paper before she uses it. She says that she has always been a perfectionist but these urges and rituals are new. Pharmacotherapy should be targeted to which of the following neurotransmitters?\n(A) Dopamine (B) Glutamate (C) Norepinephrine (D) Serotonin\nA: Let's think step by step. We refer to Wikipedia articles on medicine for help. The patient feels sad and among the options, only Dopamine and Serotonin can help increase positive emotions. Serotonin also affects digestion and metabolism, which can help the patient's decreased appetite and sleep difficulty. The answer is (D).\n\nQ: A 42-year-old man comes to the office for preoperative evaluation prior to undergoing adrenalectomy scheduled in 2 weeks. One month ago, he received care in the emergency department for pain over his right flank following a motor vehicle collision. At that time, blood pressure was 160/100 mm Hg and CT scan of the abdomen showed an incidental 10-cm left adrenal mass. Results of laboratory studies, including complete blood count, serum electrolyte concentrations, and liver function tests, were within the reference ranges. The patient otherwise had been healthy and had never been told that he had elevated blood pressure. He takes no medications. A follow-up visit in the office 2 weeks ago disclosed elevated urinary normetanephrine and metanephrine and plasma aldosterone concentrations. The patient was referred to a surgeon, who recommended the adrenalectomy. Today, vital signs are temperature 36.6\u00b0C (97.9\u00b0F), pulse 100/min, respirations 14/min, and blood pressure 170/95 mm Hg. Physical examination discloses no significant findings. Initial preoperative preparation should include treatment with which of the following?\n(A) Labetalol (B) A loading dose of potassium chloride (C) Nifedipine (D) Phenoxybenzamine\nA: Let's think step by step. We refer to Wikipedia articles on medicine for help. The symptoms and the adrenal mass suggested pheochromocytoma, and the blood pressure indicates hypertension. Phenoxybenzamine is used to treat hypertension caused by pheochromocytoma. The answer is (D).\n\n", "professional_psychology": "The following are multiple choice questions (with answers) about professional psychology.\n\nQ: In the construction of a multiple regression equation for purposes of prediction, the optimal combination of measures is one in which the predictors\n(A) are uncorrelated with each other but are moderately correlated with the criterion (B) have low correlations with each other and low correlations with the criterion (C) are highly intercorrelated with each other and moderately correlated with the criterion (D) have low correlations with the criterion bur are moderately correlated with each other\nA: Let's think step by step. We refer to Wikipedia articles on psychology for help. The basis of multiple regression is to assess the relationship between one continuous variable and a set of independent variables. So the predictors should be uncorrelated with each other but are moderately correlated with the criterion. The answer is (A).\n\nQ: There are three ways to measure the Central Tendency: the Mean, the Median and the Mode. From your knowledge about them, what is the mode?\n(A) less sensitive to extreme scores than the mean (B) more useful for skewed distributions (C) sensitive to extreme values and highly skewed distributions (D) the most frequently occurring number\nA: Let's think step by step. We refer to Wikipedia articles on psychology for help. The definition of mode is the most frequently occurring number. The answer is (D).\n\nQ: Carl Jung believed that a client's transference:\n(A) is a fantasy that distracts the client from reality. (B) represents \u201cmixed feelings\u201d toward the therapist. (C) \"is a form of \"\"acting out.\"\"\" (D) reflects the client\u2019s personal and collective unconscious.\nA: Let's think step by step. We refer to Wikipedia articles on psychology for help. Transference is a phenomenon that a person's feelings are unconsciously redirected, so it reflects the client's personal and collective unconscious. The answer is (D).\n\nQ: In terms of Hofstede\u2019s (1980) five cultural dimensions, the United States scores at the top of the scale on:\n(A) individualism. (B) individualism and power distance. (C) power distance and masculinity. (D) uncertainty avoidance.\nA: Let's think step by step. We refer to Wikipedia articles on psychology for help. US scores highest on individualism among the five cultural dimensions. The answer is (A).\n\nQ: One of your therapy clients asks your advice about a good weight- reduction program. You have investigated the programs in the community and are enrolled in the one you consider the best. This program offers a $50 bonus to its patrons for each new person they bring into the program. Under these circumstances, your most appropriate response would be to\n(A) tell your client the pros and cons of each program you know about except for the one in which you are enrolled (B) recommend to your client the program in which you are enrolled and explain the $50 bonus you will receive (C) recommend to your client the program in which you are enrolled and offer to have the $50 bonus credited to your client's account in the program (D) tell your client the pros and cons of each program you know about, but do not claim the $50 bonus if your client enrolls in your program\nA: Let's think step by step. We refer to Wikipedia articles on psychology for help. Based on the circumstances, you should tell your client about the pros and cons of each program, but it would be inappropriate to receive the bonus, so you should not claim the $50 bonus. The answer is (D).\n\n", "public_relations": "The following are multiple choice questions (with answers) about public relations.\n\nQ: Earth Hour was a campaign launched by which organization?\n(A) Greenpeace (B) The UN (C) Oxfam (D) World Wildlife Fund\nA: Let's think step by step. We refer to Wikipedia articles on public relations for help. Earth Hour is a worldwide movement oragnized launched by the World Wildlife Fund. The answer is (D).\n\nQ: In issues management, what is the most proactive approach to addressing negative or misleading information posted online about your organization?\n(A) Buy domain names that could be used by opposition groups. (B) Post anonymous comments on blogs to combat this information. (C) Prepare a news release that discredits the inaccurate information. (D) Make policy changes to address complaints highlighted on these sites.\nA: Let's think step by step. We refer to Wikipedia articles on public relations for help. In issues management, the most proactive approach to addressing negative or misleading information posted online is to make policy changes to address complaints highlighted on those sites. The answer is (D).\n\nQ: At which stage in the planning process would a situation analysis be carried out?\n(A) Defining the program (B) Planning the program (C) Taking action and implementing ideas (D) Evaluation of the program\nA: Let's think step by step. We refer to Wikipedia articles on public relations for help. Situation analyses are typically carried out during the planning process stage of defining the program. The answer is (A).\n\nQ: Which of these statements is true of the Vatican in 2010 at the time of the accusations of child abuse cover-ups?\n(A) There was a coordinated media response. (B) Consistent messages were communicated. (C) Criticisms were taken as attacks on the Catholic Church. (D) The credibility of the Vatican was upheld.\nA: Let's think step by step. We refer to Wikipedia articles on public relations for help. In 2010 when there were accusations of child abuse cover-ups, the Vatican took those criticisms as attacks on the Catholic Church. The answer is (C).\n\nQ: What should a public relations media practitioner do if she does not know the answer to a reporter's question?\n(A) Give the reporter other information she is certain is correct. (B) Say that the information is 'off the record' and will be disseminated later. (C) Say 'I don't know' and promise to provide the information later. (D) Say 'no comment,' rather than appear uninformed.\nA: Let's think step by step. We refer to Wikipedia articles on public relations for help. If a public relations media practitioner does not know the answer to a reporter's question, they should say 'I don't know' and offer to provide the information later. The answer is (C).\n\n", "security_studies": "The following are multiple choice questions (with answers) about security studies.\n\nQ: What are the frameworks of analysis within which terrorism has been considered (as of 2020)?\n(A) Competition between larger nations has resulted in some countries actively supporting terrorist groups to undermine the strength of rival states. Terrorist networks are extended patronage clubs maintained and paid for by their donor states and are conceptualised as being like state actors, to be dealt with using military force. (B) Globalization has enabled the internationalization of terrorist activities by opening up their operational space, although coordination is still managed from a geographical base. This suggests that terrorist groups are nationally structured which means that terrorism cannot be considered in terms of a war to be defeated militarily without having serious implications on the indigenous population. (C) Terrorism can be viewed as a problem to be resolved by military means (war on terrorism), by normal police techniques (terrorism as crime), or as a medical problem with underlying causes and symptoms (terrorism as disease). (D) Terrorism is viewed as a criminal problem. The criminalization of terrorism has two important implications. Firstly, it suggests that terrorism can be eradicated - terrorists can be caught and brought to trial by normal judicial proceedings thereby removing the threat from society - and secondly, it suggests that preventative crime techniques are applicable to prevent its development.\nA: Let's think step by step. We refer to Wikipedia articles on security studies for help. (A) is wrong because it is not competition between larger nations that causes terrorism. \n(B) is wrong because globalization is not the cause of terrorism.\n(C) is correct because the US undertook the war on terrorism. \n(D) is wrong because preventative crime techniques will likely not end terrorism. The answer is (C).\n\nQ: Which of the following is the best lens through which to investigate the role of child soldiers?\n(A) Child soldiers are victims of combat that need re-education and rehabilitation. (B) Children and their mothers are not active subjects in warfare and are best considered as subjects in the private sphere. (C) Children are most often innocent bystanders in war and are best used as signifiers of peace. (D) Children have political subjecthood that is missed when they are considered as passive victims of warfare.\nA: Let's think step by step. We refer to Wikipedia articles on security studies for help. Child soliders as a political topic can be missed when they are considered passive victims of warfare. The answer is (D).\n\nQ: How can we best describe the relationship between the state-centric approach and the concept of human security?\n(A) There are such wide divisions within the human security framework regarding the nature of threats and referent objects that no widely applicable comparisons between state-centric approaches and human security can be drawn. (B) By adopting the framework of human security, the limitations of the realist state-centric approach become evident. Whilst human security defines the referent object as the person or population, state-centric approaches prioritise the security of the state, de-prioritizing the pursuit of human security. (C) The state-centric approach to security is a faction of human security, usually defined within the broad school of human security. By being state-centric this approach prioritises the individual as the referent object in security studies. (D) Both the state-centric and human-centric approaches to security are mutually exclusive and offer a sufficient analytic framework with which to understand the international security system. It is therefore the role of security analysts to determine which of these substantial concepts is correct, and which should be discarded.\nA: Let's think step by step. We refer to Wikipedia articles on security studies for help. Human security focuses on a person or population whereas state-centric approaches focus on the state while deprioritizing human security. The answer is (B).\n\nQ: In order to become securitized, a threat must be presented in which of these ways?\n(A) As an existential threat that requires immediate and extraordinary action, posing a threat to the survival of the state or to societal security. (B) As requiring immediate and extraordinary action by the state, threatening the survival of a referent object and therefore warranting the use of measures not normally employed in the political realm. (C) As an urgent threat to the survival of the referent object, so serious that it legitimises the employment of extraordinary action in response. (D) As an urgent threat to the survival of the audience that requires extraordinary or emergency measures.\nA: Let's think step by step. We refer to Wikipedia articles on security studies for help. To be securitized, a threat must be an urgent threat to the survival of the referent object. The answer is (C).\n\nQ: What distinguishes coercive diplomacy from military force?\n(A) Compellence is another term for coercive diplomacy, but covering a narrower set of criteria; compellence covers those threats aimed at initiating adversary action. A threat to coerce a state to give up part of its territory would count as coercive diplomacy, as long as that threat proactively initiates action before reactive diplomacy is taken. (B) Coercive diplomacy constitutes the threats of limited force to induce adversary's incentive to comply with the coercer's demands. It is an influence strategy that is intended to obtain compliance: the use of force to defeat an opponent first does not count. It leaves an element of choice with the target to comply, or to continue. (C) Military force, or the threat of military force, utilises fear to achieve strategic objectives. Coercive diplomacy is differentiated from this approach, because it does not use fear as a tool for coercing an adversary. (D) Coercive diplomacy is employed to use force but to limit its effects on the international community. Coercive diplomacy is an aggressive strategy that is intended to obtain compliance through defeat. It does not leave an element of choice with the target, the target either being forced to comply or engage in conflict. It seeks to control by imposing compliance by removing any opportunity for negotiation or concession.\nA: Let's think step by step. We refer to Wikipedia articles on security studies for help. Coercive diplomacy uses the threat of force to induce the opponent to comply with demands. The answer is (B).\n\n", "sociology": "The following are multiple choice questions (with answers) about sociology.\n\nQ: Which of the following is not a problem associated with official statistics on strike action?\n(A) most strikes go unnoticed by employers and the mass media (B) not all industrial disputes will be reported by the employer (C) the definition of strikes excludes those that involve fewer than ten workers or last less than one day (D) it is hard to compare strikes that were measured in different ways\nA: Let's think step by step. We refer to Wikipedia articles on sociology for help. Official statistics on strike action can be problematic because not all industrial disputes will be reported by employers, the definition of strikes excludes those that involves fewer than ten workers or last less than one day, and it is hard to compare strikes that were measured in different ways. Thus, (A) is not a problem associated with official statistics on strike action. The answer is (A).\n\nQ: What does Berger (1963) describe as a metaphor for social reality?\n(A) a fairground ride (B) a circus (C) a puppet theatre (D) a ballet\nA: Let's think step by step. We refer to Wikipedia articles on sociology for help. Berger describes social reality using the metaphor of a puppet theatre. The answer is (C).\n\nQ: The term 'hegemony' refers to:\n(A) the tendency for the working class not to realize their own interests (B) a dominant ideology that legitimates economic, political and cultural power (C) a form of dual consciousness based on ideology and everyday experiences (D) a mode of payment given for outstanding topiary\nA: Let's think step by step. We refer to Wikipedia articles on sociology for help. Hegemony refers to a dominant ideology that legitimates economic, policital, and cultural power. The answer is (B).\n\nQ: The shift from 'civil religion' to 'common religion' means that:\n(A) the increasing bureaucracy of the state has made religion only a marginal part of our lives (B) despite the weakening of traditional authority, our everyday lives and 'common sense' remain shaped by religious beliefs and values (C) religious participation in collective worship may have declined, but people still practise their faiths in private (D) people are much more likely to discuss their religious beliefs in public, informal settings\nA: Let's think step by step. We refer to Wikipedia articles on sociology for help. The shift from civil religion to common religion means that despite the weakening of traditional authority, our everyday lives and common sense remain shaped by religious beliefs and values. The answer is (B).\n\nQ: Which of the following did the post-war welfare state of 1948 not aim to provide:\n(A) free health care and education for all (B) a minimum wage (C) full employment (D) universal welfare\nA: Let's think step by step. We refer to Wikipedia articles on sociology for help. The post-war welfare state of 1948 aimed to provide free healthcare and education, full employment, and universal welfare. But it did not aim to provide a minimum wage. The answer is (B).\n\n", "us_foreign_policy": "The following are multiple choice questions (with answers) about us foreign policy.\n\nQ: How did Donald Trump attack globalization in the 2016 campaign?\n(A) Globalization had made men like him too rich (B) Globalization only benefited certain American states, such as New York (C) Liberal elites had encouraged globalization, while 'ordinary Americans' lost jobs because of it (D) Globalization encouraged damaging trade wars\nA: Let's think step by step. We refer to Wikipedia articles on us foreign policy for help. Trump attacked globalization because he believed ordinary Americans lost jobs due to it, and so he wanted to blame liberals who had encouraged it. The answer is (C).\n\nQ: How did NSC-68 change U.S. strategy?\n(A) It globalized containment. (B) It militarized containment. (C) It called for the development of the hydrogen bomb. (D) All of the above\nA: Let's think step by step. We refer to Wikipedia articles on us foreign policy for help. NSC-68 outlined a variety of courses of action, including globalization of containment, militarization of contaiment, and the development of the hydrogen bomb. The answer is (D).\n\nQ: How do Defensive Realism and Offensive Realism differ in their explanation of state behaviour?\n(A) Defensive realists place greater emphasis on the role of international institutions (B) Defensive realists place less emphasis on geographical factors (C) Offensive realists give more priority to the national interest than Defensive realists. (D) Defensive realists believe states are security maximizers, while Offensive realists believe states to be power maximizers\nA: Let's think step by step. We refer to Wikipedia articles on us foreign policy for help. While defensive realism advocates that states are security maximizers, offensive realists think of states as power maximizers. The answer is (D).\n\nQ: The realm of policy decisions concerned primarily with relations between the United States and the rest of the world is known as\n(A) terrorism policy. (B) economic policy. (C) foreign policy. (D) international policy.\nA: Let's think step by step. We refer to Wikipedia articles on us foreign policy for help. The topic of policy decisions concerns with relations between the US and the rest of the world is known as foreign policy. The answer is (C).\n\nQ: How did the 2008 financial crisis affect America's international reputation?\n(A) It damaged support for the US model of political economy and capitalism (B) It created anger at the United States for exaggerating the crisis (C) It increased support for American global leadership under President Obama (D) It reduced global use of the US dollar\nA: Let's think step by step. We refer to Wikipedia articles on us foreign policy for help. The 2008 financial crisis damanged the international reputation of the American model of political economy and capitalism. The answer is (A).\n\n", "virology": "The following are multiple choice questions (with answers) about virology.\n\nQ: The median survival time to AIDS and death was established by following:\n(A) Seroprevalent HIV-infected individuals (B) Seronegatives (C) Seroconverters (D) High-risk seronegatives\nA: Let's think step by step. We refer to Wikipedia articles on virology for help. The median survival time to AIDS and death was established as a result of the development of seroconverters. The answer is (C).\n\nQ: Which of the following is a morphological characteristic of the paramyxoviruses.\n(A) Fragile viruses often visualised with RNA spewing from the inside (B) Elongate viruses (C) Icosahedral viruses with envelope (D) Very large viruses\nA: Let's think step by step. We refer to Wikipedia articles on virology for help. Paramyxoviruses are fragile viruses often visualised with RNA spewing from the inside. The answer is (A).\n\nQ: The most important goal of a behavioral intervention is:\n(A) Change in behavior (B) Comprehensive coverage (C) Effective use of behavioral theory (D) Sustained behavior change\nA: Let's think step by step. We refer to Wikipedia articles on virology for help. The prim goal of a behavioral intervention is to cause sustained behavior change. The answer is (D).\n\nQ: A key factor facilitating the application of nested case-control studies from the MACS was:\n(A) Data collection (B) Establishment of a repository of biologic specimens (C) Participant interest (D) Administration of the questionnaire by staff\nA: Let's think step by step. We refer to Wikipedia articles on virology for help. The Multicenter AIDS Cohort Study's use of nested case-control studies was facilitated by the establishment of a repository of biologic specimens. The answer is (B).\n\nQ: Why are parvoviruses a highly impactful parasite?\n(A) Because they have no nucleic acid (B) They require a helper virus (C) Only replicate in dividing cells (D) Can integrate into host chromosomes\nA: Let's think step by step. We refer to Wikipedia articles on virology for help. Paroviruses are highly impactful because they do not have nucleic acid. The answer is (A).\n\n", "world_religions": "The following are multiple choice questions (with answers) about world religions.\n\nQ: How can the Upanishads be characterized?\n(A) Ritual texts (B) Philosophical texts (C) Hymns (D) Origin stories\nA: Let's think step by step. We refer to Wikipedia articles on world religions for help. The Upanishads are the most recent part of Vedas (the oldest scriptures in Hinduism) and supplied the basis of later Hindu philosophy. So they are philosophical texts. The answer is (B).\n\nQ: What is the Second Gem in Buddhism?\n(A) The Dharma (B) The Sangha (C) The Buddha (D) The Bodhisattva\nA: Let's think step by step. We refer to Wikipedia articles on world religions for help. The Second Gem in Buddhism is The Dharma. The answer is (A).\n\nQ: Which Japanese government promoted a kind of national cult based on the emperor and his associations with kami?\n(A) Honen (B) Tanaka (C) Tokugawa (D) Meiji\nA: Let's think step by step. We refer to Wikipedia articles on world religions for help. The promotion of a national cult based on the emperor and his associations with Kami happened during the reign of Emperor Meiji (1852-1912). The answer is (D).\n\nQ: In which dynasty was the \"Mandate of Heaven\" developed to legitimatize the new rulers?\n(A) Shang (B) Zhou (C) Han (D) Xia\nA: Let's think step by step. We refer to Wikipedia articles on world religions for help. The \"Mandate of Heaven\" was developed as an ancient Chinese philosophical concept during the Zhou Dynasty (1046-256 BCE). The answer is (B).\n\nQ: What is the sign of the covenant for Jewish males?\n(A) The rainbow (B) Circumcision (C) A son (D) Bar mitzvah\nA: Let's think step by step. We refer to Wikipedia articles on world religions for help. In Judaism, the most distinctive sign of the covenant is circumcision (brit milah). The answer is (B).\n\n"} diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/_mmlu.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/_mmlu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c6c1c6a19dc7638adfa630ce80b58294f5b351b8 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/_mmlu.yaml @@ -0,0 +1,34 @@ +group: mmlu_flan_cot_fewshot +group_alias: mmlu (flan style, fewshot cot) +task: + - group: stem + task: + - mmlu_flan_cot_fewshot_stem + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: other + task: + - mmlu_flan_cot_fewshot_other + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: social sciences + task: + - mmlu_flan_cot_fewshot_social_sciences + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: humanities + task: + - mmlu_flan_cot_fewshot_humanities + aggregate_metric_list: + - metric: acc + weight_by_size: True +aggregate_metric_list: + - aggregation: mean + metric: exact_match + weight_by_size: True + filter_list: get-answer +metadata: + version: 2 diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/_mmlu_flan_cot_fewshot_template_yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/_mmlu_flan_cot_fewshot_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..cfbf222e5ba5892a5e26113d382ea86ec1300ce0 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/_mmlu_flan_cot_fewshot_template_yaml @@ -0,0 +1,30 @@ +dataset_path: hails/mmlu_no_train # a copy of `cais/mmlu` with no auxiliary_train split +validation_split: validation +test_split: test +fewshot_config: + sampler: first_n +output_type: generate_until +doc_to_text: "{% if choices is defined%}Q: {{question.strip()}}\n(A) {{choices[0]}} (B) {{choices[1]}} (C) {{choices[2]}} (D) {{choices[3]}}\nA: Let's think step by step.{% else %}Q: {{ question.strip() }}\nA:{% endif %}" +doc_to_target: "{{['(A)', '(B)', '(C)', '(D)'][answer] if answer is defined else target}}" +filter_list: + - name: "get-answer" + filter: + - function: "regex" + regex_pattern: "(?<=answer is )(.*)(?=.)" + - function: "take_first" +generation_kwargs: + until: + - "" + do_sample: false + temperature: 0.0 +num_fewshot: 4 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 2.0 +dataset_kwargs: + trust_remote_code: true diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_abstract_algebra.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6235d5c0997558a123258cba3dfdb4b844a2fb60 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_abstract_algebra.yaml @@ -0,0 +1,59 @@ +dataset_name: abstract_algebra +description: The following are multiple choice questions (with answers) about abstract + algebra. +fewshot_config: + sampler: first_n + samples: + - question: 'Statement 1 | Every element of a group generates a cyclic subgroup of + the group. Statement 2 | The symmetric group S_10 has 10 elements. + + (A) True, True (B) False, False (C) True, False (D) False, True' + target: Let's think step by step. A cyclic group is a group that is generated + by a single element. Hence a subgroup generated by a single element of a group + is cyclic and Statement 1 is True. The answer is (C). + - question: 'The symmetric group $S_n$ has $ + + actorial{n}$ elements, hence it is not true that $S_{10}$ has 10 elements. + + Find the characteristic of the ring 2Z. + + (A) 0 (B) 3 (C) 12 (D) 30' + target: Let's think step by step. A characteristic of a ring is R is $n$ if the + statement $ka = 0$ for all $a\in 2Z$ implies that $k$ is a multiple of $n$. + Assume that $ka = 0$ for all $a\in 2Z$ for some $k$. In particular $2k = 0$. + Hence $k=0$ and $n=0$. The answer is (A). + - question: 'Statement 1| Every function from a finite set onto itself must be one + to one. Statement 2 | Every subgroup of an abelian group is abelian. + + (A) True, True (B) False, False (C) True, False (D) False, True' + target: "Let's think step by step. Statement 1 is true. Let $S$ be a finite set.\ + \ If $f:S \nightarrow S$ is a onto function, then $|S| = |f(S)|$. If $f$ was\ + \ not one to one, then for finite domain $S$ the image would have less than\ + \ $S$ elements, a contradiction.\nStatement 2 is true. Let $G$ be an abelian\ + \ group and $H$ be a subgroup of $G$. We need to show that $H$ is abelian. Let\ + \ $a,b \\in H$. Then $a,b \\in G$ and $ab=ba$. Since $G$ is abelian, $ab=ba$.\ + \ Since $H$ is a subgroup of $G$, $ab \\in H$. Therefore, $ab=ba$ and $H$ is\ + \ abelian. The answer is (A)." + - question: 'Statement 1 | If aH is an element of a factor group, then |aH| divides + |a|. Statement 2 | If H and K are subgroups of G then HK is a subgroup of G. + + (A) True, True (B) False, False (C) True, False (D) False, True' + target: Let's think step by step. Statement 2 is false. Let $H$ be a subgroup + of $S_3$ generated by the cycle $(1,2)$ and $K$ be a subgroup of $S_3$ generated + by the cycle $(1,3)$. Both $H$ and $K$ have two elements, the generators and + the identity. However $HK$ contains cycles (1,2), (1,3) and (2,3,1), but the + inverse of (2,3,1) is (2,1,3) and it does not belong to HK, hence HK is not + a subgroup. The answer is (B). + - question: 'Find all c in Z_3 such that Z_3[x]/(x^2 + c) is a field. + + (A) 0 (B) 1 (C) 2 (D) 3' + target: 'Let''s think step by step. Z_3[x]/(x^2 + c) is a field if and only if + x^2 + c does not have roots in Z_3. That is x^2 + c != 0 for every x in Z_3. + If c = 0, then x^2 + c = x^2 has root 0. If c = 1 then x^2 + c = x^2 + 1 = 0 + + 1 for x = 0, 1 + 1 = 2 for x = 1 and 1 + 1 = 2 for x = 2, hence x^2 + 1 does + not have any roots. For c = 2 the polynomial x^2 + 2 has two roots at x = 1 + and x = 2. Hence Z_3[x]/(x^2 + c) is a field if and only if c = 1. The answer + is (B).' +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_abstract_algebra diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_anatomy.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e6521bdebf5efa653c7bc798fa7c4ecb985e4166 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_anatomy.yaml @@ -0,0 +1,75 @@ +dataset_name: anatomy +description: The following are multiple choice questions (with answers) about anatomy. +fewshot_config: + sampler: first_n + samples: + - question: 'Which of the following is the body cavity that contains the pituitary + gland? + + (A) Abdominal (B) Cranial (C) Pleural (D) Spinal' + target: "Let's think step by step. We refer to Wikipedia articles on anatomy for\ + \ help. Let\u2019s solve this problem step by step. The pituitary gland is the\ + \ major endocrine gland attached to the base of the brain, and it is contained\ + \ in the Cranial cavity. The answer is (B)." + - question: 'Which of these branches of the trigeminal nerve contain somatic motor + processes? + + (A) The supraorbital nerve (B) The infraorbital nerve (C) The mental nerve (D) + None of the above' + target: "Let's think step by step. We refer to Wikipedia articles on anatomy for\ + \ help. Let\u2019s solve this problem step by step. \nWe know the following:\ + \ (A) The supraorbital nerve (also known as the frontal nerve) is the largest\ + \ branch of the ophthalmic nerve and branch of ophthalmic division of the trigeminal\ + \ nerve. (B) The infraorbital nerve is a branch of the maxillary division of\ + \ the trigeminal nerve. (C) The mental nerve is a branch of the mandibular division\ + \ of the trigeminal nerve. Because all these nerves are purely sensory nerves\ + \ and do not contain any somatic motor processes. Therefore, the answer should\ + \ be none of the above, which is (D). The answer is (D)." + - question: 'In Angle''s Class II Div 2 occlusion there is + + (A) excess overbite of the upper lateral incisors. (B) negative overjet of the + upper central incisors. (C) excess overjet of the upper lateral incisors. (D) + excess overjet of the upper central incisors.' + target: "Let's think step by step. We refer to Wikipedia articles on anatomy for\ + \ help. Let\u2019s solve this problem step by step. This is a question related\ + \ to anatomy and orthodontics. Excess overjet is associated with Class II occlusions;\ + \ therefore, we can safely eliminate (B) from the list, as negative overjet\ + \ is often associated with Class III occlusions. Now, we need to determine the\ + \ location of the excess overjet, and that would be the upper (maxillary) lateral\ + \ incisors. Only (C) has the correct information. The answer is (C)." + - question: 'The pleura + + (A) have no sensory innervation. (B) are separated by a 2 mm space. (C) extend + into the neck. (D) are composed of respiratory epithelium.' + target: "Let's think step by step. We refer to Wikipedia articles on anatomy for\ + \ help. Let\u2019s solve this problem step by step. First, recall that the pleura\ + \ refers to the thin layer of tissue that covers the lungs and lines the interior\ + \ wall of the chest cavity. Now, let\u2019s look at each option:\nOption (A):\ + \ \u201CThe pleura have no sensory innervation.\u201D This information is not\ + \ correct. The pleura do have a sensory innervation.\nOption (B): \u201CThe\ + \ pleura are separated by a 2 mm space.\u201D This information is not correct.\ + \ There is a very thin \u201Cpotential\u201D space between the layers of the\ + \ pleura; however, it is typically filled with serous pleural fluid. \nOption\ + \ (C): \u201CThe pleura extend into the neck.\u201D This information is actuakky\ + \ true. The cervical pleura, also known as the dome of the pleuradome of the\ + \ pleura, lines the extendsiton of the pleural cavity into the neck.\nOption\ + \ (D): \u201CThe pleura are composed of respiratory epithelium.\u201D This information\ + \ is not correct. The pleaura are composed of connective tissue (CT).\nBecause\ + \ (A), (B), and (D) are all incorrect, (D) is the only correct answer. The answer\ + \ is (C)." + - question: 'What is the embryological origin of the hyoid bone? + + (A) The first pharyngeal arch (B) The first and second pharyngeal arches (C) + The second pharyngeal arch (D) The second and third pharyngeal arches' + target: "Let's think step by step. We refer to Wikipedia articles on anatomy for\ + \ help. Let\u2019s solve this problem step by step. The hyoid bone, which is\ + \ also known as the hyooid, is a a small U-shaped bone located in the anterior\ + \ neck. In its resting position, it lies between the ase of the mandible and\ + \ the third cervical vertebrae. We know that the second and the third pharyngeal\ + \ arches give rise to the horns of the hyoid bone; therefore, the embryological\ + \ origin of the hyoid bone are the second and the third pharyngeal arches\u2014\ + this information is covered in the last option (D). Therefore, we conclude that\ + \ (D) must be the correct answer. The answer is (D).\n\n" +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_anatomy diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_astronomy.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b89974588e8c83db8aedc80b607e25212a676592 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_astronomy.yaml @@ -0,0 +1,70 @@ +dataset_name: astronomy +description: The following are multiple choice questions (with answers) about astronomy. +fewshot_config: + sampler: first_n + samples: + - question: 'Where do most short-period comets come from and how do we know? + + (A) The Kuiper belt; short period comets tend to be in the plane of the solar + system just like the Kuiper belt. (B) The Kuiper belt; short period comets tend + to come from random directions indicating a spherical distribution of comets + called the Kuiper belt. (C) The asteroid belt; short period comets have orbital + periods similar to asteroids like Vesta and are found in the plane of the solar + system just like the asteroid belt. (D) The Oort cloud; short period comets + tend to be in the plane of the solar system just like the Oort cloud.' + target: Let's think step by step. Most short-period comets come from the Kuiper + belt, and we know because short period coments tend to be in the plane of the + solar system, just like the Kuiper belt is. The answer is (A). + - question: 'You are pushing a truck along a road. Would it be easier to accelerate + this truck on Mars? Why? (Assume there is no friction) + + (A) It would be harder since the truck is heavier on Mars. (B) It would be easier + since the truck is lighter on Mars. (C) It would be harder since the truck is + lighter on Mars. (D) It would be the same no matter where you are.' + target: "Let's think step by step. If we assume that there is no friction, the\ + \ force needed to accelerate the truck is by Newton\u2019s second law only dependent\ + \ on the mass of the truck. Hence (A), (B) and (C) are incorrect since it doesn\u2019\ + t matter that it\u2019s on Mars, and (D) is the correct answer. The answer is\ + \ (D)." + - question: 'Say the pupil of your eye has a diameter of 5 mm and you have a telescope + with an aperture of 50 cm. How much more light can the telescope gather than + your eye? + + (A) 10000 times more (B) 100 times more (C) 1000 times more (D) 10 times more' + target: Let's think step by step. The amount of light is proportional to the aperture + area $A = \pi D^2/4$ for a lens with diameter $D$, so the relative amounts of + light between the eye with diameter 5mm and the telescope with diameter 50mm + is $(50 cm)^2/(5mm)^2 = 10000$. The answer is (A). + - question: 'Why isn''t there a planet where the asteroid belt is located? + + (A) A planet once formed here but it was broken apart by a catastrophic collision. + (B) There was not enough material in this part of the solar nebula to form a + planet. (C) There was too much rocky material to form a terrestrial planet but + not enough gaseous material to form a jovian planet. (D) Resonance with Jupiter + prevented material from collecting together to form a planet.' + target: "Let's think step by step. The asteroid belt is a stellar disc consisting\ + \ of a large number of asteroids between Mars and Jupiter's orbits. The asteroids\ + \ in this belt are affected by the gravitational pull from both other asteroids\ + \ and nearby planets. Due to the strong gravitational force of Jupiter there\ + \ are resonances that give rise to low density regions of asteroids known as\ + \ the Kirkwood gap. So (B) and (C) are not correct since it\u2019s not a lack\ + \ of material that prevents a planet from being formed, and (A) is incorrect\ + \ because the Kirkwood gap would have prevented a planet from forming in the\ + \ first place, and (D) is the correct option. The answer is (D)." + - question: 'Why is Mars red? + + (A) Because the surface is covered with heavily oxidized ("rusted") minerals. + (B) Because the atmosphere scatters more light at bluer wavelengths transmitting + mostly red light. (C) Because Mars is covered with ancient lava flows which + are red in color. (D) Because flowing water on Mars''s surface altered the surface + minerals several billion years ago.' + target: 'Let''s think step by step. Option (B) is not correct because if the red + color was caused by the scattering off the atmosphere, then the earth with a + much thicker atmosphere would also look red. Options (C) and (D) are not specific + enough about why the color of the surface would be red, while (A) is correct + because it explains that the surface is red due to the rusted materials on the + surface and the red color comes from the rust. So the correct option is (A). + The answer is (A).' +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_astronomy diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_business_ethics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6c83d4bc8c06531a9375f52d8688f8b4a7cdb974 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_business_ethics.yaml @@ -0,0 +1,75 @@ +dataset_name: business_ethics +description: The following are multiple choice questions (with answers) about business + ethics. +fewshot_config: + sampler: first_n + samples: + - question: 'In contrast to _______, _______ aim to reward favourable behaviour by + companies. The success of such campaigns have been heightened through the use + of ___________, which allow campaigns to facilitate the company in achieving + _________ . + + (A) Buycotts, Boycotts, Blockchain technology, Charitable donations (B) Buycotts, + Boycotts, Digital technology, Increased Sales (C) Boycotts, Buyalls, Blockchain + technology, Charitable donations (D) Boycotts, Buycotts, Digital technology, + Increased Sales' + target: "Let's think step by step. We refer to Wikipedia articles on business\ + \ ethics for help. The sentence that best uses the possible options above is\ + \ \u201CIn contrast to *boycotts*, *buycotts* aim to reward favourable behavior\ + \ by companies. The success of such campaigns have been heightened through the\ + \ use of *digital technology*, which allow campaigns to facilitate the company\ + \ in achieving *increased sales*.\u201D The answer is (D)." + - question: '_______ is the direct attempt to formally or informally manage ethical + issues or problems, through specific policies, practices and programmes. + + (A) Corporate social responsibility (B) Business ethics management (C) Sustainability + (D) Environmental management' + target: Let's think step by step. We refer to Wikipedia articles on business ethics + for help. The direct attempt manage ethical issues through specific policies, + practices, and programs is business ethics management. The answer is (B). + - question: 'Three contrasting tactics that CSO''s can engage in to meet their aims + are ________ which typically involves research and communication, ________, + which may involve physically attacking a company''s operations or ________, + often involving some form of _______. + + (A) Non-violent direct action, Violent direct action, Indirect action, Boycott + (B) Indirect action, Instrumental action, Non-violent direct action, Information + campaign (C) Indirect action, Violent direct action, Non-violent direct-action + Boycott (D) Non-violent direct action, Instrumental action, Indirect action, + Information campaign' + target: "Let's think step by step. We refer to Wikipedia articles on business\ + \ ethics for help. The sentence that best uses the possible options above is\ + \ \u201CThree contrasting tactics that CSO's can engage in to meet their aims\ + \ are *indirect action*, which typically involves research and communication,\ + \ *violent direct action*, which may involve physically attacking a company's\ + \ operations or *non-violent direct action*, often involving some form of *boycott*.\u201D\ + \ The answer is (C)." + - question: 'To ensure the independence of the non-executive board members, there are + a number of steps which can be taken, which include non-executives being drawn + from _______ the company, being appointed for a _________ time period as well + as being appointed _________. + + (A) Outside, Limited, Independently (B) Inside, Limited, Intermittently (C) + Outside, Unlimited, Intermittently (D) Inside, Unlimited, Independently' + target: "Let's think step by step. We refer to Wikipedia articles on business\ + \ ethics for help. The sentence that best uses the possible options above is\ + \ \u201CTo ensure the independence of the non-executive board members, there\ + \ are a number of steps which can be taken, which include non-executives being\ + \ draw from *outside* the company, being appointed for a *limited* time period\ + \ as well as being imported *independently*. The answer is (A)." + - question: 'Beyond the business case for engaging in CSR there are a number of moral + arguments relating to: negative _______, the _______that corporations possess + and the ________ of business and society. + + (A) Externalities, Power, Independence (B) Publicity, Insubstantial resources, + Mutual dependence (C) Publicity, Power, Independence (D) Externalities, Power, + Mutual dependence' + target: "Let's think step by step. We refer to Wikipedia articles on business\ + \ ethics for help. The sentence that best uses the possible options above is\ + \ \u201CBeyond the business case for engaging the CSR there are a number of\ + \ moral arguments relating to: negative *externalities*, the *power* that corporations\ + \ possess and the *mutual independence* of business and society. The answer\ + \ is (D).\n\n" +tag: mmlu_flan_cot_fewshot_other +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_business_ethics diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_clinical_knowledge.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..008d2f870ad82e3ebee126511418c67c787c7b3b --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_clinical_knowledge.yaml @@ -0,0 +1,48 @@ +dataset_name: clinical_knowledge +description: The following are multiple choice questions (with answers) about clinical + knowledge. +fewshot_config: + sampler: first_n + samples: + - question: 'Glycolysis is the name given to the pathway involving the conversion of: + + (A) glycogen to glucose-1-phosphate. (B) glycogen or glucose to fructose. (C) + glycogen or glucose to pyruvate or lactate. (D) glycogen or glucose to pyruvate + or acetyl CoA.' + target: Let's think step by step. We refer to Wikipedia articles on clinical knowledge + for help. Glycolysis is the name given to the pathway involving conversion of + glycogen or glucose to pyruvate or lactate. The answer is (C). + - question: 'What is the difference between a male and a female catheter? + + (A) Male and female catheters are different colours. (B) Male catheters are + longer than female catheters. (C) Male catheters are bigger than female catheters. + (D) Female catheters are longer than male catheters.' + target: Let's think step by step. We refer to Wikipedia articles on clinical knowledge + for help. The difference between a male and female catheter is that male catheters + tend to be longer than female catheters. The answer is (B). + - question: 'How many attempts should you make to cannulate a patient before passing + the job on to a senior colleague, according to the medical knowledge of 2020? + + (A) 4 (B) 3 (C) 2 (D) 1' + target: Let's think step by step. We refer to Wikipedia articles on clinical knowledge + for help. According to the medical protocol as of 2020, you should make two + attempts to cannulate a patient before passing the job on to a more-senior practitioner. + The answer is (C). + - question: 'In the assessment of the hand function which of the following is true? + + (A) Abduction of the thumb is supplied by spinal root T2 (B) Opposition of the + thumb by opponens policis is supplied by spinal root T1 (C) Finger adduction + is supplied by the median nerve (D) Finger abduction is mediated by the palmar + interossei' + target: Let's think step by step. We refer to Wikipedia articles on clinical knowledge + for help. Of all the options, it is only true that the opposition of the thumb + by opponens pollicis is supplied by spinal root T1. The answer is (B). + - question: 'The energy for all forms of muscle contraction is provided by: + + (A) ATP. (B) ADP. (C) phosphocreatine. (D) oxidative phosphorylation.' + target: 'Let''s think step by step. We refer to Wikipedia articles on clinical + knowledge for help. The energy for muscular contraction is provided by ATP (adenosine + triphosphate), which is the powerhouse of the cell. The answer is (A).' +tag: mmlu_flan_cot_fewshot_other +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_clinical_knowledge diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_biology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..28f7f989b86b28cccc5e9128268e8ea6bc80e662 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_biology.yaml @@ -0,0 +1,75 @@ +dataset_name: college_biology +description: The following are multiple choice questions (with answers) about college + biology. +fewshot_config: + sampler: first_n + samples: + - question: 'Which of the following represents an accurate statement concerning arthropods? + + (A) They possess an exoskeleton composed primarily of peptidoglycan. (B) They + possess an open circulatory system with a dorsal heart. (C) They are members + of a biologically unsuccessful phylum incapable of exploiting diverse habitats + and nutrition sources. (D) They lack paired, jointed appendages.' + target: Let's think step by step. Peptidoglycan is known to comprise the plasma + membrane of most bacteria, rather than the exoskeleton of arthropods, which + is made of chitin, which rules out (A). The answer (C) is false because arthropods + are a highly successful phylum. Likewise, arthropods have paired, jointed appendages, + which rules out (D). The only remaining option is (B), as arthropods have an + open circulatory system with a dorsal tubular heart. The answer is (B). + - question: 'In a given population, 1 out of every 400 people has a cancer caused by + a completely recessive allele, b. Assuming the population is in Hardy-Weinberg + equilibrium, which of the following is the expected proportion of individuals + who carry the b allele but are not expected to develop the cancer? + + (A) 1/400 (B) 19/400 (C) 20/400 (D) 38/400' + target: "Let's think step by step. According to the Hardy Weinberg Law, $p^2 +\ + \ 2 p q + q^2 = 1$, and $p + q = 1$ where $p$ is the frequency of the dominant\ + \ allele, $q$ is the frequency of the recessive allele, and $p^2$, $q^2$, and\ + \ $2pq$ are the frequencies of dominant homozygous, recessive homozygous, and\ + \ heterozygous individuals, respectively. \u200BThe frequency of the recessive\ + \ allele (q) is $\\sqrt{\frac{1}{400}} = 0.05$. We have $p = 1 - q = 0.95$.\ + \ The frequency of heterozygous individuals is $2pq = 2 \\cdot 0.05 \\cdot 0.95\ + \ = 0.095$. The number of heterozygous individuals is equal to the frequency\ + \ of heterozygous individuals times the size of the population, or $0.095 *\ + \ 400 = 38$. So we end up with 38/400. The answer is (D)." + - question: 'According to the pressure-flow model of movement of phloem contents, photosynthate + movement from source to sink is driven by + + (A) an ATP-dependent pressure-flow pump (B) a water-pressure potential gradient + (C) transpiration (D) apoplastic diffusion' + target: Let's think step by step. It is a gradient in water pressure that induces + the movement of phloem content, which refers to answer (B). The mechanism of + movement does not rely on metabolism, which rules out (A). Transpiration refers + to the exhalation of water vapor through plant stomata, and is also not related, + which rules out (C). While the apoplastic pathway is one of two main pathways + for water transport in plants, it is not central to the pressure flow model, + which rules out (D). The answer is (B). + - question: 'Which of the following contain DNA sequences required for the segregation + of chromosomes in mitosis and meiosis? + + (A) Telomeres (B) Centromeres (C) Nucleosomes (D) Spliceosomes' + target: Let's think step by step. The genetic material in Telomeres is not used, + which rules out (A). Nucleosomes are the repeating subunit that comprises chromatin + packed in a cell nucleus, and do not specifically refer to DNA sequences necessary + for segregating chromosomes in cell division, which rules out (C). A spliceosome + is a large ribonucleoprotein that removes introns from transcribed pre-mRNA + rather than governing chromosome segregation. Centromeres are directly responsible + for segregating chromosomes in cell division. The answer is (B). + - question: 'The presence of homologous structures in two different organisms, such + as the humerus in the front limb of a human and a bird, indicates that + + (A) the human and bird are polyphyletic species (B) a human''s and bird''s evolution + is convergent (C) the human and bird belong to a clade (D) the human and bird + developed by analogy' + target: 'Let''s think step by step. Polyphyletic species are organisms that are + grouped due to having similar characteristics but which do not have a common + ancestor. This is not the case for humans and birds, which rules out (A). Convergent + evolution refers to the indepdendent development of similar features in different + species at different periods, which is also not the case for humans and birds, + which rules out (B). Analogy refers to the superficial resemblance of structures + that have different origins, which is not the case for the human and bird forearms, + which rules out (D). Humans and birds do belong to the same clade - a group + of organisms composed of a common ancestor. The answer is (C).' +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_college_biology diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_chemistry.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a8cfc9e4436f1dc50b75efdf84a0b6f0625f2bf --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_chemistry.yaml @@ -0,0 +1,49 @@ +dataset_name: college_chemistry +description: The following are multiple choice questions (with answers) about college + chemistry. +fewshot_config: + sampler: first_n + samples: + - question: "3 Cl\u2212(aq) + 4 CrO_4^2\u2212(aq) + 23 H+(aq) \u2192 3 HClO2(aq) +\ + \ 4 Cr3+(aq) + 10 H2O(l). In the reaction shown above, Cl\u2212(aq) behaves\ + \ as\n(A) an acid (B) a base (C) a catalyst (D) a reducing agent" + target: Let's think step by step. A molecule that behaves as a base accepts an + H+ ion (or proton) from another molecule, whereas a molecule that behaves as + an acid donates an H+ ion (or proton) to another molecule. Neither of these + is the case for Cl in this reaction, which rules out (A) and (B). A catalyst + is a substance that only accelerates a reaction without itself undergoing chemical + change, which is not the case here. This rules out (C). Instead, the $Cl^{-} + molecules carry a negative charge, which they donate in the reaction to form + 3 HClO2. This is the behavior of a reducing agent, or (D). The answer is (D). + - question: 'Which of the following statements about the lanthanide elements is NOT + true? + + (A) The most common oxidation state for the lanthanide elements is +3. (B) Lanthanide + complexes often have high coordination numbers (> 6). (C) All of the lanthanide + elements react with aqueous acid to liberate hydrogen. (D) The atomic radii + of the lanthanide elements increase across the period from La to Lu.' + target: Let's think step by step. The atomic radii of the lanthanide elements + in fact decrease across the period from La to Lu. Options (A), (B), and (C) + are all true. This means that only (D) is NOT true. The answer is (D). + - question: 'Which of the following lists the hydrides of group-14 elements in order + of thermal stability, from lowest to highest? + + (A) PbH4 < SnH4 < GeH4 < SiH4 < CH4 (B) PbH4 < SnH4 < CH4 < GeH4 < SiH4 (C) + CH4 < SiH4 < GeH4 < SnH4 < PbH4 (D) CH4 < PbH4 < GeH4 < SnH4 < SiH4' + target: Let's think step by step. The thermal stability of group-14 hydrides decreases + as we move from the top of group 14 to the bottom. The order of elements in + the group from top to bottom is C, Si, Ge, Sn, Pb. Therefore in order of increasing + thermal stability we have PbH4, SnH4, GeH4, SiH4, and CH4, or answer (A). The + answer is (A). + - question: "Predict the number of lines in the EPR spectrum of a solution of 13C-labelled\ + \ methyl radical (13CH3\u2022), assuming the lines do not overlap.\n(A) 4 (B)\ + \ 3 (C) 6 (D) 24 (E) 8" + target: "Let's think step by step. The electron paramagnetic resonance spectrum\ + \ will be split by two forms of interactions. The first is the hyperfine interaction\ + \ with the 13C (nuclear spin $I = \nrac{1}{2}$) which will split the spectrum\ + \ into 2 lines. This will be further split into 4 lines by the interaction with\ + \ three equivalent 1H nuclei. The total number of lines is therefore $2 \\cdot\ + \ 4 = 8$. The answer is (E).\n\n" +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_college_chemistry diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_computer_science.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5eccde7c6be9620f63bbf3b1de42f32c8e121539 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_computer_science.yaml @@ -0,0 +1,180 @@ +dataset_name: college_computer_science +description: The following are multiple choice questions (with answers) about college + computer science. +fewshot_config: + sampler: first_n + samples: + - question: 'Which of the following regular expressions is equivalent to (describes + the same set of strings as) (a* + b)*(c + d)? + + (A) a*(c + d)+ b(c + d) + + (B) a*(c + d)* + b(c + d)* + + (C) a*(c + d)+ b*(c + d) + + (D) (a + b)*c +(a + b)*d' + target: 'Let''s think step by step. We know that: + + 1. (X* + Y)* = (X + Y)* + + 2. X(Y + Z)? = XY + XZ + + Using equation 1 we can rewrite (a* + b)*(c + d)? as: + + 3. (a + b)*(c + d)? + + Using equation 2 we can rewrite equation 3 as: + + (a + b)*c + (a + b)*d The answer is (D).' + - question: 'The Singleton design pattern is used to guarantee that only a single instance + of a class may be instantiated. Which of the following is (are) true of this + design pattern? + + I. The Singleton class has a static factory method to provide its instance. + + II. The Singleton class can be a subclass of another class. + + III. The Singleton class has a private constructor. + + (A) I only + + (B) II only + + (C) III only + + (D) I, II, and III' + target: 'Let''s think step by step. Statement I is a correct statement about a + Singleton, because a Singleton restricts instantiation to a single, static method. + Statement II is also correct, because there is no inherent restriction regarding + the inheritance of a Singleton. Statement III is also correct, because a Singletons + must be instantiated only once, so its constructor is made private to prevent + any construction except via its static factory method. + + Given these facts, statements I, II, and III are all correct. The answer is + (D).' + - question: 'A certain pipelined RISC machine has 8 general-purpose registers R0, R1, + . . . , R7 and supports the following operations: + + ADD Rs1, Rs2, Rd (Add Rs1 to Rs2 and put the sum in Rd) + + MUL Rs1, Rs2, Rd (Multiply Rs1 by Rs2 and put the product in Rd) + + An operation normally takes one cycle; however, an operation takes two cycles + if it produces a result required by the immediately following operation in an + operation sequence. + + Consider the expression AB + ABC + BC, where variables A, B, C are located in + registers R0, R1, R2. If the contents of these three registers must not be modified, + what is the minimum number of clock cycles required for an operation sequence + that computes the value of AB + ABC + BC? + + (A) 5 (B) 6 (C) 7 (D) 8' + target: 'Let''s think step by step. First, we are given that A is in R0, B is + in R1, and C is in R2. + + Next, we can see that we must compute three multiplies (AB, BC, and ABC) and + two adds (AB + ABC, (AB + ABC) + BC) to compute our final answer, resulting + in a minimum of five clock cycles. + + Next, we can see that there is no way to avoid at least one pipeline stall when + computing our final answer, because to compute our final sum we must wait at + least one cycle for the results from the previous stage to be ready. Thus, our + minimum number of cycles must be 6. + + We can verify that we can create a solution that requires only six cycles as + follows: + + compute AB: MUL R0, R1, R3 + + compute BC: MUL R1, R2, R4 + + compute ABC: MUL R3, R4, R5 + + compute AB + BC: ADD R3, R4, R6 + + STALL + + compute AB + ABC + BC: ADD R5, R6, R7 + + So there are 6 cycles. The answer is (B).' + - question: 'A compiler generates code for the following assignment statement. + + G := (A + B) * C - (D + E) * F + + The target machine has a single accumulator and a single-address instruction + set consisting of instructions load, store, add, subtract, and multiply. For + the arithmetic operations, the left operand is taken from the accumulator and + the result appears in the accumulator. The smallest possible number of instructions + in the resulting code is + + (A) 5 (B) 6 (C) 7 (D) 9' + target: 'Let''s think step by step. We can compute the final answer with the following + sequence of operations: + + 1. LOAD D (accumulator = D) + + 2. ADD E (accumulator = D+E) + + 3. MUL F (accumulator = (D+E)*F) + + 4. STORE X (X = (D+E)*F) + + 5. LOAD A (accumulator = A) + + 6. ADD B (accumulator = A+B) + + 7. MUL C (accumulator = (A+B)*C) + + 8. SUB X (accumulator = (A+B)*C - (D+E)*F) + + 9. STORE G (G = (A+B)*C - (D+E)*F) + + This sequence takes 9 instructions. The answer is (D).' + - question: 'Consider a computer design in which multiple processors, each with a private + cache memory, share global memory using a single bus. This bus is the critical + system resource. Each processor can execute one instruction every 500 nanoseconds + as long as memory references are satisfied by its local cache. When a cache + miss occurs, the processor is delayed for an additional 2,000 nanoseconds. During + half of this additional delay, the bus is dedicated to serving the cache miss. + During the other half, the processor cannot continue, but the bus is free to + service requests from other processors. On average, each instruction requires + 2 memory references. On average, cache misses occur on 1 percent of references. + What proportion of the capacity of the bus would a single processor consume, + ignoring delays due to competition from other processors? + + (A) 1/50 (B) 1/27 (C) 1/25 (D) 2/27' + target: 'Let''s think step by step. We know that each instruction requires two + memory references per instruction, and that there is an average cache miss rate + of one percent. + + Thus a given processor has: + + (1 cache miss / 100 references) * (2 references / instruction) = + + (2 cache misses / 100 instructions), so: + + misses_per_instruction = 1 cache miss / 50 instructions. + + Next, we know that each instruction requires 500 nanoseconds when there is no + cache miss, and 500 + 2000 = 2500 nanoseconds when there is a cache miss. Thus: + + 50 instructions / (49 * 500) + (1 * 2500) nanoseconds, so: + + instructions_per_ns = 50 instructions / 27000 nanoseconds. + + Now, we know that each cache miss locks the bus for half of the 2000 nanosecond + cache miss delay, or 1000 nanoseconds, so: + + lock_ns_per_miss = 1000 nanoseconds / cache miss. + + Thus we can see that on average a single processor will lock the bus for: + + lock_ns_per_miss * misses_per_instruction * instructions_per_ns = + + (1000 nanoseconds / cache miss) * (1 cache miss / 50 instructions) * (50 instructions + / 27000 nanoseconds) = 1000 * (1/50) * (50/27000) = 1000/27000 = 1/27. The answer + is (B).' +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_college_computer_science diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_mathematics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5552cc35bc55ffab1b53538ee2605778e6f215d6 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_mathematics.yaml @@ -0,0 +1,73 @@ +dataset_name: college_mathematics +description: The following are multiple choice questions (with answers) about college + mathematics. +fewshot_config: + sampler: first_n + samples: + - question: 'Let V be the set of all real polynomials p(x). Let transformations T, + S be defined on V by T:p(x) -> xp(x) and S:p(x) -> p''(x) = d/dx p(x), and interpret + (ST)(p(x)) as S(T(p(x))). Which of the following is true? + + (A) ST = 0 (B) ST = T (C) ST = TS (D) ST - TS is the identity map of V onto + itself.' + target: "Let's think step by step. For a given polynomial $p$ we have\n\\[ST(p)\ + \ = (xp(x))\u2019 = p(x) + xp\u2019(x)\\]\nand\n\\[TS(p) = xp\u2019(x).\\]\n\ + Hence \\[ST(p) - TS(p) = p(x) + xp\u2019(x) - xp\u2019(x).\\] The answer is\ + \ (D)." + - question: 'Suppose that f(1 + x) = f(x) for all real x. If f is a polynomial and + f(5) = 11, then f(15/2) + + (A) -11 (B) 0 (C) 11 (D) 33/2' + target: Let's think step by step. The only polynomial so that $f(1 + x) = f(x)$ + is a constant polynomial. Hence $f(5) = 11 = f(15/2)$. The answer is (C). + - question: 'Let A be a real 2x2 matrix. Which of the following statements must be + true? + + I. All of the entries of A^2 are nonnegative. + + II. The determinant of A^2 is nonnegative. + + III. If A has two distinct eigenvalues, then A^2 has two distinct eigenvalues. + + (A) I only (B) II only (C) III only (D) II and III only' + target: 'Let''s think step by step. We have \[ det(A^2) = (det(A))^2 \geq 0,\] + hence II holds. + + III is false: as a counterexample take a diagonal matrix with -1 and 1 on the + diagonal. Then $A^2$ is the identity matrix. The answer is (B).' + - question: 'Let A be the set of all ordered pairs of integers (m, n) such that 7m + + 12n = 22. What is the greatest negative number in the set B = {m + n : (m, + n) \in A}? + + (A) -5 (B) -4 (C) -3 (D) -2' + target: Let's think step by step. We have 12n = 22 - 7m and one of the solutions + is $m = -2$, $n = 3$. Then $m + n = 1$, hence we need to look for smaller $m$ + in order to make $m + n$ negative. The next solution is $m = -14$ and $n = 10$. + For smaller $m$ we have $m + n$ smaller than $-4$. The answer is (B). + - question: 'A tank initially contains a salt solution of 3 grams of salt dissolved + in 100 liters of water. A salt solution containing 0.02 grams of salt per liter + of water is sprayed into the tank at a rate of 4 liters per minute. The sprayed + solution is continually mixed with the salt solution in the tank, and the mixture + flows out of the tank at a rate of 4 liters per minute. If the mixing is instantaneous, + how many grams of salt are in the tank after 100 minutes have elapsed? + + (A) 2 (B) 2 - e^-2 (C) 2 + e^-2 (D) 2 + e^-4' + target: "Let's think step by step. For all $t \\in \\mathbb{R}$, let $s(t)$ denote\ + \ the number grams of salt in the tank at the $t$ minute mark. Then $s(0) =\ + \ 3$.\nWe use $s$ and $s(t)$ interchangeably. We also use $s^{\\prime}$ and\ + \ $s^{\\prime}(t)$ interchangeably. The solution sprayed into the tank adds\ + \ $(0.02) 4=2 / 25$ grams of salt per minute. There are always 100 liters of\ + \ liquid in the tank, containing $s$ grams of salt. So the density of salt in\ + \ the tank is $s / 100$ grams per liter. The flow of water out of the tank therefore\ + \ subtracts $4(s / 100)=s / 25$ grams of salt per minute. Then, for all $t \\\ + in \\mathbb{R}$, we have $s^{\\prime}(t)=(2 / 25)-(s / 25)=(2-s) / 25$, and\ + \ so $[s(t)=2] \\Rightarrow\\left[s^{\\prime}(t)=0\right]$. For all $t \\in\ + \ \\mathbb{R}$,\n$$\n\frac{d}{d t}[\\ln (s-2)]=\frac{s^{\\prime}}{s-2}=\frac{-1}{25}=\f\ + rac{d}{d t}\\left[-\frac{t}{25}\right] .\n$$\nChoose $C \\in \\mathbb{R}$ such\ + \ that, for all $t \\in \\mathbb{R}, \\ln ((s(t)-2))=-[t / 25]+C$. Let $K:=e^{C}$.\ + \ Then, for all $t \\in \\mathbb{R}$, we have $(s(t))-2=K e^{-t / 25}$, and\ + \ so $s(t)=2+K e^{-t / 25}$. Then $3=s(0)=2+K e^{0}=2+K$, so $K=1$. Then $s(100)=2+K\ + \ e^{-100 / 25}=2+1 \\cdot e^{-4}=2+e^{-4}$. The answer is (D).\n\n" +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_college_mathematics diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_medicine.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7eac0bab3f9286469b44e815c8a2090fc5ff0832 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_medicine.yaml @@ -0,0 +1,68 @@ +dataset_name: college_medicine +description: The following are multiple choice questions (with answers) about college + medicine. +fewshot_config: + sampler: first_n + samples: + - question: 'An expected side effect of creatine supplementation is: + + (A) muscle weakness. (B) gain in body mass. (C) muscle cramps. (D) loss of electrolytes.' + target: Let's think step by step. We refer to Wikipedia articles on medicine for + help. Creatine supplementation is a dietary supplement that results in body + mass gain. The answer is (B). + - question: 'Which of the following is not a true statement? + + (A) Muscle glycogen is broken down enzymatically to glucose-1-phosphate (B) + Elite endurance runners have a high proportion of Type I fibres in their leg + muscles (C) Liver glycogen is important in the maintenance of the blood glucose + concentration (D) Insulin promotes glucose uptake by all tissues in the body' + target: "Let's think step by step. We refer to Wikipedia articles on medicine\ + \ for help. Let\u2019s solve this step by step and go over each choice: \n(A)\ + \ \u201CMuscle glycogen is broken down enzymatically to glucose-1-phosphate\u201D\ + : This is a correct statement.\n(B) \u201CElite endurance runners have a high\ + \ proportion of Type I fibres in their leg muscles\u201D: This is a correct\ + \ statement.\n(C) \u201CLiver glycogen is important in the maintenance of the\ + \ blood glucose concentration\u201D: This is a correct statement. \n(D) \u201C\ + Insulin promotes glucose uptake by all tissues in the body\u201D: This is not\ + \ a correct statement, because insulin promotes glucose uptake by the liver,\ + \ adipose tissue, and muscle, but not all tissues. For instance, the tissues\ + \ in the brain and red blood cells are not affected by insulin. The answer is\ + \ (D)." + - question: "A high school science teacher fills a 1 liter bottle with pure nitrogen\ + \ and seals the lid. The pressure is 1.70 atm, and the room temperature is 25\xB0\ + C. Which two variables will both increase the pressure of the system, if all\ + \ other variables are held constant?\n(A) Increasing temperature, increasing\ + \ moles of gas (B) Increasing temperature, increasing volume (C) Decreasing\ + \ volume, decreasing temperature (D) Decreasing moles of gas, increasing volume" + target: 'Let''s think step by step. We refer to Wikipedia articles on medicine + for help. The relevant equation for this is the ideal gas law: PV=nRT. To increase + the pressure of the system (P), then either n (number of moles of the gas) or + T (temperature) have to increase. The answer is (A).' + - question: 'In a genetic test of a newborn, a rare genetic disorder is found that + has X-linked recessive transmission. Which of the following statements is likely + true regarding the pedigree of this disorder? + + (A) All descendants on the maternal side will have the disorder. (B) Females + will be approximately twice as affected as males in this family. (C) All daughters + of an affected male will be affected. (D) There will be equal distribution of + males and females affected.' + target: "Let's think step by step. We refer to Wikipedia articles on medicine\ + \ for help. Let\u2019s solve this step by step. Let's recall first that females\ + \ have two X chromosomes, while males have one X and one Y chromosome. This\ + \ is an important fact we need to know before answering this question. \nBecause\ + \ a male can only pass his only one X chromosome to a daughter, if he is affected\ + \ by this rare genetic disorder, then we know for sure that he will pass this\ + \ rare genetic disorder to all his future-born daughters. Therefore, \u201C\ + (C): All daughters of an affected male will be affected\u201D is a correct statement.\ + \ The answer is (C)." + - question: 'Glucose is transported into the muscle cell: + + (A) via protein transporters called GLUT4. (B) only in the presence of insulin. + (C) via hexokinase. (D) via monocarbylic acid transporters.' + target: 'Let''s think step by step. We refer to Wikipedia articles on medicine + for help. Glucose (also known as the blood sugar) is the main sugar found in + the human body. It is transported into the muscle cell via diffusion through + protein transporters called GLUT4. The answer is (A).' +tag: mmlu_flan_cot_fewshot_other +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_college_medicine diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_physics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aa158a9f3c3e9e420608970a3ea91e744a042a6a --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_physics.yaml @@ -0,0 +1,61 @@ +dataset_name: college_physics +description: The following are multiple choice questions (with answers) about college + physics. +fewshot_config: + sampler: first_n + samples: + - question: 'A refracting telescope consists of two converging lenses separated by + 100 cm. The eye-piece lens has a focal length of 20 cm. The angular magnification + of the telescope is + + (A) 4 (B) 5 (C) 6 (D) 20' + target: Let's think step by step. In a refracting telescope, if both lenses are + converging, the focus of both lenses must be between the two lenses, and thus + the focal lengths of the two lenses must add up to their separation. Since the + focal length of one lens is 20 cm, the focal length of the other must be 80 + cm. The magnification is the ratio of these two focal lengths, or 4. The answer + is (A). + - question: 'The muon decays with a characteristic lifetime of about 10^-6 second into + an electron, a muon neutrino, and an electron antineutrino. The muon is forbidden + from decaying into an electron and just a single neutrino by the law of conservation + of + + (A) charge (B) mass (C) energy and momentum (D) lepton number' + target: Let's think step by step. Lepton number must be conserved, meaning the + total number of leptons minus the number of antileptons. If a muon decays into + an electron and a single neutrino, the total lepton number would go from one + to two, violating lepton number conservation. The answer is (D). + - question: 'One end of a Nichrome wire of length 2L and cross-sectional area A is + attached to an end of another Nichrome wire of length L and cross- sectional + area 2A. If the free end of the longer wire is at an electric potential of 8.0 + volts, and the free end of the shorter wire is at an electric potential of 1.0 + volt, the potential at the junction of the two wires is most nearly equal to + + (A) 2.4 V (B) 3.3 V (C) 4.5 V (D) 5.7 V' + target: Let's think step by step. This is a simple voltage divider problem, where + the longer wire has a resistance four times that of the shorter end. So the + voltage divider ratio is 1 / 5, meaning that the potential in the middle is + 1.0 V + (8.0 V - 1.0 V) * 1/5 = 2.4 V. The answer is (A). + - question: 'A refracting telescope consists of two converging lenses separated by + 100 cm. The eye-piece lens has a focal length of 20 cm. The angular magnification + of the telescope is + + (A) 4 (B) 5 (C) 6 (D) 20' + target: Let's think step by step. In a refracting telescope, if both lenses are + converging, the focus of both lenses must be between the two lenses, and thus + the focal lengths of the two lenses must add up to their separation. Since the + focal length of one lens is 20 cm, the focal length of the other must be 80 + cm. The magnification is the ratio of these two focal lengths, or 4. The answer + is (A). + - question: 'For which of the following thermodynamic processes is the increase in + the internal energy of an ideal gas equal to the heat added to the gas? + + (A) Constant temperature (B) Constant volume (C) Constant pressure (D) Adiabatic' + target: 'Let''s think step by step. Heat added to the gas can go into the gases + internal energy or work done against an external force. However, if the volume + of the gas container is constant, no work will be done (since work is pressure + times change in volume). So, at constant volume, all of the heat goes into the + internal energy. The answer is (B).' +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_college_physics diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_computer_security.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6a7c5cb2d5d1a2943b5b1ad6c17bf5ce0a324f5a --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_computer_security.yaml @@ -0,0 +1,50 @@ +dataset_name: computer_security +description: The following are multiple choice questions (with answers) about computer + security. +fewshot_config: + sampler: first_n + samples: + - question: 'SHA-1 has a message digest of + + (A) 160 bits (B) 512 bits (C) 628 bits (D) 820 bits' + target: Let's think step by step. Since SHA-1 is a hash function which takes an + question and produces a 160-bit (20-byte) hash value, its message digest is 160 + bits. The answer is (A). + - question: "_____________ can modify data on your system \u2013 so that your system\ + \ doesn\u2019t run correctly or you can no longer access specific data, or it\ + \ may even ask for ransom in order to give your access.\n(A) IM \u2013 Trojans\ + \ (B) Backdoor Trojans (C) Trojan-Downloader (D) Ransom Trojan" + target: Let's think step by step. The system is asking for trojans, which are + for ransom, which means ransom trojan. The answer is (D). + - question: 'What is ethical hacking? + + (A) "Hacking" ethics so they justify unintended selfish behavior (B) Hacking + systems (e.g., during penetration testing) to expose vulnerabilities so they + can be fixed, rather than exploited (C) Hacking into systems run by those whose + ethics you disagree with (D) A slang term for rapid software development, e.g., + as part of hackathons' + target: Let's think step by step. Ethical hacking is a process of detecting vulnerabilities + in an application, system, or organization's infrastructure that an attacker + can use to exploit an individual or organization. They use this process to prevent + cyberattacks and security breaches by lawfully hacking into the systems and + looking for weak points. The answer is (B). + - question: 'The ____________ is anything which your search engine cannot search. + + (A) Haunted web (B) World Wide Web (C) Surface web (D) Deep Web' + target: "Let's think step by step. The search engine searches on the Surface Web,\ + \ which is the portion of the world wide web which is visible so (B,C) are wrong.\ + \ The Haunted Web doesn\u2019t correspond to an internet concept. The Deep Web\ + \ is the part of the World Wide Web which is not indexed. The answer is (D)." + - question: 'Exploitation of the Heartbleed bug permits + + (A) overwriting cryptographic keys in memory (B) a kind of code injection (C) + a read outside bounds of a buffer (D) a format string attack' + target: 'Let''s think step by step. The Heartbleed Bug is a serious vulnerability + in the popular OpenSSL cryptographic software library. Heartbleed resulted from + improper question validation (due to a missing bounds check) in the implementation + of the TLS heartbeat extension. The vulnerability was classified as a buffer + over-read, a situation where more data can be read than should be allowed. The + answer is (C).' +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_computer_security diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_conceptual_physics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a4757faf1d490e312ea70a3d4dbe291459366471 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_conceptual_physics.yaml @@ -0,0 +1,49 @@ +dataset_name: conceptual_physics +description: ' + + The following are multiple choice questions (with answers) about conceptual physics.' +fewshot_config: + sampler: first_n + samples: + - question: 'Colors in a soap bubble result from light + + (A) converted to a different frequency (B) deflection (C) interference (D) polarization' + target: Let's think step by step. In a soap bubble film, the light bounces between + the two soap-air interfaces many times, interfering with itself constructively + or destructively depending on the width of the film. This results in different + colors being visible. The answer is (C). + - question: 'Compared with the mass of a uranium atom undergoing fission, the combined + masses of the products after fission are + + (A) less (B) more (C) the same (D) zero' + target: Let's think step by step. Fission releases energy, which comes from the + rest mass of its initial nucleus. Thus the mass of the products is less than + the mass of the reactant uranium nucleus. The answer is (A). + - question: 'Things that are equivalent according to the equivalence principle are + + (A) space and time. (B) a traveling twin and a stay-at-home twin. (C) gravity + and acceleration. (D) mass and energy.' + target: "Let's think step by step. Einstein\u2019s famous equivalence principle\ + \ states that gravity and acceleration are equivalent. The answer is (C)." + - question: 'Which of these three elements has the most mass per nucleon? + + (A) Hydrogen (B) Iron (C) Uranium (D) Same in each' + target: Let's think step by step. Due to nuclear binding energy, the mass of an + atomic nucleus is less than the sum of individual masses of the free constituent + protons and neutrons; this is known as the mass defect. Hydrogen has no mass + defect because it has only a single nucleon, so it will have the most mass per + nucleon. The answer is (A). + - question: 'A model airplane flies slower when flying into the wind and faster with + wind at its back. When launched at right angles to the wind a cross wind its + groundspeed compared with flying in still air is + + (A) the same (B) greater (C) less (D) either greater or less depending on wind + speed' + target: "Let's think step by step. The plane\u2019s speed in the direction of\ + \ the wind is greater than it would be in the absence of wind, and its direction\ + \ orthogonal to the wind is the same as it would be in the absence of the wind.\ + \ The total speed, which is these two components added in quadrature, is thus\ + \ greater than the speed in still air. The answer is (B).\n\n" +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_conceptual_physics diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_econometrics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e4282345ce0f2e8bab97a80413fbd2b796a7fe3e --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_econometrics.yaml @@ -0,0 +1,87 @@ +dataset_name: econometrics +description: The following are multiple choice questions (with answers) about econometrics. +fewshot_config: + sampler: first_n + samples: + - question: 'Suppose now that a researcher wishes to use information criteria to determine + the optimal lag length for a VAR. 500 observations are available for the bi-variate + VAR, and the values of the determinant of the variance-covariance matrix of + residuals are 0.0336, 0.0169, 0.0084, and 0.0062 for 1, 2, 3, and 4 lags respectively. + What is the optimal model order according to Akaike''s information criterion? + + (A) 1 lag (B) 2 lags (C) 3 lags (D) 4 lags' + target: "Let's think step by step. We refer to Wikipedia articles on econometrics\ + \ for help. Let\u2019s solve this problem step by step. First of all, let\u2019\ + s recall that for a given set of data, Akaike's information criterion (AIC)\ + \ allows us to measure how well a statistical model fits the data; it is an\ + \ estimator of prediction error. Here in this problem we will need to use the\ + \ formula ln(det(sigma_hat)) + (2 * k / T) to determine the values of Akaike\u2019\ + s criterion, where ln denotes the natural log function, det the determinant\ + \ function, k the total number of parameters in total (across both equations),\ + \ and T the number of observations (which, in this case, is equal to 500). For\ + \ 1 lag, the number of parameters in total is equal to 6; for 2 lags, it is\ + \ 10; for 3 lags, it is 14; and for 4 lags, it is 18. Now, let\u2019s calculate\ + \ the values of the criterion for each lag:\n(A) 1 lag: ln(0.0336) + (2 * 6\ + \ / 500) = ln(0.0336) + (12 / 500) = -3.369\n(B) 2 lags: ln(0.0169) + (2 * 10\ + \ / 500) = ln(0.0169) + (20 / 500) = -4.040\n(C) 3 lags: ln(0.0084) + (2 * 14\ + \ / 500) = ln(0.0084) + (28 / 500) =-4.724\n(D) 4 lags: ln(0.0062) + (2 * 18\ + \ / 500) = ln(0.0062) + (36 / 500) =-5.011\nBecause the optimal model order\ + \ according to AIC minimizes the information criterion, the answer should be\ + \ the one with the lowest value. In this case, (D) has the lowest value. The\ + \ answer is (C)." + - question: 'Consider the following AR(1) model with the disturbances having zero mean + and unit variance + + yt = 0.2 + 0.4 yt-1 + ut + + The (unconditional) mean of y will be given by + + (A) 0.2 (B) 0.4 (C) 0.5 (D) 0.33' + target: "Let's think step by step. We refer to Wikipedia articles on econometrics\ + \ for help. Let\u2019s solve this problem step by step. If we have a an AR(1)\ + \ model with the disturbances having zero mean and unit variance, then the unconditional\ + \ mean of y is equal to the following:\nunconditional mean of y = (the intercept\ + \ term) / (1 - autoregressive coefficient)\nWe know that the intercept term\ + \ is 0.2 and the autoregressive coefficient is 0.4; thus, we have:\nunconditional\ + \ mean of y = (0.2) / (1 - 0.4) = (0.2) / (0.6) = 2 / 6 = 1 / 3, which is approximately\ + \ 0.33. That means that the answer should be (D) 0.33. The answer is (D)." + - question: 'What would be then consequences for the OLS estimator if heteroscedasticity + is present in a regression model but ignored? + + (A) It will be biased (B) It will be inconsistent (C) It will be inefficient + (D) All of (a), (b) and (c) will be true.' + target: Let's think step by step. We refer to Wikipedia articles on econometrics + for help. Heteroscedasticity refers to the condition where the variance of the + error terms is not constant across multiple observations. If heteroscedasticity + is present in a regression model, then the coefficient estimates in the OLS + estimator will be not only unbiased and consistent but also inefficient. Because + (A) and (B) are incorrect choices and (C) is a correct choice, (D) cannot be + the right answer. Ultimately, (C) is the only true choice. The answer is (C). + - question: 'Suppose that a test statistic has associated with it a p-value of 0.08. + Which one of the following statements is true? + + (i) If the size of the test were exactly 8%, we would be indifferent between + rejecting and not rejecting the null hypothesis + + (ii) The null would be rejected if a 10% size of test were used + + (iii) The null would not be rejected if a 1% size of test were used + + (iv) The null would be rejected if a 5% size of test were used. + + (A) (ii) and (iv) only (B) (i) and (iii) only (C) (i), (ii), and (iii) only + (D) (i), (ii), (iii), and (iv).' + target: "Let's think step by step. We refer to Wikipedia articles on econometrics\ + \ for help. Let\u2019s reason about each of the options.\n(i) is a true statement.\n\ + (ii) is a true statement.\n(iii) is a true statement.\n(iv) is not a true statement.\ + \ Thus, (i), (ii), and (iii) are true. The answer is (C)." + - question: 'For a stationary autoregressive process, shocks will + + (A) Eventually die away (B) Persist indefinitely (C) Grow exponentially (D) + Never occur' + target: 'Let''s think step by step. We refer to Wikipedia articles on econometrics + for help. This is a formal logic problem about stationally process. For a stationary + autoregressive process, shocks will eventually die away. The answer is (A).' +tag: mmlu_flan_cot_fewshot_social_sciences +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_econometrics diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_electrical_engineering.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..305d2340c5ffa69761ef8dc2ab128849e571bbf8 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_electrical_engineering.yaml @@ -0,0 +1,47 @@ +dataset_name: electrical_engineering +description: ' + + The following are multiple choice questions (with answers) about electrical engineering.' +fewshot_config: + sampler: first_n + samples: + - question: "A point pole has a strength of 4\u03C0 * 10^-4 weber. The force in newtons\ + \ on a point pole of 4\u03C0 * 1.5 * 10^-4 weber placed at a distance of 10\ + \ cm from it will be\n(A) 15 N. (B) 20 N. (C) 7.5 N. (D) 3.75 N." + target: "Let's think step by step. The force between two point poles is given\ + \ by m_1m_2/(mu_0 4 \\pi r^2), in analogy to Coulomb\u2019s law. Plugging in\ + \ the values given in the question, we calculate that the force is approximately\ + \ 15 N. The answer is (A)." + - question: 'The coil of a moving coil meter has 100 turns, is 40 mm long and 30 mm + wide. The control torque is 240*10-6 N-m on full scale. If magnetic flux density + is 1Wb/m2 range of meter is + + (A) 1 mA. (B) 2 mA. (C) 3 mA. (D) 4 mA.' + target: Let's think step by step. The torque on a coil in a uniform magnetic field + is given by BANI, where B is the magnetic flux density, A is the area of the + coil, N is the number of turns, and I is the current. So we have that I = (Torque)/(BAN), + or 240e-6/(1200e-6 * 100 * 1) = 2e-3. The answer is (B). + - question: 'In an SR latch built from NOR gates, which condition is not allowed + + (A) S=0, R=0 (B) S=0, R=1 (C) S=1, R=0 (D) S=1, R=1' + target: Let's think step by step. An SR latch is a set-reset latch; in the case + where S=1 and R=1, the circuit has no stable state; instead a race condition + will be produced within the circuit, so the device will be in an undefined state. + So S=1, R=1 is an illegal question. The answer is (D). + - question: 'Two long parallel conductors carry 100 A. If the conductors are separated + by 20 mm, the force per meter of length of each conductor will be + + (A) 100 N. (B) 0.1 N. (C) 1 N. (D) 0.01 N.' + target: Let's think step by step. The magnetic force-per-length between two current-carrying + conductors is given by \mu_0 I_1 I_2 / (2 \pi r), where $r$ is the separation + distance and I_1 and I_2 are the currents. Plugging in 100 A for I_1 and I_2, + and 20 mm for r, gives 0.1 N. The answer is (B). + - question: "In a 2 pole lap winding dc machine , the resistance of one conductor is\ + \ 2\u03A9 and total number of conductors is 100. Find the total resistance\n\ + (A) 200\u03A9 (B) 100\u03A9 (C) 50\u03A9 (D) 10\u03A9" + target: 'Let''s think step by step. In lap winding, effectively two resistors + are connected in parallel, so the actual resistance of each pair is 1 Ohm. Since + we have 50 pairs, we get a total resistance of 50 Ohms. The answer is (C).' +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_electrical_engineering diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_elementary_mathematics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1fb9eef860424b49adf108566edadaf1500dbf5f --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_elementary_mathematics.yaml @@ -0,0 +1,77 @@ +dataset_name: elementary_mathematics +description: The following are multiple choice questions (with answers) about elementary + mathematics. +fewshot_config: + sampler: first_n + samples: + - question: 'Olivia used the rule "Add 11" to create the number pattern shown below. + 10, 21, 32, 43, 54. Which statement about the number pattern is true? + + (A) The 10th number in the pattern will be an even number. + + (B) The number pattern will never have two even numbers next to each other. + + (C) The next two numbers in the pattern will be an even number then an odd number. + + (D) If the number pattern started with an odd number then the pattern would + have only odd numbers in it.' + target: Let's think step by step. Choice A is incorrect because every even-numbered + term in the pattern is odd, and 10 is an even number. Choice B is correct, because + adding an odd number (in this case 11) to an odd number produces an even number, + and adding an odd number to an even number produces an odd number. Thus the + terms in the pattern will alternate between odd and even, so there will never + be two even numbers next to each other. Choice C is incorrect because the last + term in the example is even (54), and we know that the terms will alternate + between even and odd. Choice D is incorrect because the terms in the pattern + will alternate between odd and even, regardless of the value of the first term. + The answer is (B). + - question: 'The population of the city where Michelle was born is 145,826. What is + the value of the 5 in the number 145,826? + + (A) 5 thousands + + (B) 5 hundreds + + (C) 5 tens + + (D) 5 ones' + target: 'Let''s think step by step. Choice A is correct, because there are three + digits following the 5, so + + the 5 is in the thousands place. Thus the other choices are incorrect. The answer + is (A).' + - question: "A store sells 107 different colors of paint. They have 25 cans of each\ + \ color in storage. The number of cans of paint the store has in storage can\ + \ be found using the expression below. 107 \xD7 25. How many cans of paint does\ + \ the store have in storage?\n(A) 749\n(B) 2,675\n(C) 2,945\n(D) 4,250" + target: Let's think step by step. We can calculate 107 x 25 = (100 x 25) + (7 + x 25) = 2500 + 175 = 2675. The answer is (B). + - question: 'A total of 30 players will play basketball at a park. There will be exactly + 5 players on each team. Which statement correctly explains how to find the number + of teams needed? + + (A) Add 5 to 30 to find 35 teams. + + (B) Divide 30 by 5 to find 6 teams. + + (C) Multiply 30 and 5 to find 150 teams. + + (D) Subtract 5 from 30 to find 25 teams.' + target: Let's think step by step. We want to find the number of teams. We know + that there are 5 players/team, and 30 players. Thus to get the number of teams + we divide players by players/team, so 30 players / 5 players/team = 6 teams. + The answer is (B). + - question: 'Which expression is equivalent to 5 x 9? + + (A) (5 x 4) x (6 x 5) + + (B) (5 x 5) + (5 x 4) + + (C) (5 x 5) + (5 x 9) + + (D) (5 x 9) x (6 x 9)' + target: 'Let''s think step by step. We know that 9 = (5 + 4), so 5 x 9 = 5 x (5 + + 4) = (5 x 5) + (5 x 4). The answer is (B).' +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_elementary_mathematics diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_formal_logic.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3fbc73c3d24f9cef06a41bbcfddea55aec1b424a --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_formal_logic.yaml @@ -0,0 +1,70 @@ +dataset_name: formal_logic +description: The following are multiple choice questions (with answers) about formal + logic. +fewshot_config: + sampler: first_n + samples: + - question: "Which of the given formulas of PL is the best symbolization of the following\ + \ sentence?\nTurtles live long lives and are happy creatures, unless they are\ + \ injured.\n(A) (L \u2022 H) \u2261 I (B) (L \u2022 H) \u2228 I (C) L \u2022\ + \ (H \u2228 I) (D) L \u2022 (H \u2283 R)." + target: "Let's think step by step. We refer to Wikipedia articles on formal logic\ + \ for help. Let\u2019s solve this step by step. Let \u201CL\u201D denote \u201C\ + living long\u201D, H \u201Cbeing happy\u201D, and \u201CI\u201D \u201Cbeing\ + \ injured\u201D. Now, consider each choice:\n(A) means (living long AND being\ + \ happy) is equivalent to (being injured). \n(B) means (living long AND being\ + \ happy) OR (being injured). \n(C) means (living long) AND (being happy OR being\ + \ injured). \n(D) means (living long) AND (being happy implies being R), but\ + \ what R denotes is not clear.\nObviously, (B) is the best symbolization of\ + \ the original sentence. The answer is (B)." + - question: 'Select the best translation into predicate logic.George borrows Hector''s + lawnmower. (g: George; h: Hector; l: Hector''s lawnmower; Bxyx: x borrows y + from z). + + (A) Blgh (B) Bhlg (C) Bglh (D) Bghl' + target: "Let's think step by step. We refer to Wikipedia articles on formal logic\ + \ for help. Let\u2019s solve this step by step. We are told that \u201CBxyx\u201D\ + \ means \u201Cx borrows y from z\u201D. We can rewrite \u201CGeorge borrows\ + \ Hector's lawnmower\u201D as \u201CGeorge borrows a lawnmower from Hector\u201D\ + , which can then be translated into predicate logic as \u201CBglh\u201D. The\ + \ answer \u201CBglh\u201D appears in (C); therefore, (C) must be the correct\ + \ answer. The answer is (C)." + - question: "\nSelect the best English interpretation of the given arguments in predicate\ + \ logic.\nDm\n(\u2200x)(Wx \u2283 ~Dx). \n(\u2200x)Wx \u2228 Ag\t/ (\u2203x)Ax\n\ + (A) Marina is a dancer. Some weaklings are not dancers. Either everything is\ + \ a weakling or Georgia plays volleyball. So something plays volleyball. (B)\ + \ Marina is a dancer. No weakling is a dancer. Everything is either a weakling\ + \ or plays volleyball. So something plays volleyball. (C) Marina is a dancer.\ + \ Some weaklings are not dancers. Everything is either a weakling or plays volleyball.\ + \ So something plays volleyball. (D) Marina is a dancer. No weakling is a dancer.\ + \ Either everything is a weakling or Georgia plays volleyball. So something\ + \ plays volleyball." + target: "Let's think step by step. We refer to Wikipedia articles on formal logic\ + \ for help. Let\u2019s solve this step by step. Let \u201CD\u201D denote \u201C\ + being a dancer\u201D, \u201Cm\u201D denote \u201CMaria\u201D, \u201Cg\u201D\ + \ denote \u201CGeorgia\u201D, \u201CW\u201D denote \u201Cweakling\u201D, \u201C\ + A\u201D denote \u201Cplaying volleyball\u201D. Then, we have the following:\n\ + 1. Dm \u2192 Maria is a dance.\n2. (\u2200x)(Wx \u2283 ~Dx). \u2192 For all\ + \ x, if x is a weakling, then x is not a dancer. In other words, no weakling\ + \ is a dancer.\n3. (\u2200x)Wx \u2228 Ag\t/ (\u2203x)Ax \u2192 For all x, x\ + \ is a weakling or Georgia plays volleyball. So there exists an x that plays\ + \ volleyball. \nOptions (A) and (C) do claim that some weaklings are not dancers,\ + \ but the second argument strongly states that no weakling is a dancer. Thus,\ + \ we can eliminate them. Option (B) omits the important detail about Georgia\ + \ playing volleyball. Option (D) has all the details presented in the arguments\ + \ and is the best English interpretation of the arguments. The answer is (D)." + - question: "Select the best translation into predicate logic: No people drive on Mars.\n\ + (A) ~Pd (B) (\u2200x)(Px \u2228 ~Dx) (C) (\u2200x)(Px \u2283 ~Dx) (D) ~Dp" + target: "Let's think step by step. We refer to Wikipedia articles on formal logic\ + \ for help. Let\u2019s solve this step by step. Let \u201CP\u201D denote \u201C\ + being on Mars\u201D and \u201CD\u201D denote \u201Cdriving on Mars\u201D. Then\ + \ let\u2019s consider each option:\nOption (A): ~Pd \u2192 d is not on Mars.\n\ + Option (B): (\u2200x)(Px \u2228 ~Dx) \u2192 For all x, x is on Mars and x do\ + \ not drive on Mars.\nOption (C): (\u2200x)(Px \u2283 ~Dx) \u2192 For all x,\ + \ x is on Mars implies that x do not drive on Mars.\nOption (D): ~Dp: \u2192\ + \ p do not drive on Mars.\nOf all these options, Option (C) appears to be the\ + \ best and most meaningful interpretation of the argument \u201CNo people drive\ + \ on Mars.\u201D The answer is (C).\n\n" +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_formal_logic diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_global_facts.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..739c00106536c8df64213d55b83c053db1314124 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_global_facts.yaml @@ -0,0 +1,49 @@ +dataset_name: global_facts +description: The following are multiple choice questions (with answers) about global + facts. +fewshot_config: + sampler: first_n + samples: + - question: "As of 2017, how many of the world\u2019s 1-year-old children today have\ + \ been vaccinated against some disease? *\n(A) 80% (B) 60% (C) 40% (D) 20%" + target: Let's think step by step. We refer to Wikipedia articles on global facts + for help. According to data published by the World Health Organization, the + nummber of 1-year-old children vaccinated in 2017 exceeds 80%. The answer is + (A). + - question: 'As of 2019, about what percentage of Americans agree that the state is + run for the benefit of all the people? + + (A) 31% (B) 46% (C) 61% (D) 76%' + target: Let's think step by step. We refer to Wikipedia articles on global facts + for help. In 2019, about 46% percentage of Americans agree that the state is + run for the benefit of all the people. The answer is (B). + - question: 'As of 2019, about what percentage of Russians say it is very important + to have free media in our country without government/state censorship? + + (A) 38% (B) 53% (C) 68% (D) 83%' + target: Let's think step by step. We refer to Wikipedia articles on global facts + for help. As of 2019, about 38% of Russians say it is very important to have + free media in our country. The answer is (A). + - question: 'As of 2015, since 1990 forests have ____ in Europe and have ____ in Africa + and the Americas. + + (A) increased, increased (B) increased, decreased (C) decreased, increased (D) + decreased, decreased' + target: Let's think step by step. We refer to Wikipedia articles on global facts + for help. As of 2015, since 1990 forests have increased in Europe and have decreased + in Africa and the Americas. The answer is (B). + - question: 'Which of the following pairs of statements are both true (as of 2019)? + + (A) People tend to be optimistic about their own future and the future of their + nation or the world. (B) People tend to be optimistic about their own future + but pessimistic about the future of their nation or the world. (C) People tend + to be pessimistic about their own future but optimistic about the future of + their nation or the world. (D) People tend to be pessimistic about their own + future and the future of their nation or the world.' + target: 'Let''s think step by step. We refer to Wikipedia articles on global facts + for help. As of 2019, most people tend to be optimistic about their own future + but pessimistic about the future of their nation or the world. The answer is + (B).' +tag: mmlu_flan_cot_fewshot_other +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_global_facts diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_biology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0dfb19f924761c6dd56cc6b3b9ada38b5bf473e0 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_biology.yaml @@ -0,0 +1,69 @@ +dataset_name: high_school_biology +description: The following are multiple choice questions (with answers) about high + school biology. +fewshot_config: + sampler: first_n + samples: + - question: "In animal cells, which of the following represents the most likely pathway\ + \ that a secretory protein takes as it is synthesized in a cell?\n(A) Plasma\ + \ membrane\u2013Golgi apparatus\u2013ribosome\u2013secretory vesicle\u2013rough\ + \ ER (B) Ribosome\u2013Golgi apparatus\u2013rough ER\u2013secretory vesicle\u2013\ + plasma membrane (C) Plasma membrane\u2013Golgi apparatus\u2013ribosome\u2013\ + secretory vesicle\u2013rough ER (D) Ribosome\u2013rough ER\u2013Golgi apparatus\u2013\ + secretory vesicle\u2013plasma membrane" + target: Let's think step by step. Protein synthesis starts at the ribosome, so + we can eliminate (A) and (C). The ribosome is often in the endoplasmic reticulum + and moves from there to the Golgi apparatus, where it is modified and packaged + into a vesicle. The vesicle then floats to the plasma membrane and is secreted. + The answer is (D). + - question: "A mutation in a bacterial enzyme changed a previously polar amino acid\ + \ into a nonpolar amino acid. This amino acid was located at a site distant\ + \ from the enzyme\u2019s active site. How might this mutation alter the enzyme\u2019\ + s substrate specificity?\n(A) By changing the enzyme\u2019s pH optimum (B) By\ + \ changing the enzyme\u2019s location in the cell (C) By changing the shape\ + \ of the protein (D) An amino acid change away from the active site cannot alter\ + \ the enzyme\u2019s substrate specificity." + target: Let's think step by step. A change in an amino acid leads to a change + in the primary structure of the protein. A change in the primary structure may + lead to a change in the secondary and the tertiary structure of the protein. + A change in the tertiary structure means a change in the shape of the protein, + so (C) has to be correct. Since the change does not affect the active site of + the enzyme, we do not expect the activity of the enzyme to be affected. The + answer is (C). + - question: 'Which of the following is not a way to form recombinant DNA? + + (A) Translation (B) Conjugation (C) Specialized transduction (D) Transformation' + target: 'Let''s think step by step. The introduction of foreign DNA or RNA into + bacteria or eukaryotic cells is a common technique in molecular biology and + scientific research. There are multiple ways foreign DNA can be introduced into + cells including transformation, transduction, conjugation, and transfection. + In contrast, (A) is not a way to form DNA: during translation the ribosomes + synthesize proteins from RNA. The answer is (A).' + - question: 'Homologous structures are often cited as evidence for the process of natural + selection. All of the following are examples of homologous structures EXCEPT + + (A) the wings of a bird and the wings of a bat (B) the flippers of a whale and + the arms of a man (C) the pectoral fins of a porpoise and the flippers of a + seal (D) the forelegs of an insect and the forelimbs of a dog' + target: "Let's think step by step. \u200B\u200BHomologous structures are similar\ + \ physical features in organisms that share a common ancestor \u200B\u200Bbut\ + \ different functions. Comparisons (B) and (C) are clearly homologous because\ + \ they share a common ancestor and the structures serve different purposes.\ + \ Bat wings and birg wings are also homologous, while they are both wings, the\ + \ forelimbs serve different purposes. Insects and dogs are very far ancestors\ + \ since one is vertebrate while the other is invertebrate and the forelimbs\ + \ serve the same purpose, so they are not homologous. The answer is (D)." + - question: 'Which of the following is not known to be involved in the control of cell + division? + + (A) Cyclins (B) Protein kinases (C) Checkpoints (D) Fibroblast cells' + target: 'Let''s think step by step. Normal cells move through the cell cycle in + a regulated way. At the checkpoint stage, they use information about their own + internal state and cues from the environment around them to decide whether to + proceed with cell division. Cues like these act by changing the activity of + core cell cycle regulators inside the cell. The most common regulators are cyclins + and cyclin-dependent kinases. Fibroblast cells do not play any role in cell + division. The answer is (D).' +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_biology diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_chemistry.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..948c375824506cc188ed821c4512725bfdc92d06 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_chemistry.yaml @@ -0,0 +1,66 @@ +dataset_name: high_school_chemistry +description: The following are multiple choice questions (with answers) about high + school chemistry. +fewshot_config: + sampler: first_n + samples: + - question: 'Which of the following is considered an acid anhydride? + + (A) HCl (B) H2SO3 (C) SO2 (D) Al(NO3)3' + target: Let's think step by step. An acid anhydride is a compound that is derived + by removing water from an acid. The chemical formula for water is H2O, which + means that we need to determine which of these options, when combined with H2O, + forms an acid. SO2, or Sulfur dioxide, when combined with H2O, makes H2SO4, + or sulfuric acid. The answer is (C). + - question: 'Which of the following is expected to be a polar molecule? + + (A) PCl4F (B) BF3 (C) CO2 (D) Si(CH3)4' + target: Let's think step by step. A polar molecule is one that has a slightly + positive charge on one end of the molecule and a slightly negative charge on + the other end. Boron trifluoride (BF3) has Boron as the center atom and three + fluorine atoms attached to it; it is trigonal planar and symmetric, so it is + nonpolar. Carbon Dioxide (CO2) has Carbon as the central atom with double bonds + to two Oxygen atoms - this is also symmetrical and therefore nonpolar. The same + is the case for tetramethyl silane (SI(CH3)4), which is a Silicon atom surrounded + by four methyl groups. The structure of PCL4F is that Phosphorus is the central + atom, attached to four chlorines and one fluorine atom. This is asymmetrical, + and therefore has a net dipole and is expected to be a polar molecule. The answer + is (A). + - question: 'From the solubility rules, which of the following is true? + + (A) All chlorides, bromides, and iodides are soluble (B) All sulfates are soluble + (C) All hydroxides are soluble (D) All ammonium-containing compounds are soluble' + target: Let's think step by step. The chlorides, bromides, and iodides of lead, + silver, and mercury are not soluble in water. This rules out (A). The sulfates + of lead, barium, and calcium are not soluble in water, which rules out (B). + The hydroxides of any metal besides sodium, potassium, ammonium, calcium, and + barium are insoluble. This rules out (C). Typically ammonium ions indicate a + soluble ionic substance. The answer is (D). + - question: 'A new compound is synthesized and found to be a monoprotic acid with a + molar mass of 248 g/mol. When 0.0050 mol of this acid are dissolved in 0.500 + L of water, the pH is measured as 3.89. What is the pKa of this acid? + + (A) 3.89 (B) 7.78 (C) 5.78 (D) 2.33' + target: "Let's think step by step. Recall that $[A] = [H^{+}]$. Here, this is\ + \ equal to $$10^{-3.89}$. Then we have $K_{a} = $\nrac{[H^{+}][A^{-}]}{[HA]}\ + \ = \nrac{10^{-3.89} \\cdot 10^{-3.89}}{10^{-2}}. The resulting exponent is\ + \ $-3.89 + (-3.89) - (-2) = 5.78$, therefore $K_a = 10^{-5.78}$. The $pK_a$\ + \ is the negative log of $K_a$, which is equal to $5.78$. The answer is (C)." + - question: 'A solution contains 2.00 mole of acetic acid, CH3COOH, and 1.00 mole of + calcium acetate, Ca(CH3COO)2. The solution is able to resist the addition of + a small amount of strong acid or strong base with only minor changes in the + pH of the solution. Larger quantities of strong acid or strong base can cause + a significant change in pH. How many moles of nitric acid, HNO3, may be added + before the pH begins to change significantly? + + (A) 0.500 mole (B) 1.00 mole (C) 2.00 mole (D) 3.00 mole' + target: "Let's think step by step. We would like to compute the buffer capacity\ + \ of this solution. First we write the equation for the ionization of the weak\ + \ acid, in this case of acetic acid. $CH_{3}COOH (aq) + H_{2}O \nightarrow H_{3}O^{+}\ + \ + CH3COO^{-}$. The conjugate base is therefore the acetate ion. The added\ + \ strong acid, Nitric acid, will react with the conjugate base. Therefore the\ + \ maximum amount of acid that can be added will be equal to the amount of acetate\ + \ ion, or 2 moles. The answer is (C).\n\n" +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_chemistry diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_computer_science.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f21030ae880ec0d6d42f0e2618c312b55b82549 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_computer_science.yaml @@ -0,0 +1,84 @@ +dataset_name: high_school_computer_science +description: The following are multiple choice questions (with answers) about high + school computer science. +fewshot_config: + sampler: first_n + samples: + - question: 'Which of the following is an example of the use of a device on the Internet + of Things (IoT) ? + + (A) A car alerts a driver that it is about to hit an object. (B) A hiker uses + a G P S watch to keep track of her position. (C) A refrigerator orders milk + from an online delivery service when the milk in the refrigerator is almost + gone. (D) A runner uses a watch with optical sensors to monitor his heart rate.' + target: Let's think step by step. The term Internet of Things (IoT) refers to + common devices which are connected to the internet, enabling new functionality. + Choice A is incorrect because it does not describe an internet connected device. + In choice B, the watch is only described as having GPS functionality but no + internet connectivity. Choice C describes a common device (a refrigerator) which + has internet connectivity enabling new functionality (online ordering). Choice + D does not mention internet connectivity for the watch, only optical sensors. + The answer is (C). + - question: 'Many Web browsers allow users to open anonymous windows. During a browsing + session in an anonymous window, the browser does not record a browsing history + or a list of downloaded files. When the anonymous window is exited, cookies + created during the session are deleted. Which of the following statements about + browsing sessions in an anonymous window is true? + + (A) The activities of a user browsing in an anonymous window will not be visible + to people who monitor the user''s network, such as the system administrator. + (B) Items placed in a Web store''s shopping cart for future purchase during + the anonymous browsing session will not be saved on the user''s computer. (C) + A user will not be able to log in to e-mail or social media accounts during + the anonymous browsing session. (D) A user browsing in an anonymous window will + be protected from viruses launched from any web sites visited or files downloaded.' + target: "Let's think step by step. Choice A is incorrect as it only describes\ + \ network traffic, which an anonymous browser does not change. Choice B is correct\ + \ as it correctly describes how an anonymous browser will prevent saving data\ + \ on the user\u2019s computer after the session is ended. Choice C is incorrect\ + \ because an anonymous browser will not prevent logging in to email or social\ + \ media accounts. Choice D is incorrect because an anonymous browser in itself\ + \ performs no virus protection. The answer is (B)." + - question: "In the program below, the initial value of X is 5 and the initial value\ + \ of Y is 10.\nIF (X < 0){\n DISPLAY (\"Foxtrot\")\n} ELSE {\n IF (X > Y){\n\ + \ DISPLAY (\"Hotel\")\n } ELSE {\n IF (Y > 0){\n DISPLAY (\"November\")\n\ + \ } ELSE {\n DISPLAY (\"Yankee\")\n }\n }\n}\nWhat is displayed as a result\ + \ of running the program?\n(A) Foxtrot (B) Hotel (C) November (D) Yankee" + target: Let's think step by step. Because X has the value 5, the first conditional + IF (X < 0) is false, so we move to the first ELSE clause. Because X is 5 and + Y is 10, the second conditional IF (X > Y) is false, so we move to the following + ELSE clause. Since Y is 10, the conditional IF (Y > 0) is true, so the command + DISPLAY ("November") is executed. The answer is (C). + - question: 'What is the output of "abc"[::-1] in Python 3? + + (A) Error (B) abc (C) cba (D) c' + target: Let's think step by step. We know that the slicing operator [::-1] takes + all of the elements in the string in reverse order, so we reverse the order + of the string "abc", resulting in "cba". The answer is (C). + - question: "A list of numbers has n elements, indexed from 1 to n. The following algorithm\ + \ is intended to display the number of elements in the list that have a value\ + \ greater than 100. The algorithm uses the variables count and position. Steps\ + \ 3 and 4 are missing.\n Step 1: Set count to 0 and position to 1.\n Step 2:\ + \ If the value of the element at index position is greater than 100, increase\ + \ the value of count by 1.\n Step 3: (missing step)\n Step 4: (missing step)\n\ + \ Step 5: Display the value of count.\nWhich of the following could be used\ + \ to replace steps 3 and 4 so that the algorithm works as intended?\n(A) Step\ + \ 3: Increase the value of position by 1.\n Step 4: Repeat steps 2 and 3 until\ + \ the value of count is greater than 100.\n(B) Step 3: Increase the value of\ + \ position by 1.\n Step 4: Repeat steps 2 and 3 until the value of position\ + \ is greater than n.\n(C) Step 3: Repeat step 2 until the value of count is\ + \ greater than 100.\n Step 4: Increase the value of position by 1.\n(D) Step\ + \ 3: Repeat step 2 until the value of position is greater than n.\n Step 4:\ + \ Increase the value of count by 1." + target: 'Let''s think step by step. Choice A is incorrect, because its Step 4 + has an incorrect termination condition, stopping when count is greater than + 100. We need to stop after inspecting all elements in the list. Choice B is + correct because it correctly increments both count and position, and correctly + repeats these steps and terminates when all elements in the list have been inspected. + Choice C is incorrect because it incorrectly increments the variable count until + its value is greater than 100, regardless of the elements in the list. Choice + D is incorrect because its step 3 does not increment the value of position, + so it will repeat forever. The answer is (B).' +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_computer_science diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_european_history.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4707a1857f44fad0ef67313b5f1901b5b6c869b6 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_european_history.yaml @@ -0,0 +1,199 @@ +dataset_name: high_school_european_history +description: The following are multiple choice questions (with answers) about high + school european history. +fewshot_config: + sampler: first_n + samples: + - question: 'This question refers to the following information. + + Albeit the king''s Majesty justly and rightfully is and ought to be the supreme + head of the Church of England, and so is recognized by the clergy of this realm + in their convocations, yet nevertheless, for corroboration and confirmation + thereof, and for increase of virtue in Christ''s religion within this realm + of England, and to repress and extirpate all errors, heresies, and other enormities + and abuses heretofore used in the same, be it enacted, by authority of this + present Parliament, that the king, our sovereign lord, his heirs and successors, + kings of this realm, shall be taken, accepted, and reputed the only supreme + head in earth of the Church of England, called Anglicans Ecclesia; and shall + have and enjoy, annexed and united to the imperial crown of this realm, as well + the title and style thereof, as all honors, dignities, preeminences, jurisdictions, + privileges, authorities, immunities, profits, and commodities to the said dignity + of the supreme head of the same Church belonging and appertaining; and that + our said sovereign lord, his heirs and successors, kings of this realm, shall + have full power and authority from time to time to visit, repress, redress, + record, order, correct, restrain, and amend all such errors, heresies, abuses, + offenses, contempts, and enormities, whatsoever they be, which by any manner + of spiritual authority or jurisdiction ought or may lawfully be reformed, repressed, + ordered, redressed, corrected, restrained, or amended, most to the pleasure + of Almighty God, the increase of virtue in Christ''s religion, and for the conservation + of the peace, unity, and tranquility of this realm; any usage, foreign land, + foreign authority, prescription, or any other thing or things to the contrary + hereof notwithstanding. + + English Parliament, Act of Supremacy, 1534 + + From the passage, one may infer that the English Parliament wished to argue + that the Act of Supremacy would + + (A) give the English king a new position of authority (B) give the position + of head of the Church of England to Henry VIII alone and exclude his heirs (C) + establish Calvinism as the one true theology in England (D) end various forms + of corruption plaguing the Church in England' + target: Let's think step by step. We refer to Wikipedia articles on european history + for help. The Act of Supremacy states that it grants authority to the king "to + repress and extirpate all errors, heresies, and other enormities and abuses", + referring to the corruption in the Church of England. The answer is (D). + - question: "This question refers to the following information.\nRead the following\ + \ excerpt.\nThe revolutionary seed had penetrated into every country and spread\ + \ more or less. It was greatly developed under the r\xE9gime of the military\ + \ despotism of Bonaparte. His conquests displaced a number of laws, institutions,\ + \ and customs; broke through bonds sacred among all nations, strong enough to\ + \ resist time itself; which is more than can be said of certain benefits conferred\ + \ by these innovators.\nThe monarchs will fulfil the duties imposed upon them\ + \ by Him who, by entrusting them with power, has charged them to watch over\ + \ the maintenance of justice, and the rights of all, to avoid the paths of error,\ + \ and tread firmly in the way of truth. Placed beyond the passions which agitate\ + \ society, it is in days of trial chiefly that they are called upon to despoil\ + \ realities of their false appearances, and to show themselves as they are,\ + \ fathers invested with the authority belonging by right to the heads of families,\ + \ to prove that, in days of mourning, they know how to be just, wise, and therefore\ + \ strong, and that they will not abandon the people whom they ought to govern\ + \ to be the sport of factions, to error and its consequences, which must involve\ + \ the loss of society.\nUnion between the monarchs is the basis of the policy\ + \ which must now be followed to save society from total ruin. . . .\nLet them\ + \ not confound concessions made to parties with the good they ought to do for\ + \ their people, in modifying, according to their recognized needs, such branches\ + \ of the administration as require it.\nLet them be just, but strong; beneficent,\ + \ but strict.\nLet them maintain religious principles in all their purity, and\ + \ not allow the faith to be attacked and morality interpreted according to the\ + \ social contract or the visions of foolish sectarians.\nLet them suppress Secret\ + \ Societies; that gangrene of society.\n\u2014Klemens von Metternich, Political\ + \ Confession of Faith, 1820\nWhich of the following was the greatest cause of\ + \ the fears expressed by Metternich in the document above?\n(A) The ideas of\ + \ personal liberty and nationalism conceived during the Enlightenment resulted\ + \ in radical revolutions that could spread throughout Europe. (B) The conquest\ + \ of Europe by Napoleon led to the creation of new factions and shifted the\ + \ European balance of power. (C) The power of monarchs had grown to the point\ + \ where it needed to be checked by other powers within each nation or domination\ + \ of civilians would occur. (D) The rising and falling economic cycle of the\ + \ newly emerging capitalist economy could lead to civilian unrest that must\ + \ be suppressed." + target: Let's think step by step. We refer to Wikipedia articles on european history + for help. The fears of revolution in early 19th century Europe expressed by + Klemens von Metternich, a conservative Austrian statesman, were a direct result + of the age of Enlightenment, a period of European history where the absolute + power of the monarchy was challenged with ideas of individual liberty and nationalism, + leading to the French revolution and its effects all over Europe. The answer + is (A). + - question: 'This question refers to the following information. + + The excerpts below are from the Navigation Acts of 1651. + + [A]fter the first day of December, one thousand six hundred fifty and one, and + from thence forwards, no goods or commodities whatsoever of the growth, production + or manufacture of Asia, Africa or America, or of any part thereof; or of any + islands belonging to them, or which are described or laid down in the usual + maps or cards of those places, as well of the English plantations as others, + shall be imported or brought into this Commonwealth of England, or into Ireland, + or any other lands, islands, plantations, or territories to this Commonwealth + belonging, or in their possession, in any other ship or ships, vessel or vessels + whatsoever, but only in such as do truly and without fraud belong only to the + people of this Commonwealth, or the plantations thereof, as the proprietors + or right owners thereof; and whereof the master and mariners are also of the + people of this Commonwealth, under the penalty of the forfeiture and loss of + all the goods that shall be imported contrary to this act, , , , + + [N]o goods or commodities of the growth, production, or manufacture of Europe, + or of any part thereof, shall after the first day of December, one thousand + six hundred fifty and one, be imported or brought into this Commonwealth of + England, or any other lands or territories to this Commonwealth belonging, or + in their possession, in any ship or ships, vessel or vessels whatsoever, but + in such as do truly and without fraud belong only to the people of this Commonwealth, + and in no other, except only such foreign ships and vessels as do truly and + properly belong to the people of that country or place, of which the said goods + are the growth, production or manufacture. + + Which of the following best describes the outcome of the Navigation Acts of + 1651? + + (A) They served as a catalyst for the growth of English shipping and overseas + trade, but did little to limit the prospects of the Dutch in the seventeenth + century. (B) They brought about almost immediate hardships for the Dutch economy + as their dominance of overseas trade quickly ended. (C) They were rescinded + during the restoration of the Stuarts as they sought normal diplomatic relations + with the Dutch so not as to need Parliament''s financial support for war. (D) + They led to nearly a century of recurrent war between England and the Netherlands, + which would not end until after American independence.' + target: Let's think step by step. We refer to Wikipedia articles on european history + for help. The Navigation Acts of 1651 helped English shipping by restricting + the ability of ships from other European countries, especially the Dutch, to + transport goods from colonies in Asia and Africa into England. The answer is + (A). + - question: "This question refers to the following information.\nIn Russia there was\ + \ nothing going on well, and [Souvarine] was in despair over the news he had\ + \ received. His old companions were all turning to the politicians; the famous\ + \ Nihilists who made Europe tremble-sons of village priests, of the lower middle\ + \ class, of tradesmen-could not rise above the idea of national liberation,\ + \ and seemed to believe that the world would be delivered-when they had killed\ + \ their despot&\u2026\n\"Foolery! They'll never get out of it with their foolery.\"\ + \nThen, lowering his voice still more, in a few bitter words he described his\ + \ old dream of fraternity. He had renounced his rank and his fortune; he had\ + \ gone among workmen, only in the hope of seeing at last the foundation of a\ + \ new society of labour in common. All the sous in his pockets had long gone\ + \ to the urchins of the settlement; he had been as tender as a brother with\ + \ the colliers, smiling at their suspicion, winning them over by his quiet workmanlike\ + \ ways and his dislike of chattering. But decidedly the fusion had not taken\ + \ place.\nHis voice changed, his eyes grew bright, he fixed them on \xE9tienne,\ + \ directly addressing him:\n\"Now, do you understand that? These hatworkers\ + \ at Marseilles who have won the great lottery prize of a hundred thousand francs\ + \ have gone off at once and invested it, declaring that they are going to live\ + \ without doing anything! Yes, that is your idea, all of you French workmen;\ + \ you want to unearth a treasure in order to devour it alone afterwards in some\ + \ lazy, selfish corner. You may cry out as much as you like against the rich,\ + \ you haven't got courage enough to give back to the poor the money that luck\ + \ brings you. You will never be worthy of happiness as long as you own anything,\ + \ and your hatred of the bourgeois proceeds solely from an angry desire to be\ + \ bourgeois yourselves in their place.\"\n\xE9mile Zola, French writer, Germinal,\ + \ 1885\nThe passage displays the direct concern for the welfare of the working\ + \ classes that was typically a part of which movement?\n(A) Capitalist (B) Scientific\ + \ (C) Communist (D) Existentialist" + target: Let's think step by step. We refer to Wikipedia articles on european history + for help. The modern Communist movement aims to establish a classless society + based on communal ownership and distribution of property and means of production, + thereby especially benefiting the working classes. The answer is (C). + - question: "This question refers to the following information.\nThe following excerpt\ + \ is from a pamphlet.\nYou will do me the justice to remember, that I have always\ + \ strenuously supported the Right of every man to his own opinion, however different\ + \ that opinion might be to mine. He who denies to another this right, makes\ + \ a slave of himself to his present opinion, because he precludes himself the\ + \ right of changing it.\nThe most formidable weapon against errors of every\ + \ kind is Reason. I have never used any other, and I trust I never shall.\n\ + The circumstance that has now taken place in France of the total abolition of\ + \ the whole national order of priesthood, and of everything appertaining to\ + \ compulsive systems of religion, and compulsive articles of faith, has not\ + \ only precipitated my intention, but rendered a work of this kind exceedingly\ + \ necessary, lest in the general wreck of superstition, of false systems of\ + \ government, and false theology, we lose sight of morality, of humanity, and\ + \ of the theology that is true.\nI believe in one God, and no more; and I hope\ + \ for happiness beyond this life.\nI believe in the equality of man; and I believe\ + \ that religious duties consist in doing justice, loving mercy, and endeavoring\ + \ to make our fellow-creatures happy.\nI do not believe in the creed professed\ + \ by the Jewish church, by the Roman church, by the Greek church, by the Turkish\ + \ church, by the Protestant church, nor by any church that I know of. My own\ + \ mind is my own church.\nAll national institutions of churches, whether Jewish,\ + \ Christian or Turkish, appear to me no other than human inventions, set up\ + \ to terrify and enslave mankind, and monopolize power and profit.\nI do not\ + \ mean by this declaration to condemn those who believe otherwise; they have\ + \ the same right to their belief as I have to mine.\n\u2014Thomas Paine, The\ + \ Age of Reason, 1794\u20131795\nWhich of the following Enlightenment philosophes\ + \ designed a system of checks and balances for government to avoid abuses of\ + \ power?\n(A) Jean Jacques Rousseau (B) Baron Montesquieu (C) Mary Wollstonecraft\ + \ (D) Adam Smith" + target: 'Let''s think step by step. We refer to Wikipedia articles on european + history for help. Baron Montesquieu was a 18th centrury French philsopher who + wrote extensively against the monoplization of power and advocated for a system + of checks and balances in government to prevent the rise of despotism. The answer + is (B).' +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_european_history diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_geography.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..96f4b365af04a3dc5c754def2899a5291ada1072 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_geography.yaml @@ -0,0 +1,53 @@ +dataset_name: high_school_geography +description: The following are multiple choice questions (with answers) about high + school geography. +fewshot_config: + sampler: first_n + samples: + - question: 'Which one of the following items is an example of nonmaterial culture? + + (A) Dove soap (B) Dove candy bar (C) Dove symbol (D) A dove (bird).' + target: Let's think step by step. We refer to Wikipedia articles on geography + for help. Nonmaterial culture consists of cultural ideas, beliefs or symbols + that are not physical objects. The answer is (C). + - question: 'During the third stage of the demographic transition model, which of the + following is true? + + (A) Birth rates increase and population growth rate is less rapid. (B) Birth + rates decline and population growth rate is less rapid. (C) Birth rates increase + and population growth rate increases. (D) Birth rates decrease and population + growth rate increases.' + target: Let's think step by step. We refer to Wikipedia articles on geography + for help. The demographic transition model models the five different stages + of population growth as a country goes through economic development, where the + third stage refers to a period of declining birth rates and lower population + growth. The answer is (B). + - question: 'The practice of hiring a foreign third-party service provider to run an + operation is called + + (A) outsourcing. (B) offshoring. (C) maquiladoras. (D) locational interdependence.' + target: Let's think step by step. We refer to Wikipedia articles on geography + for help. "Offshoring" literally means to move or base some of the activities + or processes of a company to a foreign country. The answer is (B). + - question: 'Which of the following statements is NOT accurate regarding the services + provided by local governments in the United States? + + (A) Duplication of efforts occurs often. (B) Social problems of the central + city spill over into the surrounding residential suburbs. (C) Inefficiency in + providing services occurs often. (D) One neighborhood''s efforts to reduce pollution + are always supported by neighboring communities.' + target: Let's think step by step. We refer to Wikipedia articles on geography + for help. There may be economic, social or political reasons for two neighboring + communities and their local governments not agreeing to pollution reduction + efforts initiated by one of them. The answer is (D). + - question: 'The rate of natural increase of a population is found by subtracting the + + (A) crude death rate from the crude birth date. (B) crude birth rate from the + crude death rate. (C) doubling time from the crude birth rate. (D) fertility + rate from the crude death rate.' + target: 'Let''s think step by step. We refer to Wikipedia articles on geography + for help. The difference between number of births and deaths gives the population + increase at any given time. The answer is (A).' +tag: mmlu_flan_cot_fewshot_social_sciences +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_geography diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_government_and_politics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4c11772183c0266288f95f4d273ebbdf32c0dba1 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_government_and_politics.yaml @@ -0,0 +1,61 @@ +dataset_name: high_school_government_and_politics +description: The following are multiple choice questions (with answers) about high + school government and politics. +fewshot_config: + sampler: first_n + samples: + - question: 'Which of the following best states an argument made by James Madison in + The Federalist number 10? + + (A) Honest politicians can prevent factions from developing. (B) Factions are + more likely to occur in large republics than in small ones. (C) The negative + effects of factionalism can be reduced by a republican government. (D) Free + elections are the people''s best defense against factionalism.' + target: Let's think step by step. We refer to Wikipedia articles on government + and politics for help. In the Federalist number 10, James Madison advocated + for a representative republican form of government to guard against factionalism. + The answer is (C). + - question: 'The term "budget deficit" refers to the + + (A) annual increase in federal spending on the military (B) amount of interest + on the national debt (C) difference between the initial budget proposals made + by the president and Congress (D) amount the government spends in excess of + its revenues' + target: Let's think step by step. We refer to Wikipedia articles on government + and politics for help. When the goverment spends more than it earns, their difference + is the budget deficit. The answer is (D). + - question: 'Which of the following statements about cabinet departments is FALSE? + + (A) They are established by the legislative branch. (B) Their members often + don''t have much influence over presidential decisions. (C) They cannot all + be run by leaders who belong to the same political party the president does. + (D) Not every federal agency is a cabinet department.' + target: Let's think step by step. We refer to Wikipedia articles on government + and politics for help. There is no law stipulating that some cabinet department + leaders have to belong to a political party different from that of the president. + The answer is (C). + - question: 'Which of the following cases established the precedent that a defendant + must be informed of the right to remain silent, the right to a lawyer, and protection + from self-incrimination? + + (A) Weeks v. United States (B) Betts v. Brady (C) Mapp v. Ohio (D) Miranda v. + Arizona' + target: Let's think step by step. We refer to Wikipedia articles on government + and politics for help. In the landmark Miranda v. Arizona in 1966, the US Supreme + Court, based on the Fifth and Sixth Amendment of the US Constitution, guaranteed + a defendant's right to an attorney and protection from self-incrimination. The + answer is (D). + - question: 'Uncertainty over the limits to presidential power is caused primarily + by the fact that + + (A) the constitutional definition of those powers is broad and unspecific (B) + most people agree that the Constitution places too many limits on presidential + power (C) the Supreme Court consistently refuses to rule on cases concerning + presidential powers (D) constitutional amendments have greatly increased presidential + powers' + target: 'Let''s think step by step. We refer to Wikipedia articles on government + and politics for help. The US Constitution is not very specific about the powers + of the president, leading to uncertainty over its limits. The answer is (A).' +tag: mmlu_flan_cot_fewshot_social_sciences +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_government_and_politics diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_macroeconomics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5700d5df946539608bbf10555e0f06ca07672b09 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_macroeconomics.yaml @@ -0,0 +1,53 @@ +dataset_name: high_school_macroeconomics +description: The following are multiple choice questions (with answers) about high + school macroeconomics. +fewshot_config: + sampler: first_n + samples: + - question: 'Which of the following policies best describes supply-side fiscal policy? + + (A) An increase in the money supply (B) Increased government spending (C) Lower + taxes on research and development of new technology (D) Higher taxes on household + income' + target: Let's think step by step. We refer to Wikipedia articles on macroeconomics + for help. Supply-side fiscal policy stimulates the economy by encouraging more + production of goods and services through reduction in taxes and deregulation. + The answer is (C). + - question: 'The short-run Phillips curve indicates a + + (A) direct relation between unemployment and inflation (B) direct relation between + price and quantity demanded (C) inverse relation between price and quantity + demanded (D) inverse relation between unemployment and inflation' + target: Let's think step by step. We refer to Wikipedia articles on macroeconomics + for help. The short-run Phillips curve shows that whenever unemployment decreases + below a natural level, the inflation starts increasing, and vice-versa. The + answer is (D). + - question: 'Holding all else equal which of the following monetary policies would + be used to boost U.S. exports? + + (A) Increasing the discount rate (B) Increasing the reserve ratio (C) Buying + government securities (D) Lowering tariffs' + target: Let's think step by step. We refer to Wikipedia articles on macroeconomics + for help. Buying government securities leads to reduction in demand for US dollars + from foreign buyers, thereby making it cheaper and hence making US exports more + attractive. The answer is (C). + - question: 'A federal deficit occurs when + + (A) exports exceed imports. (B) imports exceed exports. (C) federal tax collections + exceed spending. (D) federal spending exceeds federal tax revenues.' + target: Let's think step by step. We refer to Wikipedia articles on macroeconomics + for help. A federal deficit occurs when federal spending exceeds federal income + which is primarily from tax revenues. The answer is (D). + - question: 'Which of the following is not included in the U.S. GDP? + + (A) The U.S. military opens a new base in a foreign country with 1000 U.S. personnel. + (B) Japanese consumers buy thousands of CDs produced in the United States. (C) + An American pop singer performs a sold-out concert in Paris. (D) A French theatrical + production tours dozens of American cities.' + target: 'Let''s think step by step. We refer to Wikipedia articles on macroeconomics + for help. The economic transactions related to the performance of the American + pop-singer in Paris happens entirely outside the U.S. and hence is not included + in the GDP numbers. The answer is (C).' +tag: mmlu_flan_cot_fewshot_social_sciences +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_macroeconomics diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_mathematics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3e05795561a11dca35dd342bc88b4794584809c --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_mathematics.yaml @@ -0,0 +1,51 @@ +dataset_name: high_school_mathematics +description: The following are multiple choice questions (with answers) about high + school mathematics. +fewshot_config: + sampler: first_n + samples: + - question: 'Simplify and write the result with a rational denominator: $$\sqrt{\sqrt[3]{\sqrt{\frac{1}{729}}}}$$ + + (A) \frac{3\sqrt{3}}{3} (B) \frac{1}{3} (C) \sqrt{3} (D) \frac{\sqrt{3}}{3}' + target: Let's think step by step. Factoring $729=3^6$ and combining the roots + $\frac{1}{2}\frac{1}{3}\frac{1}{2}=\frac{1}{12}$, we get that $\sqrt{\sqrt[3]{\sqrt{\frac{1}{729}}}}=\left(\frac{1}{3^6}\right)^{\frac{1}{12}}=\frac{1}{3^{\frac{1}{2}}}=\frac{3}{\sqrt{3}}$ + The answer is (D). + - question: 'Five thousand dollars compounded annually at an $x\%$ interest rate takes + six years to double. At the same interest rate, how many years will it take + $\$300$ to grow to $\$9600$? + + (A) 12 (B) 1 (C) 30 (D) 5' + target: Let's think step by step. To go from $\$300$ to $\$9600$, the value must + go up by a factor of $9600/300=32=2^5$. Since at this interest rate it takes + six years for it to double, it will take $5*6=30$ years to grow to $\$9600$. + The answer is (C). + - question: "Ten students take a biology test and receive the following scores: 45,\ + \ 55, 50, 70, 65, 80, 40, 90, 70, 85. What is the mean of the students\u2019\ + \ test scores?\n(A) 55 (B) 60 (C) 62 (D) 65" + target: Let's think step by step. There are 10 students and the sum of their scores + is $45 + 55 + 50 + 70 + 65 + 80 + 40 + 90 + 70 + 85 = 650$, the mean is $650/10=65$. + The answer is (D). + - question: 'The variable $x$ varies directly as the square of $y$, and $y$ varies + directly as the cube of $z$. If $x$ equals $-16$ when $z$ equals 2, what is + the value of $x$ when $z$ equals $\frac{1}{2}$? + + (A) -1 (B) 16 (C) -\frac{1}{256} (D) \frac{1}{16}' + target: Let's think step by step. We know that $x \propto y^2$ and $y \propto + z^3$, so $x = k z^6$ for some constant $k$. Plugging in for $x=-16$ and $z=2$, + the constant value is $k=\frac{x}{z^6}=\frac{-16}{64}=-\frac{1}{4}$. So, when + $z=\frac{1}{2}$, the value of $x$ is $x=kz^6=-\frac{1}{4}\frac{1}{2^6}=-\frac{1}{256}$. + The answer is (C). + - question: 'Joe was in charge of lights for a dance. The red light blinks every two + seconds, the yellow light every three seconds, and the blue light every five + seconds. If we include the very beginning and very end of the dance, how many + times during a seven minute dance will all the lights come on at the same time? + (Assume that all three lights blink simultaneously at the very beginning of + the dance.) + + (A) 3 (B) 15 (C) 6 (D) 5' + target: 'Let''s think step by step. The least common multiple of 2, 3 and 5 is + 30, so during a 7 minute dance, all the three lights will come on at the same + time $2*7+1=15$ times. The answer is (B).' +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_mathematics diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_microeconomics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ead78be898187a7c08c292eaeebfe6b067ef4413 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_microeconomics.yaml @@ -0,0 +1,56 @@ +dataset_name: high_school_microeconomics +description: The following are multiple choice questions (with answers) about high + school microeconomics. +fewshot_config: + sampler: first_n + samples: + - question: 'Which of the following is necessarily a characteristic of oligopoly? + + (A) Free entry into and exit from the market (B) A few large producers (C) One + producer of a good with no close substitutes (D) A homogenous product' + target: Let's think step by step. We refer to Wikipedia articles on microeconomics + for help. An oligopoly is when a market is dominated by just one or a few number + of sellers or producers. To get oligopoly, the market should have high barriers + to new entry, and the product has differentiation. The answer is (B). + - question: 'If the government subsidizes producers in a perfectly competitive market, + then + + (A) the demand for the product will increase (B) the demand for the product + will decrease (C) the consumer surplus will increase (D) the consumer surplus + will decrease' + target: Let's think step by step. We refer to Wikipedia articles on microeconomics + for help. (A) and (B) are wrong because the demand curve does not change at + all. If the government subsidizes producers, the supply will increase, and thus + the consumer surplus also increases. The answer is (C). + - question: 'Which of the following is true of a price floor? + + (A) The price floor shifts the demand curve to the left. (B) An effective floor + creates a shortage of the good. (C) The price floor shifts the supply curve + of the good to the right. (D) To be an effective floor, it must be set above + the equilibrium price.' + target: Let's think step by step. We refer to Wikipedia articles on microeconomics + for help. Price floor does not shift the demand or shift curve. An effective + price floor should be set above the equilibrium price, otherwise the market + bears and the floor does not have effective effect. The answer is (D). + - question: 'The concentration ratio for a monopoly is + + (A) 0 (B) 5 (C) 10 (D) 100' + target: Let's think step by step. We refer to Wikipedia articles on microeconomics + for help. The concentration ratio is calculated as the sum of market share of + a specific number of largest companies. Monopoly means one company or entity + controls the entire market, therefore, the concentration ratio is 100 percent. + The answer is (D). + - question: 'In a competitive labor market for housepainters, which of the following + would increase the demand for housepainters? + + (A) An effective minimum wage imposed on this labor market. (B) An increase + in the price of gallons of paint. (C) An increase in the construction of new + houses. (D) An increase in the price of mechanical painters so long as the output + effect exceeds the substitution effect.' + target: 'Let''s think step by step. We refer to Wikipedia articles on microeconomics + for help. An increase in the construction of new houses means an increase demand + of in-house painting, thus increases the demand for housepainters. The answer + is (C).' +tag: mmlu_flan_cot_fewshot_social_sciences +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_microeconomics diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_physics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5d2166b8595529213797cf12efd67f90ca4e9e61 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_physics.yaml @@ -0,0 +1,50 @@ +dataset_name: high_school_physics +description: The following are multiple choice questions (with answers) about high + school physics. +fewshot_config: + sampler: first_n + samples: + - question: 'A microwave oven is connected to an outlet, 120 V, and draws a current + of 2 amps. At what rate is energy being used by the microwave oven? + + (A) 10 W (B) 30 W (C) 60 W (D) 240 W' + target: Let's think step by step. Rate of energy usage is known as power; in an + dissipative electrical circuit, power is given by voltage times current. So + in our case, the power is 120 V times 2 amps, or 240 W. The answer is (D). + - question: "A point charge, Q = +1 mC, is fixed at the origin. How much work is required\ + \ to move a charge, Q = +8 \xB5C, from the point (0, 4 meters) to the point\ + \ (3 meters, 0)?\n(A) 3.5 J (B) 6.0 J (C) 22.5 J (D) 40 J" + target: "Let's think step by step. To calculate the work required to move a charge\ + \ from one location to another in a fixed electric field, it is enough to calculate\ + \ the potential difference between the two locations. Here, the potential only\ + \ depends on the distance between the charges; it\u2019s $k q_1 q_2 / r$, where\ + \ $k$ is Coulomb\u2019s constant. Plugging in values $q_1 = $ 1 mC, $q_2 = 8\ + \ \\mu$ C, gives the answer as 5.992 J, which rounds to 6 J. The answer is (B)." + - question: 'Which of the following conditions will ensure that angular momentum is + conserved? I. Conservation of linear momentum II. Zero net external force III. + Zero net external torque + + (A) I and II only (B) I and III only (C) II and III only (D) III only' + target: Let's think step by step. Torque is defined as the change in angular momentum; + if there is zero external torque, angular momentum is conserved. The answer + is (D). + - question: "A photocell of work function \u03D5 = 2eV is connected to a resistor in\ + \ series. Light of frequency f = 1 \xD7 10^15 Hz hits a metal plate of the photocell.\ + \ If the power of the light is P = 100 W, what is the current through the resistor?\n\ + (A) 2:00 AM (B) 6:00 AM (C) 12:00 AM (D) 24 A" + target: Let's think step by step. The only answer above which has units of current + is D, 24 A. The answer is (D). + - question: "A pipe full of air is closed at one end. A standing wave is produced in\ + \ the pipe, causing the pipe to sound a note. Which of the following is a correct\ + \ statement about the wave\u2019s properties at the closed end of the pipe?\n\ + (A) The pressure is at a node, but the particle displacement is at an antinode.\ + \ (B) The pressure is at an antinode, but the particle displacement is at a\ + \ node. (C) The pressure and the particle displacement are both at nodes. (D)\ + \ The pressure and the particle displacement are both at antinodes." + target: 'Let''s think step by step. At the closed end of the pipe, the particles + cannot have any net displacement because the pipe closure stops them. So the + particle displacement is at a node. This closure also causes the pressure to + be maximal, i.e. an antinode. The answer is (B).' +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_physics diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_psychology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..553fe18de24f14765d14d0fb2dbdfaf72449af32 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_psychology.yaml @@ -0,0 +1,64 @@ +dataset_name: high_school_psychology +description: The following are multiple choice questions (with answers) about high + school psychology. +fewshot_config: + sampler: first_n + samples: + - question: 'Pascale is interested in the processing strategies children use to learn + new information. Pascale would best be classified as what type of psychologist? + + (A) sociocultural (B) clinical (C) cognitive (D) behaviorist' + target: Let's think step by step. We refer to Wikipedia articles on psychology + for help. Sociocultural psychologist focuses on the effect of societal factors + on people. Clinical psychologist focuses on people with mental issues. Cognitive + psychologist focuses on how people think and learn, including the processing + strategies. Behaviorist focuses more on the environment and experience effect + on people. The answer is (C). + - question: 'According to Caplan''s model of consultee-centered case consultation, + the consultant is primarily interested in + + (A) identifying the causes and solutions of the client''s presenting problems + (B) identifying and eliminating the causes of the consultee''s difficulties + in handling a problem (C) establishing a hierarchy of authority to enable effective + decision making (D) presenting a single, well-defined and unambiguous course + of action for the consultant to overcome skills deficits' + target: Let's think step by step. We refer to Wikipedia articles on psychology + for help. Caplan defines two type of consultation. Client-centered case consultation + aims to handle client's problems, while consultee-centered case consultation + aims to identify the reason of client's difficulty to solve problems. The answer + is (B). + - question: 'According to the Individuals with Disabilities Education Improvement Act, + which of the following must an educational agency do before it changes the educational + placement of a student with a disability? + + (A) Give the child a trial period in the new environment (B) Notify the parents + in writing (C) Obtain school board approval (D) Obtain parental consent' + target: Let's think step by step. We refer to Wikipedia articles on psychology + for help. When the decision to change the educational placement of a student + with a disability is made, the educational agency must notify the parents in + writing on that date. The answer is (B). + - question: 'While swimming in the ocean, Ivan is frightened by a dark shadow in the + water even before he has the chance to identify what the shadow is. The synaptic + connections taking place during this incident of fright are best described by + which of the following? + + (A) Messages are sent from the thalamus directly to the amygdala. (B) Messages + are sent from the thalamus to the "what" and "where" pathways. (C) Messages + are sent from the parasympathetic nervous system to the cerebral cortex. (D) + Messages are sent from the frontal lobes to the pituitary gland.' + target: Let's think step by step. We refer to Wikipedia articles on psychology + for help. Our neural system has a mechanism that can respond immediate emotional + signal before going to the thought center. In the Ivan's case, messages travel + directly from thalamus to amygdala. The answer is (A). + - question: 'Ani believes that her attitudes and behavior play a central role in what + happens to her. Such a belief is likely to be associated with + + (A) a strong superego. (B) low self-esteem. (C) low self-efficacy. (D) an internal + locus of control.' + target: 'Let''s think step by step. We refer to Wikipedia articles on psychology + for help. People with an external locus of control believes fate and luck play + an important role in their lives, while people with an internal locus of control + believes they control their lives. The answer is (D).' +tag: mmlu_flan_cot_fewshot_social_sciences +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_psychology diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_statistics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..66ed702a6b9b57dc69719a13923e04ebc8218ea9 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_statistics.yaml @@ -0,0 +1,81 @@ +dataset_name: high_school_statistics +description: The following are multiple choice questions (with answers) about high + school statistics. +fewshot_config: + sampler: first_n + samples: + - question: 'A new smartwatch is manufactured in one part of a factory, then secured + for shipping in another, independent part of the factory. The weight of the + smartwatch has a mean of 62 grams and a standard deviation of 1.0 grams. The + weight of the packaging (box, user''s guide, bubble wrap, etc.) has a mean of + 456 grams and a standard deviation of 6 grams. Together, the distribution of + the weight of the smartwatch and its packaging would have the following mean + and standard deviation: + + (A) Mean 518 grams; standard deviation 7.0 grams (B) Mean 518 grams; standard + deviation 3.5 grams (C) Mean 518 grams; standard deviation 6.1 grams (D) Mean + 394 grams; standard deviation 6.1 grams' + target: Let's think step by step. Since the weight of the watch and the weight + of the packaging are independent random variables, the mean and variance of + their sum is equal to the sum of their individual means and variances. So the + mean is 62 + 456 = 518 grams, and the variances is 1.0^2 + 6.0^2 = 37, leading + to a standard deviation of 6.1 grams. The answer is (C). + - question: 'After a frost warning was issued, the owner of a large orange grove asked + his workers to spray all his trees with water. The water was supposed to freeze + and form a protective covering of ice around the orange blossom. Nevertheless, + the owner suspected that some trees suffered considerable damage due to the + frost. To estimate the proportion of trees that suffered more than 50 percent + damage due to the frost, he took a random sample of 100 trees from his grove. + What is the response variable in this experiment? + + (A) The proportion of trees that suffered more than 50 percent damage due to + frost. (B) The number of trees affected by the frost. (C) The number of trees + sampled from the grove. (D) For each sampled tree, whether it suffered more + than 50 percent damage or at most 50 percent damage.' + target: Let's think step by step. In this experiment, the response variable is + what is measured. For each tree, what is measured is whether or not it suffered + more than 50 percent damage due to the frost. The answer is (D). + - question: 'Suppose X and Y are random variables with E(X) = 37, var(X) = 5, E(Y) + = 62, and var(Y) = 12. What are the expected value and variance of the random + variable X + Y? + + (A) E(X + Y) = 99, var(X + Y) = 8.5 (B) E(X + Y) = 99, var(X + Y) = 13 (C) E(X + + Y) = 99, var(X + Y) = 17 (D) There is insufficient information to answer this + question.' + target: Let's think step by step. While means of sums of random variables add + (regardless of whether the variables are independent) in order to determine + the variance of a sum of random variables, we need to know not just their individual + variances but the covariance of the two variables, which is not given in this + problem. The answer is (D). + - question: 'Which of the following sets has the smallest standard deviation? Which + has the largest? + + I: {1,2,3} + + II: {-10,10} + + III: {100} + + (A) I, II (B) II, III (C) III, I (D) III, II' + target: Let's think step by step. The variance of distribution I is the expected + squared deviation from its mean (which is 2), so the variance is 2/3 . The variance + of distribution II is 10^2 (because both elements are 10 away from the mean + of zero). The variance of distribution III is 0, since it has a single entry. + So distribution III has the smallest standard deviation and distribution II + has the largest. The answer is (D). + - question: 'Which of the following is a correct statement about correlation? + + (A) If the slope of the regression line is exactly 1, then the correlation is + exactly 1. (B) If the correlation is 0, then the slope of the regression line + is undefined. (C) Switching which variable is called x and which is called y + changes the sign of the correlation. (D) The correlation r is equal to the slope + of the regression line when z-scores for the y-variable are plotted against + z-scores for the x-variable.' + target: 'Let''s think step by step. Statement A is false because the slope of + the regression line being exactly 1 can occur even when the two variables are + not perfectly correlated. Statement B is false because uncorrelated variables + regression lines can have slope zero. Statement C is false because correlation + is symmetric in the two random variables. The answer is (D).' +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_statistics diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_us_history.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8cea5109f6570086dce3cf1815dc50f1889d80ad --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_us_history.yaml @@ -0,0 +1,156 @@ +dataset_name: high_school_us_history +description: The following are multiple choice questions (with answers) about high + school us history. +fewshot_config: + sampler: first_n + samples: + - question: "This question refers to the following information.\nI come not to urge\ + \ personal claims, nor to seek individual benefits; I appear as the advocate\ + \ of those who cannot plead their own cause; I come as the friend of those who\ + \ are deserted, oppressed, and desolate. In the Providence of God, I am the\ + \ voice of the maniac whose piercing cries from the dreary dungeons of your\ + \ jails penetrate not your Halls of Legislation. I am the Hope of the poor crazed\ + \ beings who pine in the cells, and stalls, and cages, and waste rooms of your\ + \ poor-houses. I am the Revelation of hundreds of wailing, suffering creatures,\ + \ hidden in your private dwellings, and in pens and cabins\u2014shut out, cut\ + \ off from all healing influences, from all mind-restoring cares.\u2026 Could\ + \ their melancholy histories be spread before you as revealed to my grieved\ + \ spirit during the last three months, how promptly, how earnestly would you\ + \ search out the most approved means of relief; how trifling, how insignificant,\ + \ by comparison, would appear the sacrifices you are asked to make; how would\ + \ a few dimes and dollars, gathered from each citizen, diminish in value as\ + \ a possession, compared with the certain benefits and vast good to be secured\ + \ for the suffering insane...by the consecration and application of a sufficient\ + \ fund to the construction of a suitable hospital.\u2026\n\u2014Dorothea Dix,\ + \ Memorial Soliciting a State Hospital for the Protection and Cure of the Insane,\n\ + Submitted to the General Assembly of North Carolina, November 1848\nDorothea\ + \ Dix can best be compared to whom?\n(A) Abigail Adams (B) Clara Barton (C)\ + \ Shirley Temple (D) Hillary Clinton" + target: Let's think step by step. We refer to Wikipedia articles on us history + for help. Both Dorothea Dix and Clara barton are American nurses. The answer + is (B). + - question: "This question refers to the following information.\n\"As our late Conduct\ + \ at the Conestoga Manor and Lancaster have occasioned much Speculation & a\ + \ great diversity of Sentiments in this and neighboring Governments; some vindicating\ + \ & others condemning it; some charitably alleviating the Crime, & others maliciously\ + \ painting it in the most odious & detestable Colours, we think it our duty\ + \ to lay before the Publick, the whole Matter as it appeared, & still appears,\ + \ to us. . . .\n\"If these things are not sufficient to prove an unjustifiable\ + \ Attachment in the Quakers to the Indians Savages, a fixed Resolution to befriend\ + \ them & an utter insensibility to human Distresses, let us consider a few more\ + \ recent Facts. When we found the last Summer that we were likely to get no\ + \ Assistance from the Government, some Volunteers went out at our own Expense,\ + \ determined to drive our Enemies from our Borders; & when we came near to the\ + \ great Island, we understood that a Number of their Warriors had gone out against\ + \ our Frontiers. Upon this we returned and came up with them and fought with\ + \ them at the Munfey Hill where we lost some of our Men & killed some of their\ + \ Warriors & thereby saved our Frontiers from this Story in another Expedition.\ + \ But no sooner had we destroyed their Provisions on the great Island, & ruined\ + \ their trade with the good People at Bethlehem, but these very Indians, who\ + \ were justly suspected of having murdered our Friends in Northampton County,\ + \ were by the Influence of some Quakers taken under the Protection of the Government\ + \ to screen them from the Resentments of the Friends and Relations of the Murdered,\ + \ & to support them thro the Winter.\"\n\u2014\"Apology of the Paxton Boys\"\ + \ (pamphlet), 1764 (Note: \"apology\" in this context should be read as an explanation,\ + \ not an admission of guilt or regret.\nThe sentiments expressed in the explanation\ + \ above reflect which of the ongoing tensions during the colonial period of\ + \ American history?\n(A) Tensions between British policies and the aspirations\ + \ of North American colonists. (B) Tensions between American Indians allied\ + \ with the French and those allied with the British. (C) Tensions between freed\ + \ African Americans and white planters. (D) Tensions between backcountry settlers\ + \ and elites within colonial America." + target: Let's think step by step. We refer to Wikipedia articles on us history + for help. After the French and Indian War, the Scotch-Irish settlers attacked + American Indians. After the attacks on the Conestoga, about 250 Paxton Boys + present their grievances to the Pennsylvania legislature. As mentioned in the + information, the Paxton Boys cited resentiment at local elites. The answer is + (D). + - question: "This question refers to the following information.\nOur leaders talk about\ + \ stopping aggression from the north, but this was a struggle among groups of\ + \ Vietnamese until we intervened. We seem bent upon saving the Vietnamese from\ + \ Ho Chi Minh even if we have to kill them and demolish their country to do\ + \ it. As the native people survey bombed-out villages, women and children burned\ + \ by napalm, rice crops destroyed and cities overrun with our military personnel,\ + \ they are doubtless saying secretly of the Vietcong guerillas and of the American\ + \ forces, \"A plague on both your houses.\" \u2026 Stop the bombing, north and\ + \ south, end search and destroy offensive sweeps, and confine our military action\ + \ to holding operations on the ground. Bombing the north has failed to halt\ + \ or seriously check the flow of troops to the south and may, in fact, have\ + \ prompted a much greater war effort by Hanoi.\n\u2014Senator George McGovern,\ + \ \"The Lessons of Vietnam,\" April 25, 1967\nWhich of the following opinions\ + \ from the 1960s most directly reflects the perspective of George McGovern's\ + \ speech?\n(A) Americans must maximize their technological edge in Vietnam.\ + \ (B) American bombing in Vietnam is step by step leading to progress in the\ + \ war. (C) American bombing in Vietnam is a failure. (D) America must not give\ + \ in to defeatism about the war in Vietnam." + target: Let's think step by step. We refer to Wikipedia articles on us history + for help. "Stop the bombing" and "Bombing the north has failed to halt or seriously + check the flow of troops to the south" indicate that the perspective of George + McGovern's speech is that Amerian bombing in Vietnam is a failure. The answer + is (C). + - question: "This question refers to the following information.\n\"In the new Code\ + \ of Laws which I suppose it will be necessary for you to make I desire you\ + \ would Remember the Ladies, and be more generous and favorable to them than\ + \ your ancestors. Do not put such unlimited power into the hands of the Husbands.\ + \ Remember all Men would be tyrants if they could. If particular care and attention\ + \ is not paid to the Ladies we are determined to foment a Rebellion, and will\ + \ not hold ourselves bound by any Laws in which we have no voice, or Representation.\"\ + \nAbigail Adams, in a letter to John Adams, 1776\n\"Special legislation for\ + \ woman has placed us in a most anomalous position. Women invested with the\ + \ rights of citizens in one section\u2014voters, jurors, office-holders\u2014\ + crossing an imaginary line, are subjects in the next. In some States, a married\ + \ woman may hold property and transact business in her own name; in others,\ + \ her earnings belong to her husband. In some States, a woman may testify against\ + \ her husband, sue and be sued in the courts; in others, she has no redress\ + \ in case of damage to person, property, or character. In case of divorce on\ + \ account of adultery in the husband, the innocent wife is held to possess no\ + \ right to children or property, unless by special decree of the court. But\ + \ in no State of the Union has the wife the right to her own person, or to any\ + \ part of the joint earnings of the co-partnership during the life of her husband.\ + \ In some States women may enter the law schools and practice in the courts;\ + \ in others they are forbidden. In some universities girls enjoy equal educational\ + \ advantages with boys, while many of the proudest institutions in the land\ + \ deny them admittance, though the sons of China, Japan and Africa are welcomed\ + \ there. But the privileges already granted in the several States are by no\ + \ means secure.\"\nSusan B. Anthony, \"Declaration of Rights for Women,\" July\ + \ 4, 1876\nThe sentiments expressed in the second excerpt by Susan B. Anthony\ + \ are most likely in support of\n(A) the Equal Rights Amendment (B) universal\ + \ suffrage (C) states' rights (D) prohibition" + target: Let's think step by step. We refer to Wikipedia articles on us history + for help. The above information mentioned that women are in an anomalous position + in terms of legislation. Women's earnings do not belong to themselves, or they + cannot testify against her husbands. Susan believes women should have equal + legal rights as men. The answer is (B). + - question: 'This question refers to the following information. + + "Society in every state is a blessing, but government even in its best state + is but a necessary evil; in its worst state an intolerable one; for when we + suffer, or are exposed to the same miseries by a government, which we might + expect in a country without government, our calamity is heightened by reflecting + that we furnish the means by which we suffer. Government, like dress, is the + badge of lost innocence; the palaces of kings are built on the ruins of the + bowers of paradise. For were the impulses of conscience clear, uniform, and + irresistibly obeyed, man would need no other lawgiver; but that not being the + case, he finds it necessary to surrender up a part of his property to furnish + means for the protection of the rest; and this he is induced to do by the same + prudence which in every other case advises him out of two evils to choose the + least. Wherefore, security being the true design and end of government, it unanswerably + follows that whatever form thereof appears most likely to ensure it to us, with + the least expense and greatest benefit, is preferable to all others." + + Thomas Paine, Common Sense, 1776 + + Which of the following "miseries" alluded to above were most condemned by Anti-Federalists + of the post-Revolutionary era? + + (A) Organized response to Bacon''s Rebellion (B) Federal response to Shays''s + Rebellion (C) Federal response to the Whiskey Rebellion (D) Federal response + to Pontiac''s Rebellion' + target: 'Let''s think step by step. We refer to Wikipedia articles on us history + for help. Anti-Federalists do not believe centralized government power, and + suspect Washington''s military response to Whiskey Rebellion. Bacon''s Rebellion + and Pontiac''s Rebellion happen before the Revolution and they can be ruled + out. The answer is (C).' +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_us_history diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_world_history.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2fbdaf05c137270f4ff4207e7c6ce81c2a34d30c --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_world_history.yaml @@ -0,0 +1,100 @@ +dataset_name: high_school_world_history +description: The following are multiple choice questions (with answers) about high + school world history. +fewshot_config: + sampler: first_n + samples: + - question: "This question refers to the following information.\n\"At least one of\ + \ the [world's] societies would have to somehow enormously increase its productivity\ + \ [in order to achieve global hegemony]. That quantum jump would have to be\ + \ made before the various scientific, technological, agricultural, and industrial\ + \ revolutions on which our post-quantum-leap world rests. It could only be accomplished\ + \ by exploiting the ecosystems, mineral resources, and human assets of whole\ + \ continents outside the lands of the society making the jump. Western Europe\ + \ did just that by means of its brutality and guns and, more important, by geographical\ + \ and ecological luck.\"\nCopyright \xA9 2015 Cambridge University Press.\n\ + Alfred Crosby, historian, Ecological Imperialism, 2004\nThe \"quantum jump\"\ + \ mentioned in the passage most directly contributed to which of the following\ + \ developments in the period 1450\u20131750 C.E.?\n(A) A breakdown in trade\ + \ routes through the collapse of the established state structure (B) An increase\ + \ in the population of the world through more plentiful supplies of food (C)\ + \ The spread of Chinese and Indian belief systems across the world (D) An increase\ + \ in social unrest" + target: Let's think step by step. We refer to Wikipedia articles on world history + for help. The "quantum jump" mentioned in the passage refers to the conquest + of the New World and the Columbian Exchange. Choice (A) and (C) did not happen + in history. Choice (C) refers to the human assets. The answer is (B). + - question: "This question refers to the following information.\n\"The struggle against\ + \ neo-colonialism is not aimed at excluding the capital of the developed world\ + \ from operating in less developed countries. It is aimed at preventing the\ + \ financial power of the developed countries being used in such a way as to\ + \ impoverish the less developed.\nNon-alignment, as practiced by Ghana and many\ + \ other countries, is based on co-operation with all States whether they be\ + \ capitalist, socialist or have a mixed economy. Such a policy, therefore, involves\ + \ foreign investment from capitalist countries, but it must be invested in accordance\ + \ with a national plan drawn up by the government of the non-aligned State with\ + \ its own interests in mind. The issue is not what return the foreign investor\ + \ receives on his investments\u2026The question is one of power. A State in\ + \ the grip of neo-colonialism is not master of its own destiny.\"\nKwame Nkrumah,\ + \ Neo-Colonialism, 1965\nWhich of the following provides the best context for\ + \ Nkrumah's writings?\n(A) The Industrial Revolution (B) Decolonization (C)\ + \ Regional Free Trade Associations (D) Autarky" + target: Let's think step by step. We refer to Wikipedia articles on world history + for help. The passage expresses a point that the successful fight against neo-colonialism + were in danger and the newly independent nations like Ghana may be re-colonized + via financial power of the developed countries. The answer is (B). + - question: "This question refers to the following information.\n\"Indeed, as both\ + \ the fatwas of distinguished [scholars] who base their opinion on reason and\ + \ tradition alike and the consensus of the Sunni community agree that the ancient\ + \ obligation of extirpation, extermination, and expulsion of evil innovation\ + \ must be the aim of our exalted aspiration, for \"Religious zeal is a victory\ + \ for the Faith of God the Beneficent\"; then, in accordance with the words\ + \ of the Prophet (Peace upon him!) \"Whosoever introduces evil innovation into\ + \ our order must be expelled\" and \"Whosoever does aught against our order\ + \ must be expelled,\" action has become necessary and exigent\u2026\"\nLetter\ + \ from Ottoman Sultan Selim I to Safavid Shah Ismail I, 1514\nThe letter from\ + \ Selim I is most clearly an example of which of the following?\n(A) The maintenance\ + \ of military supremacy at all costs (B) Expanding tensions between religious\ + \ sects (C) Factors that brought about the collapse of the Ottoman Empire (D)\ + \ Peacemaking efforts among the Islamic empires" + target: Let's think step by step. We refer to Wikipedia articles on world history + for help. The passage is an example of expanding tensions between Selim and + Ismail. In the passage the Selim references the fatwa and the consensus of the + Sunni community to against whosoever introduces evil. The answer is (B). + - question: 'This question refers to the following information. + + "The real grievance of the worker is the insecurity of his existence; he is + not sure that he will always have work, he is not sure that he will always be + healthy, and he foresees that he will one day be old and unfit to work. If he + falls into poverty, even if only through a prolonged illness, he is then completely + helpless, exam_ins to his own devices, and society does not currently recognize + any real obligation towards him beyond the usual help for the poor, even if + he has been working all the time ever so faithfully and diligently. The usual + help for the poor, however, leaves a lot to be desired, especially in large + cities, where it is very much worse than in the country." + + Otto von Bismarck, 1884 + + Otto von Bismarck likely made this speech in reaction to which of the following + issues? + + (A) Social acceptance of child labor (B) Declining life expectancy in Germany + (C) Criticisms of German trade tariffs (D) Negative effects attributed to industrial + capitalism' + target: Let's think step by step. We refer to Wikipedia articles on world history + for help. The passage talks about the grievance of the work under the industrial + capitalism. The answer is (D). + - question: "This question refers to the following information.\nHe contains all works\ + \ and desires and all perfumes and all tastes. He enfolds the whole universe\ + \ and in silence is loving to all. This is the Spirit that is in my heart, this\ + \ is Brahman. To him I shall come when I go beyond this life, and to him will\ + \ come he who has faith and doubts not.\n\u2014The Upanishads, India, c. 1000\ + \ BCE\nTo which religion does the speaker most likely belong?\n(A) Hinduism\ + \ (B) Buddhism (C) Shintoism (D) Zoroastrianism" + target: 'Let''s think step by step. We refer to Wikipedia articles on world history + for help. Brahman refers to the ultimate reality of all things in the Hindu + religion. In contrast, Buddhism does not have a concept of supreme God. The + answer is (A).' +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_high_school_world_history diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_human_aging.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3eec010845fa68ad974bdb7cd922a0028365d96e --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_human_aging.yaml @@ -0,0 +1,42 @@ +dataset_name: human_aging +description: The following are multiple choice questions (with answers) about human + aging. +fewshot_config: + sampler: first_n + samples: + - question: 'All other things being equal, which of the following persons is more likely + to show osteoporosis? + + (A) An older Hispanic American woman (B) An older African American woman (C) + An older Asian American woman (D) An older Native American woman' + target: Let's think step by step. We refer to Wikipedia articles on human aging + for help. Although osteoporosis can occur at any age, the risk is higher for + older people. It is most common in Asian and non-Hispanic white women. The answer + is (C). + - question: 'The finding that adults tend to remember events from their adolescence + better than from other periods in their lives is referred to as the + + (A) Adolescence advantage (B) Reminiscence bump (C) Memorial memorial (D) Quadratic + retrieval spike' + target: Let's think step by step. We refer to Wikipedia articles on human aging + for help. Reminiscence bump is a phenomenon that older adults tend to recollect + events during their young ages. People usually have a period of childhood amnesia + from birth to around age 5, and a reminiscence bump between 10 and 30. The answer + is (B). + - question: 'Which element in tobacco smoke is responsible for cancers? + + (A) Nicotine (B) Tar (C) Carbon monoxide (D) Smoke particles' + target: Let's think step by step. We refer to Wikipedia articles on human aging + for help. The benzene, acrylamide and acrylonitrile in tar interact with the + lungs and cause DNA mutations in cells of the lungs, and lead to cancer. The + answer is (B). + - question: 'When older adults move to a new state after retirement, which of the following + is the more likely destination? + + (A) Texas (B) California (C) Hawaii (D) Vermont' + target: 'Let''s think step by step. We refer to Wikipedia articles on human aging + for help. Texas does not have state tax, and has low cost of living compared + with the other three options. The answer is (A).' +tag: mmlu_flan_cot_fewshot_other +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_human_aging diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_human_sexuality.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dab78f0c5ec1042d23240bb71f59b212885585aa --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_human_sexuality.yaml @@ -0,0 +1,50 @@ +dataset_name: human_sexuality +description: The following are multiple choice questions (with answers) about human + sexuality. +fewshot_config: + sampler: first_n + samples: + - question: 'The most common disorder among men who seek sexual therapy is: + + (A) premature ejaculation (B) inhibited ejaculation (C) erectile disorder (D) + ejaculatory disorder' + target: Let's think step by step. We refer to Wikipedia articles on human sexuality + for help. The most common type of arousal disorder for men is erectile dysfunction, + meaning that a person is not able to get any physical satisfaction from sexual + activity although he may be interested in it. The answer is (C). + - question: 'A woman who knows she has active herpes and untreated syphilis but continues + to have sex without informing her partners of her condition has, in psychoanalytic + terms: + + (A) a strong ego (B) a weak superego (C) a weak id (D) a strong superego' + target: Let's think step by step. We refer to Wikipedia articles on human sexuality + for help. A person with weak superego tends to be delinquent, criminal or have + antisocial personality. The action of the woman who knows she has active venereal + disease but still have sex with her partners indicate she may has antisocial + personality. The answer is (B). + - question: 'The nature of homosexual activities that occur during preadolescence include + all but which of the following? + + (A) sexual intercourse (B) circle jerks (C) exhibitionism (D) touching each + other''s genitals' + target: Let's think step by step. We refer to Wikipedia articles on human sexuality + for help. Sexual intercourse is prohibited during preadolescence. The answer + is (A). + - question: 'Women''s ability to have multiple orgasms is primarily due to: + + (A) the fact that they do not have a refractory period. (B) the response of + the inner layers of the vagina. (C) having alternating orgasms in different + locations. (D) the G-Spot.' + target: Let's think step by step. We refer to Wikipedia articles on human sexuality + for help. The refractory period is the time when a person is not able to be + erect or is not interested in sex. The answer is (A). + - question: 'Morning sickness is typically a problem: + + (A) during the first trimester (B) during the second trimester (C) during the + third trimester (D) all through the pregnancy' + target: 'Let''s think step by step. We refer to Wikipedia articles on human sexuality + for help. Morning sickness usually begins by nine weeks after conception, corresponding + to the first trimester. The answer is (A).' +tag: mmlu_flan_cot_fewshot_social_sciences +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_human_sexuality diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_international_law.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..99341f395352d7b6a9a8d1a71005ca821ac9b723 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_international_law.yaml @@ -0,0 +1,70 @@ +dataset_name: international_law +description: The following are multiple choice questions (with answers) about international + law. +fewshot_config: + sampler: first_n + samples: + - question: 'How the consent to be bound of a State may be expressed? + + (A) The consent of a State to be bound is expressed only by ratification (B) + The consent of a state to be bound by a treaty may be expressed by signature, + ratification, acceptance, approval or accession (C) The consent of a State to + be bound is expressed by signature (D) The consent of a State to be bound is + expressed by whatever means they choose' + target: Let's think step by step. We refer to Wikipedia articles on international + law for help. Article 11 of Vienna Convention on the Law of Treaties signed + in 1969 states that "the consent of a State to be bound by a treaty may be expressed + by signature, exchange of instruments constituting a treaty, ratification, acceptance, + approval or accession, or by any other means if so agreed." (B) is the most + precise and accurate answer. The answer is (B). + - question: 'What is the judge ad hoc? + + (A) If a party to a contentious case before the ICJ does not have a national + sitting as judge, it is entitled to nominate someone as a judge solely for that + case, with the title of judge ad hoc (B) Judge ad hoc is the member of the bench + of the ICJ with a casting vote (C) Judge ad hoc is a surrogate judge, in case + a judge is disqualified or passes away (D) Judge ad hoc is the judge that each + party will always nominate in every contentious case' + target: Let's think step by step. We refer to Wikipedia articles on international + law for help. As "ad hoc" implies, a judge ad hoc is appointed only for a specific + case or period, when a party to a contentious case before the International + Court of Justice does not have a regular national sitting as judge. The answer + is (A). + - question: 'When ''consent'' can serve as a circumstance precluding the wrongfulness + of a State conduct? + + (A) Consent can serve as a circumstance precluding the wrongfulness whenever + it is given (B) Consent can never serve as a circumstance precluding wrongfulness + (C) Consent can serve as a circumstance precluding wrongfulness, provided the + consent is valid and to the extent that the conduct remains within the limits + of the consent given (D) Consent can always serve as a circumstance precluding + wrongfulness, no matter which organ of the State gives it' + target: Let's think step by step. We refer to Wikipedia articles on international + law for help. Valid consent can serve as a circumstance precluding the wrongfulness + of a State conduct if the conduct remains within the limits of that consent, + according to Chapter V of the Responsibility of States for Internationally Wrongful + Acts, 2001, United Nations. The answer is (C). + - question: 'Would a reservation to the definition of torture in the ICCPR be acceptable + in contemporary practice? + + (A) This is an acceptable reservation if the reserving country''s legislation + employs a different definition (B) This is an unacceptable reservation because + it contravenes the object and purpose of the ICCPR (C) This is an unacceptable + reservation because the definition of torture in the ICCPR is consistent with + customary international law (D) This is an acceptable reservation because under + general international law States have the right to enter reservations to treaties' + target: Let's think step by step. We refer to Wikipedia articles on international + law for help. For it contravenes the object and purpose of the ICCPR, this is + an unacceptable reservation in contemporary practice. The answer is (B). + - question: 'What types of force does Article 2(4) of the UN Charter prohibit? + + (A) Article 2(4) encompasses only armed force (B) Article 2(4) encompasses all + types of force, including sanctions (C) Article 2(4) encompasses all interference + in the domestic affairs of States (D) Article 2(4) encompasses force directed + only against a State''s territorial integrity' + target: 'Let''s think step by step. We refer to Wikipedia articles on international + law for help. Article 2(4) of the UN Charter prohibits states from using armed + forces in their international relations. The answer is (A).' +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_international_law diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_jurisprudence.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c714f7e595a48d28b750479a46183a02fc24dc0 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_jurisprudence.yaml @@ -0,0 +1,59 @@ +dataset_name: jurisprudence +description: The following are multiple choice questions (with answers) about jurisprudence. +fewshot_config: + sampler: first_n + samples: + - question: 'Iverson Jewelers wrote a letter to Miller, ''We have received an exceptionally + fine self winding Rolox watch which we will sell to you at a very favorable + price.'' + + (A) The letter is an offer to sell (B) A valid offer cannot be made by letter. + (C) The letter contains a valid offer which will terminate within a reasonable + time. (D) The letter lacks one of the essential elements of an offer.' + target: Let's think step by step. We refer to Wikipedia articles on jurisprudence + for help. An offer shows the intent to enter into a mutually-beneficial contract + with specific terms. An offer can be made by a letter. While this letter indicates + the willingness to sell, the lack of specific terms, such as transaction price + and offer expiration date, makes it an incomplete offer. The answer is (D). + - question: 'Functions of the law include all but which of the following? + + (A) maximizing individual freedom (B) providing a basis for compromise (C) keeping + the peace (D) promoting the principles of the free enterprise system' + target: Let's think step by step. We refer to Wikipedia articles on jurisprudence + for help. Laws are fundamentally about helping resolve disputes between individuals, + and therefore essential for maximizing individual freedom, providing a basis + for compromise, and keeping the peace. The answer is (D). + - question: 'The ________ School of jurisprudence postulates that the law is based + on what is "correct." + + (A) Natural Law (B) Analytical (C) Historical (D) Sociological' + target: Let's think step by step. We refer to Wikipedia articles on jurisprudence + for help. Natural Law School of jurisprudence focuses on the laws of nature, + and states that the law should be based on ethics, morals, and what is "correct". + Analytical deals with the law as it already exists, Historical postulates that + the law was found and not made, and Sociological studies how the law and society + impact each other. The answer is (A). + - question: 'Which word best summarizes Weber''s explanation of the development of + formally rational law? + + (A) Authority. (B) Charisma. (C) Co-operation. (D) Capitalism.' + target: Let's think step by step. We refer to Wikipedia articles on jurisprudence + for help. Weber explained the development of formal rationality in laws as how + the modern society moved from tradition to rationality, where people decide + actions based less on how they were culturally done and more on expected utilities. + How rational individuals optimize efficiency of accomplishing tasks for higher + rewards is a core principle of Capitalism. The answer is (D). + - question: 'Which position does Rawls claim is the least likely to be adopted by the + POP (people in the original position)? + + (A) The POP would choose equality above liberty. (B) The POP would opt for the + ''maximin'' strategy. (C) The POP would opt for the ''difference principle''. + (D) The POP would reject the ''system of natural liberty.''' + target: 'Let''s think step by step. We refer to Wikipedia articles on jurisprudence + for help. The POP would opt for the ''maximin'' strategy, opt for the ''difference + principle'', and reject the ''system of natural liberty'', but the POP would + not choose equality above liberty, since the POP assume both equal and free + citizens. The answer is (A).' +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_jurisprudence diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_logical_fallacies.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dd72cab8cf8ea0553a633810e59c0ba4059c2176 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_logical_fallacies.yaml @@ -0,0 +1,61 @@ +dataset_name: logical_fallacies +description: The following are multiple choice questions (with answers) about logical + fallacies. +fewshot_config: + sampler: first_n + samples: + - question: 'When an arguer causes confusion during refutation because of real or feigned + lack of an ability to engage in refutation, that arguer may have committed the + fallacy of + + (A) poor sportsmanship (B) appeal to compassion (C) argument against the person + (D) ignorance of refutation' + target: Let's think step by step. We refer to Wikipedia articles on logical fallacies + for help. Ignorance of refutation, one of Aristotle's original list of logical + fallacies in his Organon, is when someone causes confusion in an argument through + real or feigned inability to engage in refutation, in order to win the argument. + The answer is (D). + - question: 'The complex question fallacy consists of + + (A) arguing something is inferior just because it doesn''t do something it was + never intended to do. (B) including more than one claim in the proposition and + treating proof for one claim as proof for all the claims. (C) drawing a conclusion + before examining the evidence, and only considering evidence that supports that + conclusion. (D) asking a question that includes either an unproven assumption + or more than one question, thus making a straightforward yes or no answer meaningless.' + target: Let's think step by step. We refer to Wikipedia articles on logical fallacies + for help. The complex question fallacy is when someone makes a single yes or + no answer to a question meaningless, by including either an unproven assumption + or many questions. The latter is also known as the many questions fallacy. The + answer is (D). + - question: 'Arguing that what is true of the parts must be true of the whole is the + fallacy of... + + (A) Division (B) Composition (C) Appeal to the person (D) Appeal to ignorance' + target: Let's think step by step. We refer to Wikipedia articles on logical fallacies + for help. Fallacy of composition occurs when someone argues what is true of + the parts must be true of the whole. The answer is (B). + - question: 'Which of the following is true of a valid categorical syllogism? + + (A) The minor premise must deny the antecedent (B) The major premise must affirm + the consequent (C) The middle term must be used in at least one premise in a + universal or unqualified sense (D) All of the above' + target: 'Let''s think step by step. We refer to Wikipedia articles on logical + fallacies for help. A valid categorical syllogism must satisfy several conditions: + (1) the syllogism must have exactly three terms (2) every term of the syllogism + must be used twice exactly, (3) a term may be used only once in any premise, + and (4) the middle term must be used in at least one premise in a universal + or unqualified sense, etc. Only (C) is true. The answer is (C).' + - question: 'If someone attacks the character of an opposing arguer, instead of responding + to that opponent''s arguments, the first person has probably committed which + of the following fallacies? + + (A) tu quoque (B) horse laugh (C) argument against the person (D) ignoratio + elenchi' + target: 'Let''s think step by step. We refer to Wikipedia articles on logical + fallacies for help. The argument against the person fallacy occurs when someone + irrelevantly attacks the character of an opposing arguer, instead of addressing + that opponent''s arguments. The answer is (C).' +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_logical_fallacies diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_machine_learning.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..33622ac4e7291eb380b6e58382e4fe84052a5bdc --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_machine_learning.yaml @@ -0,0 +1,74 @@ +dataset_name: machine_learning +description: The following are multiple choice questions (with answers) about machine + learning. +fewshot_config: + sampler: first_n + samples: + - question: 'Which image data augmentation is most common for natural images? + + (A) random crop and horizontal flip (B) random crop and vertical flip (C) posterization + (D) dithering' + target: Let's think step by step. Data augmentation is used to increase the diversity + of images in the training dataset. It is important that natural images are kept + natural after being augmented. Vertical flips of images are not natural, so + (B) is false. Posterization makes the image look like a poster and and dithering + increases color depth. None of these two preserve the natural property. The + only natural data augmentation technique is (A). The answer is (A). + - question: "Traditionally, when we have a real-valued question attribute during decision-tree\ + \ learning we consider a binary split according to whether the attribute is\ + \ above or below some threshold. Pat suggests that instead we should just have\ + \ a multiway split with one branch for each of the distinct values of the attribute.\ + \ From the list below choose the single biggest problem with Pat\u2019s suggestion:\n\ + (A) It is too computationally expensive. (B) It would probably result in a decision\ + \ tree that scores badly on the training set and a testset. (C) It would probably\ + \ result in a decision tree that scores well on the training set but badly on\ + \ a testset. (D) It would probably result in a decision tree that scores well\ + \ on a testset but badly on a training set." + target: "Let's think step by step. Because the question is real valued, it is unlikely\ + \ that the same values appear both at training and test time. This means that\ + \ while such a decision tree could yield good performance on the training data,\ + \ when evaluated on the test data it will perform badly because the decision\ + \ tree won\u2019t know what to do with numbers that did not appear in the training\ + \ data. The answer is (C)." + - question: "You are reviewing papers for the World\u2019s Fanciest Machine Learning\ + \ Conference, and you see submissions with the following claims. Which ones\ + \ would you consider accepting?\n(A) My method achieves a training error lower\ + \ than all previous methods! (B) My method achieves a test error lower than\ + \ all previous methods! (Footnote: When regularisation parameter \u03BB is chosen\ + \ so as to minimise test error.) (C) My method achieves a test error lower than\ + \ all previous methods! (Footnote: When regularisation parameter \u03BB is chosen\ + \ so as to minimise cross-validaton error.) (D) My method achieves a cross-validation\ + \ error lower than all previous methods! (Footnote: When regularisation parameter\ + \ \u03BB is chosen so as to minimise cross-validaton error.)" + target: "Let's think step by step. In machine learning, we train with some data\ + \ and fixed hyperparameters and the training error can be arbitrarily low, so\ + \ (A) can\u2019t be right. Then, one compares different hyperparameters by selecting\ + \ the model with the lowest cross-validation error, this means that (B) and\ + \ (D) are not the right procedure. The only relevant number after these is the\ + \ test error and thus (C) is the right answer. The answer is (C)." + - question: 'A 6-sided die is rolled 15 times and the results are: side 1 comes up + 0 times; side 2: 1 time; side 3: 2 times; side 4: 3 times; side 5: 4 times; + side 6: 5 times. Based on these results, what is the probability of side 3 coming + up when using Add-1 Smoothing? + + (A) 2.0/15 (B) 1.0/7 (C) 3.0/16 (D) 1.0/5' + target: 'Let''s think step by step. Add-1 smoothing adds the value of one to the + different counts and then normalizes the probabilities accordingly. The counts + after adding one will be: side 1 comes up 1 time; side 2: 2 times; side 3: 3 + times; side 4: 4 times; side 5: 5 times; side 6: 6 times. The number of sum + one die rolls will be 21, so the probability of drawing a three is 3/21 = 1/7. + The answer is (B).' + - question: 'To achieve an 0/1 loss estimate that is less than 1 percent of the true + 0/1 loss (with probability 95%), according to Hoeffding''s inequality the IID + test set must have how many examples? + + (A) around 10 examples (B) around 100 examples (C) between 100 and 500 examples + (D) more than 1000 examples' + target: "Let's think step by step. By the Hoeffding\u2019s inequality, we expect\ + \ that with 95% probability the in-sample and out-of-sample errors differ by\ + \ epsilon when we have N samples if 2 exp(-2 epsilon^2 N)<0.05, this implies\ + \ that N > -1/(2*epsilon**2) log ( 0.05/2 )= log (40)*5000. Since log(40)>1,\ + \ we have that one needs more than 1000 examples. The answer is (D).\n\n" +tag: mmlu_flan_cot_fewshot_stem +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_machine_learning diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_management.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..87d9ba8c9aa31733a5849695213d97deff9c2ded --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_management.yaml @@ -0,0 +1,46 @@ +dataset_name: management +description: The following are multiple choice questions (with answers) about management. +fewshot_config: + sampler: first_n + samples: + - question: 'How can organisational structures that are characterised by democratic + and inclusive styles of management be described? + + (A) Hierarchical (B) Bureaucratic (C) Flat (D) Functional' + target: Let's think step by step. We refer to Wikipedia articles on management + for help. Flat organizational structures are characterized by democratic and + inclusive styles of management, and have few (if any) levels of management between + the workers and managers. The answer is (C). + - question: 'Hygiene factors are associated with which writer? + + (A) Frederick Hertzberg (B) D.C. McClelland (C) Abraham Maslow (D) Douglas McGregor' + target: Let's think step by step. We refer to Wikipedia articles on management + for help. Hygiene factors include compensation, company policies, supervision, + interpersonal relations, and work environments. Hertzberg lists them as factors + that cannot motivate employees but can minimize job dissatisfaction. The answer + is (A). + - question: 'What characteristic is not a key feature of the ''open systems'' model + of management? + + (A) Morale (B) Innovation (C) Growth resource (D) Adaptation' + target: Let's think step by step. We refer to Wikipedia articles on management + for help. The key characteristics of an open system in management include innovation, + growth resource, and adaption, but do not include morale. The answer is (A). + - question: 'Which element of the cultural web forms regalia? + + (A) Symbols (B) Rituals and routines (C) Power structures (D) Control systems' + target: Let's think step by step. We refer to Wikipedia articles on management + for help. The cultural web is a tool for mapping an organization's culture, + where symbols form the regalia that visually expresses the values that the organization + holds as important. The answer is (A). + - question: 'What are the two main dimensions of the Ohio Studies into leadership? + + (A) Starting position and end position (B) Initial environment and changed environment + (C) Organisational structure and conditioning (D) Initiating structure and considerations' + target: 'Let''s think step by step. We refer to Wikipedia articles on management + for help. The Ohio State Leadership Studies conducted in the 1940s identified + initiating structure and consideration as the two main dimensions of leader + behavior. The answer is (D).' +tag: mmlu_flan_cot_fewshot_other +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_management diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_marketing.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..182eb52ec509c34c225a176774be653d747e120e --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_marketing.yaml @@ -0,0 +1,56 @@ +dataset_name: marketing +description: The following are multiple choice questions (with answers) about marketing. +fewshot_config: + sampler: first_n + samples: + - question: 'Although the content and quality can be as controlled as direct mail, + response rates of this medium are lower because of the lack of a personal address + mechanism. This media format is known as: + + (A) Care lines. (B) Direct mail. (C) Inserts. (D) Door to door.' + target: Let's think step by step. We refer to Wikipedia articles on marketing + for help. Door to door marketing delivers non-addressed items within all buildings + within a geographic area. While it can control the content and quality as well + as direct mail marketing, its response rate is lower because of the lack of + a personal address mechanism. The answer is (D). + - question: 'In an organization, the group of people tasked with buying decisions is + referred to as the _______________. + + (A) Outsourcing unit. (B) Procurement centre. (C) Chief executive unit. (D) + Decision-making unit.' + target: Let's think step by step. We refer to Wikipedia articles on marketing + for help. In an organization, the group of the people tasked with buying decision + is referred to as the decision-making unit. The answer is (D). + - question: 'The single group within society that is most vulnerable to reference group + influence is: + + (A) The older consumer who feels somewhat left out of things. (B) The married + women, many of whom feel a need for stability in their lives. (C) New immigrants + who really want to assimilate into their new culture. (D) Children, who base + most of their buying decisions on outside influences.' + target: Let's think step by step. We refer to Wikipedia articles on marketing + for help. Children, who mostly based their buying decisions on outside influences, + are the single group within society that is more vulnerable to reference group + influence. The answer is (D). + - question: 'Which of the following is an assumption in Maslow''s hierarchy of needs? + + (A) Needs are dependent on culture and also on social class. (B) Lower-level + needs must be at least partially satisfied before higher needs can affect behaviour. + (C) Needs are not prioritized or arranged in any particular order. (D) Satisfied + needs are motivators, and new needs emerge when current needs remain unmet.' + target: Let's think step by step. We refer to Wikipedia articles on marketing + for help. Maslow's hierarchy of needs, from the bottom upwards, are physiological + (food and clothing), safety, love and belonging needs, esteem, and self-actualization. + Lower-level needs must be at least partially satisfied before higher ones can + affect behavior. The answer is (B). + - question: '_____________ is a natural outcome when combining demographic and geographic + variables. + + (A) Geodemographics (B) Product differentiation. (C) ANSOFF matrix. (D) Brand + management.' + target: 'Let''s think step by step. We refer to Wikipedia articles on marketing + for help. Geodemographics is a natural outcome when combining demographic and + geographic variables. The answer is (A).' +tag: mmlu_flan_cot_fewshot_other +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_marketing diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_medical_genetics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..757010bfed0f08f882995eed787d4f68e0f8121c --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_medical_genetics.yaml @@ -0,0 +1,51 @@ +dataset_name: medical_genetics +description: The following are multiple choice questions (with answers) about medical + genetics. +fewshot_config: + sampler: first_n + samples: + - question: 'The stage of meiosis in which chromosomes pair and cross over is: + + (A) prophase I (B) metaphase I (C) prophase II (D) metaphase II' + target: Let's think step by step. We refer to Wikipedia articles on medical genetics + for help. Prophase I is the stage of meiosis where homologous chromosomes pair + with each other and exchange genetic material. The answer is (A). + - question: 'DNA ligase is + + (A) an enzyme that joins fragments in normal DNA replication (B) an enzyme of + bacterial origin which cuts DNA at defined base sequences (C) an enzyme that + facilitates transcription of specific genes (D) an enzyme which limits the level + to which a particular nutrient reaches' + target: Let's think step by step. We refer to Wikipedia articles on medical genetics + for help. DNA ligase is a type of enzyme (EC 6.5.1.1) responsible for joining + DNA strands together by catalyzing a phosphodiester bond. The answer is (A). + - question: 'Which of the following conditions does not show multifactorial inheritance? + + (A) Pyloric stenosis (B) Schizophrenia (C) Spina bifida (neural tube defects) + (D) Marfan syndrome' + target: Let's think step by step. We refer to Wikipedia articles on medical genetics + for help. Multifactorial inheritance is when more than a single factor is responsible + for causing a given trait or health problem. Genes cannot be the only factor. + Marfan syndrome, on the other hand, requires only one abnormal copy of the of + the Marfan gene, from one parent, to inherit the trait. The answer is (D). + - question: 'A gene showing codominance + + (A) has both alleles independently expressed in the heterozygote (B) has one + allele dominant to the other (C) has alleles tightly linked on the same chromosome + (D) has alleles expressed at the same time in development' + target: Let's think step by step. We refer to Wikipedia articles on medical genetics + for help. Codominance, as it relates to genetics, refers to a type of genetic + inheritance where the phenotype of both the parents is easily observed in the + offspring. A heterozygote is an individual having two different alleles of a + gene. The answer is (A). + - question: 'Large triplet repeat expansions can be detected by: + + (A) polymerase chain reaction. (B) single strand conformational polymorphism + analysis. (C) Southern blotting. (D) Western blotting.' + target: 'Let''s think step by step. We refer to Wikipedia articles on medical + genetics for help. A Southern blot is a method in molecular biology for detecting + specific DNA sequences in a sample. Large triplet repeat expansions are usually + detected with this method. The answer is (C).' +tag: mmlu_flan_cot_fewshot_other +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_medical_genetics diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_miscellaneous.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2fe892eb06f522df6e99606013d26e0df1517cf3 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_miscellaneous.yaml @@ -0,0 +1,43 @@ +dataset_name: miscellaneous +description: The following are multiple choice questions (with answers) about miscellaneous. +fewshot_config: + sampler: first_n + samples: + - question: 'Which of these songs was a Top 10 hit for the rock band The Police? + + (A) ''Radio Ga-Ga'' (B) ''Ob-la-di Ob-la-da'' (C) ''De Do Do Do De Da Da Da'' + (D) ''In-a-Gadda-Da-Vida''' + target: Let's think step by step. We refer to Wikipedia for help. Radio Ga-Ga + is by Queen. Ob-la-di Ob-la-da is by The Beatles. And In-a-Gadda-Da-Vida is + by Iron Butterfly. Leaving 'De Do Do Do De Da Da Da' as the only song by The + Police, and also a Top 10 hit. The answer is (C). + - question: 'What place is named in the title of the 1979 live album by rock legends + Cheap Trick? + + (A) Budapest (B) Budokan (C) Bhutan (D) Britain' + target: Let's think step by step. We refer to Wikipedia for help. Nippon Budokan + is an indoor arena in Tokyo, Japan renowned for hosting rock music concerts + including Cheap Trick in 1978. 'Cheap Trick at Budokan' became the name of their + album. The answer is (B). + - question: 'What is produced during photosynthesis? + + (A) hydrogen (B) nylon (C) oxygen (D) light' + target: Let's think step by step. We refer to Wikipedia for help. Photosynthesis + is the process in which green plants use the green pigment chlorophyll to synthesize + foods with water and carbon dioxide. Oxygen is the byproduct of this process. + The answer is (C). + - question: 'Who is the shortest man to ever win an NBA slam dunk competition? + + (A) Anthony ''Spud'' Webb (B) Michael ''Air'' Jordan (C) Tyrone ''Muggsy'' Bogues + (D) Julius ''Dr J'' Erving' + target: Let's think step by step. We refer to Wikipedia for help. In 1986, Spud + Webb, standing only 5'7" became the shortest NBA player in history to win an + official slam dunk contest. The answer is (A). + - question: 'How many axles does a standard automobile have? + + (A) one (B) two (C) four (D) eight' + target: 'Let''s think step by step. We refer to Wikipedia for help. Most cars + have two axles to rotate the wheels.. The answer is (B).' +tag: mmlu_flan_cot_fewshot_other +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_miscellaneous diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_moral_disputes.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..45a92e075582d6c0e2eb11c0310f87fc5debb4bb --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_moral_disputes.yaml @@ -0,0 +1,64 @@ +dataset_name: moral_disputes +description: The following are multiple choice questions (with answers) about moral + disputes. +fewshot_config: + sampler: first_n + samples: + - question: 'Baron admits that the versions of the ticking bomb hypothetical she discusses + are "stunningly stupid," but she claims this is actually evidence of + + (A) the stupidity of most traditional philosophical examples. (B) a general + lack of intelligence among people with advanced degrees. (C) the wrongness of + torture. (D) the readiness on the part of many intelligent people to see torture + as the best solution to deal with terrorism.' + target: Let's think step by step. We refer to Wikipedia articles on moral disputes + for help. The ticking bomb hypothetical poses a problem where many people will + die to an exploding bomb, if the hypothetical terrorist does not disclose how + to defuse it. Baron sees this hypothetical as silly, but its prevalence does + suggest intelligent people, particularly utilitarians, see torture as justifiable + to save the lives in this scenario. The answer is (D). + - question: 'A fertilized ovum is also known as + + (A) a zygote. (B) an embryo. (C) a viability. (D) a blastocyst.' + target: Let's think step by step. We refer to Wikipedia articles on moral disputes + for help. Once a single sperm penetrates the layers of an egg to form a new + cell, that cell is called a zygote. The answer is (A). + - question: 'Pence compares six different cases of reproduction, from natural twinning + to SCNT. What conclusion does he draw from this comparison? + + (A) SCNT is not a different kind of reproduction because there are no morally + relevant differences between it and other permissible means of reproduction. + (B) Because there is a low risk of harm for natural twinning, there will be + a low risk of harm for SCNT. (C) both A and B (D) neither A nor B' + target: Let's think step by step. We refer to Wikipedia articles on moral disputes + for help. SCNT, known as 'somatic cell nuclear transfer', is a technique in + which the nucleus of a somatic (body) cell is transferred to the cytoplasm of + an enucleated egg. This is used in farm animal cloning. Pence may compare this + to other processes of reproduction, and see little moral separation. The answer + is (A). + - question: 'Which of the following is an intrinsic good, according to Singer? + + (A) being in the upper class (B) the ability to learn and solve problems (C) + being more beautiful than most of the population (D) being shorter than average' + target: Let's think step by step. We refer to Wikipedia articles on moral disputes + for help. Bioethicist Peter Singer sees intrinsic value as innate values conferred + by oneself, for oneself. Innanimate objects can be beautiful, short, or have + some valuable criteria, but capabilities are intrinsically good. The answer + is (B). + - question: 'According to Metz, what is wrong with consequentialist arguments against + capital punishment based on African values? + + (A) It is unclear as of yet whether or not capital punishment deters harm to + the community. (B) It is unclear as of yet whether or not capital punishment + deters harm to any individuals. (C) Consequentialism is not supported by African + values. (D) Even though consequentialism is supported by African values, no + consequentialist arguments framed in terms of African values have been offered.' + target: 'Let''s think step by step. We refer to Wikipedia articles on moral disputes + for help. Thaddeus Metz is a humanities research professor focusing on ethical + philosophy in South Africa. Metz has written the death penalty is unjustified + as it treats individuals as incapable of communal relations. It is unclear that + capital punishment is to the benefit of, or a deterrent of harm to the community. + The answer is (A).' +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_moral_disputes diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_moral_scenarios.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b4a0231930d62fe73a1cb84b58420baf0182365c --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_moral_scenarios.yaml @@ -0,0 +1,62 @@ +dataset_name: moral_scenarios +description: The following are multiple choice questions (with answers) about moral + scenarios. +fewshot_config: + sampler: first_n + samples: + - question: 'For which of these two scenarios does the main character (who uses I/me/my) + do something clearly morally wrong, according to ordinary moral standards in + the US as of 2020? Scenario 1 | I took his wallet after he harassed me at the + bar. Scenario 2 | I made love to my newborn baby boy. + + (A) Wrong, Wrong (B) Wrong, Not wrong (C) Not wrong, Wrong (D) Not wrong, Not + wrong' + target: Let's think step by step. We refer to Wikipedia articles on moral scenarios + for help. Stealing from someone is not considered an ethical response to harrassment. + And having sex with a minor is egregiously unethical. The answer is (A). + - question: 'For which of these two scenarios does the main character (who uses I/me/my) + do something clearly morally wrong, according to ordinary moral standards in + the US as of 2020? Scenario 1 | I asked for a cigarette from the guy outside + the store. Scenario 2 | When the security guard left his post, I pocketed a + rare coin. + + (A) Wrong, Wrong (B) Wrong, Not wrong (C) Not wrong, Wrong (D) Not wrong, Not + wrong' + target: Let's think step by step. We refer to Wikipedia articles on moral scenarios + for help. Asking for something is not morally wrong. Stealing is morally wrong. + The answer is (C). + - question: 'For which of these two scenarios does the main character (who uses I/me/my) + do something clearly morally wrong, according to ordinary moral standards in + the US as of 2020? Scenario 1 | During the house''s demolition, I pushed him + into a wall. Scenario 2 | When I choose a babysitter for my twin nieces, I don''t + care about her racial background. + + (A) Wrong, Wrong (B) Wrong, Not wrong (C) Not wrong, Wrong (D) Not wrong, Not + wrong' + target: Let's think step by step. We refer to Wikipedia articles on moral scenarios + for help. Physical violence is morally wrong. Being open to hiring people from + different racial backgrounds is an expected moral stance. The answer is (B). + - question: 'For which of these two scenarios does the main character (who uses I/me/my) + do something clearly morally wrong, according to ordinary moral standards in + the US as of 2020? Scenario 1 | I bought an old pick-up truck for work. Scenario + 2 | I looked up at the gorgeous waterfall and threw a rock at it. + + (A) Wrong, Wrong (B) Wrong, Not wrong (C) Not wrong, Wrong (D) Not wrong, Not + wrong' + target: Let's think step by step. We refer to Wikipedia articles on moral scenarios + for help. Neither of these are wrong as no one is wronged by buing a pick-up + truck or throwing a rock into a stream. The answer is (D). + - question: 'For which of these two scenarios does the main character (who uses I/me/my) + do something clearly morally wrong, according to ordinary moral standards in + the US as of 2020? Scenario 1 | I really loved my newborn baby boy. Scenario + 2 | The new girl is embarrassed to live in a trailer park, so I invite the class + to her house. + + (A) Wrong, Wrong (B) Wrong, Not wrong (C) Not wrong, Wrong (D) Not wrong, Not + wrong' + target: 'Let''s think step by step. We refer to Wikipedia articles on moral scenarios + for help. Loving someone is not wrong. However, exposing something that someone + is embarrassed about could be considered quite mean. The answer is (C).' +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_moral_scenarios diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_nutrition.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..66498dc564350f893e7bd45078528b5750bce0ca --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_nutrition.yaml @@ -0,0 +1,63 @@ +dataset_name: nutrition +description: The following are multiple choice questions (with answers) about nutrition. +fewshot_config: + sampler: first_n + samples: + - question: 'What is the first-line drug for patients with type 2 diabetes and obesity, + as of 2020? + + (A) Acarbose (B) Metformin (C) Sulphonylureas (D) Insulin' + target: Let's think step by step. We refer to Wikipedia articles on nutrition + for help. Metformin (Fortamet, Glumetza, or others) is usually the first medication + prescribed for type 2 diabetes, as well as obesity. It works by lowering glucose + production in the liver and improving the body's sensitivity to insulin. The + answer is (B). + - question: 'Which of the following statements is correct (according to knowledge in + 2020)? + + (A) Consumers with phenylketonuria must avoid the consumption of the sweetener + aspartame (B) Consumers with phenylketonuria must avoid the consumption of the + sweetener saccharin (C) Consumers with phenylketonuria must avoid the consumption + of the sweetener sucralose (D) Consumers with phenylketonuria must avoid the + consumption of the sweetener acesulfame K' + target: Let's think step by step. We refer to Wikipedia articles on nutrition + for help. People with phenylketonuria (PKU) cannot break down the amino acid + phenylalanine. As it builds up in the blood and brain it can lead to brain damage. + People with PKU should avoid foods that are converted to phenylalanine in the + body, such as aspartame. The answer is (A). + - question: 'Which of the following statements about iodine is correct, as of 2020? + + (A) 50% of adults consume iodine at levels below the RNI (B) Dairy products + are a poor source of iodine (C) The iodine content of organic milk is generally + lower that the level in non-organic milk (D) UK dietary reference values recommend + an increase in iodine intake in pregnancy' + target: Let's think step by step. We refer to Wikipedia articles on nutrition + for help. Organic milk usually has less iodine content than non-organic milk. + The answer is (C). + - question: 'Which of the following is the most plausible explanation for the protective + effect of dietary fibre against cancer of the colon, as of 2020? + + (A) Propionic acid, formed during colonic fibre fermentation inhibits liver + fatty acid synthesis (B) Butyric acid, formed during colonic fibre fermentation + stimulates "silencing" of the SLC5A8 tumour suppressor gene (C) None of these + options are correct (D) Butyric acid, formed during colonic fibre fermentation + stimulates anti-oxidant defences in the colon' + target: Let's think step by step. We refer to Wikipedia articles on nutrition + for help. Dietary fibre is inversely proportional to the risk of colorectal + cancer. This is presumed because butyric acid (BA) stimulates antioxidants which + help protect the colon from cancerous tumors. The answer is (D). + - question: 'In a cohort study, the risk ratio of developing diabetes was 0.86 when + comparing consumers of tea (the exposed) to those who did not drink tea (the + unexposed). Which one statement is correct (according to knowledge in 2020)? + + (A) The tea drinkers have lower risk of developing diabetes. (B) The tea drinkers + have higher risk of developing diabetes. (C) Based on the information given + we cannot tell if the observed difference in disease risk is the result of chance. + (D) The risk ratio is close to the value one, so there is no difference in disease + risk between the two groups.' + target: 'Let''s think step by step. We refer to Wikipedia articles on nutrition + for help. The risk ratio is not sufficiently reduced that it could not be explained + by random chance given the studies sample size. The answer is (C).' +tag: mmlu_flan_cot_fewshot_other +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_nutrition diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_philosophy.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b5116be01d6a463f23f9b7422de1b0b48bbdb55e --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_philosophy.yaml @@ -0,0 +1,44 @@ +dataset_name: philosophy +description: The following are multiple choice questions (with answers) about philosophy. +fewshot_config: + sampler: first_n + samples: + - question: 'The study of reality in the broadest sense, an inquiry into the elemental + nature of the universe and the things in it, is known as _____. + + (A) metaphysics (B) epistemology (C) quantum physics (D) axiology' + target: Let's think step by step. We refer to Wikipedia articles on philosophy + for help. Among the options, only metaphysics studies the nature of reality + and existence. The answer is (A). + - question: "According to Moore\u2019s \u201Cideal utilitarianism,\u201D the right\ + \ action is the one that brings about the greatest amount of:\n(A) pleasure.\ + \ (B) happiness. (C) good. (D) virtue." + target: Let's think step by step. We refer to Wikipedia articles on philosophy + for help. Moore's "ideal utilitarianism" states that one's actions should maximize + intrinsic goods. The answer is (C). + - question: 'Before Tolstoy''s Christian conversion, what was his perspective on the + meaning of life? + + (A) optimist (B) satisfied (C) nominally religious (D) pessimist' + target: Let's think step by step. We refer to Wikipedia articles on philosophy + for help. Before his conversion, Tolstoy feels that life was uncertain, which + is a pessimist's point of view. The answer is (D). + - question: 'According to d''Holbach, people always act according to _____. + + (A) free choices (B) dictates of the soul (C) necessary natural laws (D) undetermined + will' + target: Let's think step by step. We refer to Wikipedia articles on philosophy + for help. d'Holbach believes that people act according to necessary laws, and + it proves nothing about people's free will. The answer is (C). + - question: 'Psychological egoism is: + + (A) an ethical theory about how we ought to behave. (B) a generalization concerning + the way people tend to behave. (C) a claim about human nature and the ways people + are capable of behaving. (D) none of the above.' + target: 'Let''s think step by step. We refer to Wikipedia articles on philosophy + for help. Psychological egoism suggests that one behaves based on what makes + one feels good, hence it is a claim about human nature and how humans are capable + of behaving. The answer is (C).' +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_philosophy diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_prehistory.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f9e5d81664445497a13a9adf7ca818ed6d2c7ef --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_prehistory.yaml @@ -0,0 +1,59 @@ +dataset_name: prehistory +description: The following are multiple choice questions (with answers) about prehistory. +fewshot_config: + sampler: first_n + samples: + - question: 'What is the approximate mean cranial capacity of Homo erectus? + + (A) under 650 cc (B) about 800 cc (C) just under 1000 cc (D) 1200 cc' + target: Let's think step by step. We refer to Wikipedia articles on prehistory + for help. The average cranium capacity of Homo erectus is less than 1000 cubic + cm. The answer is (C). + - question: 'According to Timothy Pauketat, the evidence for social stratification + and political power at Cahokia suggests: + + (A) a center of Mississippian civilization with conditions similar to the rise + of early states. (B) the limitations of authority in a Native American society + of egalitarian foragers. (C) a simple chiefdom or perhaps a complex chiefdom + had evolved by A.D. 1500. (D) a center of Mississippian civilization with conditions + similar to societies on the Northwest Coast of North America.' + target: Let's think step by step. We refer to Wikipedia articles on prehistory + for help. Timothy Pauketat is known for his research on Cahokia, the center + of the Mississippian culture, where he found similar conditions to the rise + of early states. The answer is (A). + - question: 'Recent research on hominid species dating from the Middle Pliocene indicates + there was (as of 2020): + + (A) a great amount of species diversity, or a single species that exhibited + a lot of diversity. (B) very little species diversity during this period and + very few hominids. (C) decreased species diversity due to a prolonged ice age + followed by a severe drought. (D) decreased species diversity but increased + numbers of hammerstones and flakes, indicating stone tool manufacture.' + target: Let's think step by step. We refer to Wikipedia articles on prehistory + for help. Recent research has recognized multiple hominid species from the Middle + Pliocene, meaning that there is a great amount of species diversity or diversity + in a single species. The answer is (A). + - question: 'Researchers now believe that the decline of the Maya was caused chiefly + by: + + (A) a cataclysm of some kind, such as an earthquake, volcano, or tsunami. (B) + ecological degradation resulting from slash-and-burn farming techniques. (C) + endless wars between neighboring Mayan city-states. (D) practices of interbreeding + that led to a steep rise in congenital disorders.' + target: Let's think step by step. We refer to Wikipedia articles on prehistory + for help. Researchers believe that the Maya collapse was mainly caused by over-exploitation + of natural resources like the slash-and-burn farming techniques. The answer + is (B). + - question: 'The great Mayan king Pacal built temples in the city of Palenque in order + to: + + (A) satisfy the powerful Mayan astronomer priests. (B) display his generosity + to the common people, since they were allowed to live in the temples. (C) frighten + away enemies, in particular the Spaniards. (D) legitimize his kingship, since + his father was not royal.' + target: 'Let''s think step by step. We refer to Wikipedia articles on prehistory + for help. Pacal built the temples as the funerary monument to legitimize his + kingship. The answer is (D).' +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_prehistory diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_accounting.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d18fc22626b952d18491b849cabc720706c17c9 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_accounting.yaml @@ -0,0 +1,63 @@ +dataset_name: professional_accounting +description: The following are multiple choice questions (with answers) about professional + accounting. +fewshot_config: + sampler: first_n + samples: + - question: "An auditor traces the serial numbers on equipment to a nonissuer\u2019\ + s subledger. Which of the following management assertions is supported by this\ + \ test?\n(A) Valuation and allocation (B) Completeness (C) Rights and obligations\ + \ (D) Presentation and disclosure" + target: Let's think step by step. We refer to Wikipedia articles on accounting + for help. The completeness assertion is tested by tracing supporting documents + to the record entries. The answer is (B). + - question: 'One hundred years ago, your great-great-grandmother invested $100 at 5% + yearly interest. What is the investment worth today? + + (A) $13,000 (B) $600 (C) $15,000 (D) $28,000' + target: Let's think step by step. We refer to Wikipedia articles on accounting + for help. A $100 investment at 5% yearly interest is worth 100*(1.05)^100=13150 + after 100 years, which is around $13,000. The answer is (A). + - question: 'On January 1, year 1, Alpha Co. signed an annual maintenance agreement + with a software provider for $15,000 and the maintenance period begins on March + 1, year 1. Alpha also incurred $5,000 of costs on January 1, year 1, related + to software modification requests that will increase the functionality of the + software. Alpha depreciates and amortizes its computer and software assets over + five years using the straight-line method. What amount is the total expense + that Alpha should recognize related to the maintenance agreement and the software + modifications for the year ended December 31, year 1? + + (A) $5,000 (B) $13,500 (C) $16,000 (D) $20,000' + target: Let's think step by step. We refer to Wikipedia articles on accounting + for help. The maintenance period begins on March 1, so only 10 months of expenses + should be recognized, which is $15,000/12*10=$12,500. The software modification + cost is amortized over 5 years, so each year is $5,000/5=$1,000. So the total + expense is $12,500+$1,000=$13,500. The answer is (B). + - question: 'Krete is an unmarried taxpayer with income exclusively from wages. By + December 31, year 1, Krete''s employer has withheld $16,000 in federal income + taxes and Krete has made no estimated tax payments. On April 15, year 2, Krete + timely filed for an extension request to file her individual tax return, and + paid $300 of additional taxes. Krete''s year 1 tax liability was $16,500 when + she timely filed her return on April 30, year 2, and paid the remaining tax + liability balance. What amount would be subject to the penalty for underpayment + of estimated taxes? + + (A) $0 (B) $500 (C) $1,650 (D) $16,500' + target: Let's think step by step. We refer to Wikipedia articles on accounting + for help. The tax due after withholding is $16,500-$16,000=$500, which is less + than $1000, hence there is no underpayment penalty of estimated taxes. The answer + is (A). + - question: 'Box a nongovernmental not-for-profit organization had the following transactions + during the year: Proceeds from sale of investments $80000 Purchase of property + plant and equipment $10000 Proceeds from long-term debt $100000 Loss on sale + of investment $5000 What amount should be reported as net cash provided by financing + activities in Box''s statement of cash flows? + + (A) $70,000 (B) $75,000 (C) $80,000 (D) 100000' + target: 'Let''s think step by step. We refer to Wikipedia articles on accounting + for help. Among the four transactions, only Proceeds from long-term debt belongs + to the financing activities section of cashflow, hence the amount reported should + be $100000. The answer is (D).' +tag: mmlu_flan_cot_fewshot_other +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_professional_accounting diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_law.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..307f8940bc445305fdbf00e89910cd5237a41312 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_law.yaml @@ -0,0 +1,122 @@ +dataset_name: professional_law +description: The following are multiple choice questions (with answers) about professional + law. +fewshot_config: + sampler: first_n + samples: + - question: 'A son owed a creditor $5,000. The son''s father contacted the creditor + and told him that he wanted to pay the son''s debt. The father signed a document + that stated the father would pay the son''s debt at a rate of $500 a month for + 10 months. The creditor made no written or oral commitment to forbear to sue + the son to collect the $5,000 debt, and the father made no oral or written request + for any such forbearance. For the next five months, the father made and the + creditor accepted the $500 monthly payments as agreed. During that period, the + creditor, in fact, did forbear to take any legal action against the son. However, + the father then informed the creditor that he would make no further payments + on the debt. Which of the following is the most persuasive argument that the + father is liable to the creditor under the terms of their agreement? + + (A) The father''s promise and the creditor''s reliance thereon, if proved, gave + rise to a valid claim by the creditor against the father based on the doctrine + of promissory estoppel. (B) Because it was foreseeable that the father''s promise + would induce the creditor to forbear taking any action against the son, such + forbearance was, as a matter of law, a bargained-for consideration for the father''s + promise. (C) The father''s five payments to the creditor totaling $2,500 manifested + a serious intent on the father''s part to be contractually bound, and such manifestation + is generally recognized as an effective substitute for consideration. (D) By + assuming the antecedent debt obligation that the son owed to the creditor, the + father became a surety whose promise to the creditor was enforceable, since + it was in writing and supported by adequate consideration. ' + target: Let's think step by step. We refer to Wikipedia articles on law for help. + The doctrine of promissory estoppel stops a person from going back on a promise + in contract law, hence option (A) should be the most persuasive argument. The + answer is (A). + - question: 'A state has recently enacted a statute prohibiting the disposal of any + nuclear wastes within the state. This law does not contravene or conflict with + any federal statutes. A man operates a company in the state that is engaged + in the disposal of nuclear wastes. Subsequent to the passage of the state statute, + the man, not yet aware of the new law, entered into contracts with many out-of-state + firms to dispose of their nuclear wastes in the state. On account of this new + law, however, the man will be unable to perform these contracts. Assume that + the man has standing to challenge this state law. Which of the following presents + his strongest constitutional grounds to challenge the state law prohibiting + the disposal of nuclear wastes within the state? + + (A) The commerce clause. (B) The equal protection clause of the Fourteenth Amendment. + (C) The privileges and immunities clause of Article IV, Section 2. (D) The contract + clause.' + target: Let's think step by step. We refer to Wikipedia articles on law for help. + The commerce clause states that Congress shall have the power to regulate commerce + with foreign Nations, and among the several States, and with the Indian Tribes. + The statute affects inter-state commerce which puts it into question. Hence + the man's strongest argument should be the commerce clause. The answer is (A). + - question: 'On October 1, 1980, a developer, owner of several hundred acres in a rural + county, drafted a general development plan for the area. The duly recorded plan + imposed elaborate limitations and restrictions upon the land in the plan, which + was to be developed as a residential district. The restrictions were to extend + to all persons acquiring any of the lots and to their heirs, assigns, and lessees. + It was further provided that all subsequent owners would be charged with due + notice of the restrictions. Among those restrictions in the general plan were + the following:(22) A franchise right is created in a strip of land 10 feet in + width along the rear of each lot for the use of public utility companies with + right of ingress and egress. (23) No house or structure of any kind shall be + built on the aforementioned strip of land running through the said blocks. In + 2000, a retiree purchased one of the lots, built a house, and erected a fence + in the rear of his property within the restricted area. In 2004, a teacher purchased + a lot adjacent to the retiree''s property and built a new house. Two years later, + a librarian purchased the lot that adjoined the teacher''s property. The three + deeds to those properties each contained references to the deed book where the + general plan was recorded. In 2008, the librarian began the construction of + a seven-foot post-and-rail fence along the line dividing his lot with the teacher''s, + and along the center of the area subject to the franchise right. Although the + teacher objected to its construction, the fence was completed. If the teacher + seeks a mandatory injunction to compel removal of the librarian''s fence, the + court will most likely + + (A) grant relief, because the fence was in violation of the easement restriction. + (B) grant relief, because the encroachment of the fence violated the restriction + in the original plan. (C) deny relief, because the teacher failed to enforce + the restriction against the retiree. (D) deny relief, because the fence would + not be construed as "a structure" within the terms of the restriction. ' + target: Let's think step by step. We refer to Wikipedia articles on law for help. + The restrictions in the original plan say no house or structure of any kind + shall be built on the aforementioned strip of land running through the said + blocks. Hence the court will most likely grant relief because the fence violated + the restriction in the original plan. The answer is (B). + - question: 'Judge took judicial notice of some facts at the beginning of the trial. + Which of the following is not an appropriate kind of fact for judicial notice? + + (A) Indisputable facts. (B) Facts that have been asserted by individual political + organizations. (C) Facts recognized to be true by common knowledge. (D) Facts + capable of scientific verification.' + target: Let's think step by step. We refer to Wikipedia articles on law for help. + Among the options, facts that have been asserted by individual political organizations + is not an appropriate kind of fact for judicial notice. The answer is (B). + - question: 'A state legislature has recently enacted a statute making it a misdemeanor + to curse or revile or use obscene or opprobrious language toward or in reference + to a police officer perfonning his duties. A student at a state university organized + a demonstration on campus to protest the war. The rally was attended by a group + of 50 students who shouted anti-war messages at cars passing by. To show his + contempt for the United States, the student sewed the American flag to the rear + of his jeans. When a police officer saw the flag sown on the student''s jeans, + he approached and told him to remove the flag or he would be placed under arrest. + The student became angered and shouted at the police officer, "Listen, you bastard, + I''ll wear this rag anywhere I please. " The student was subsequently placed + under arrest and charged with violating the state statute. The student subsequently + brings suit in state court challenging the constitutionality of the statute. + The strongest constitutional argument for the student is that + + (A) the statute is void for vagueness under the Fourteenth Amendment''s due + process clause. (B) the statute is invalid because it violates the petitioner''s + freedom of speech under the First Amendment. (C) the statute is an abridgment + of freedom of speech under the First Amendment because less restrictive means + are available for achieving the same purpose. (D) the statute is overbroad and + consequently invalid under the First and Fourteenth Amendments.' + target: 'Let''s think step by step. We refer to Wikipedia articles on law for + help. The Fourteenth Amendment further supports the First Amendment by establishing + a due process clause. Hence the strongest argument should be the statute is + overbroad and consequently invalid under the First and Fourteenth Amendments. + The answer is (D).' +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_professional_law diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_medicine.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a5b0d995ae7d0897eaea3a6cfd0719562cdc3c9 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_medicine.yaml @@ -0,0 +1,82 @@ +dataset_name: professional_medicine +description: The following are multiple choice questions (with answers) about professional + medicine. +fewshot_config: + sampler: first_n + samples: + - question: "A 22-year-old male marathon runner presents to the office with the complaint\ + \ of right-sided rib pain when he runs long distances. Physical examination\ + \ reveals normal heart and lung findings and an exhalation dysfunction at ribs\_\ + 4-5 on the right. Which of the following muscles or muscle groups will be most\ + \ useful in correcting this dysfunction utilizing a direct method?\n(A) anterior\ + \ scalene (B) latissimus dorsi (C) pectoralis minor (D) quadratus lumborum" + target: Let's think step by step. We refer to Wikipedia articles on medicine for + help. Among the options, only pectoralis minor muscle origins from the outer + surfaces of the 3rd to 5th ribs. The answer is (C). + - question: "A 36-year-old male presents to the office with a\_3-week\_history of low\ + \ back pain. He denies any recent trauma but says that he climbs in and out\ + \ of his truck numerous times a day for his job. Examination of the patient\ + \ in the prone position reveals a deep sacral sulcus on the left, a posterior\ + \ inferior lateral angle on the right, and a lumbosacral junction that springs\ + \ freely on compression. The most likely diagnosis is\n(A) left-on-left sacral\ + \ torsion (B) left-on-right sacral torsion (C) right unilateral sacral flexion\ + \ (D) right-on-right sacral torsion" + target: Let's think step by step. We refer to Wikipedia articles on medicine for + help. The deep sulcus on the left, a posterior ILA on the right, with a negative + spring test suggests a right-on-right sacral torsion. All other options have + a deep sulcus on the right. The answer is (D). + - question: "A 44-year-old man comes to the office because of a 3-day history of sore\ + \ throat, nonproductive cough, runny nose, and frontal headache. He says the\ + \ headache is worse in the morning and ibuprofen does provide some relief. He\ + \ has not had shortness of breath. Medical history is unremarkable. He takes\ + \ no medications other than the ibuprofen for pain. Vital signs are temperature\ + \ 37.4\xB0C (99.4\xB0F), pulse 88/min, respirations 18/min, and blood pressure\ + \ 120/84 mm Hg. Examination of the nares shows erythematous mucous membranes.\ + \ Examination of the throat shows erythema and follicular lymphoid hyperplasia\ + \ on the posterior oropharynx. There is no palpable cervical adenopathy. Lungs\ + \ are clear to auscultation. Which of the following is the most likely cause\ + \ of this patient's symptoms?\n(A) Allergic rhinitis (B) Epstein-Barr virus\ + \ (C) Mycoplasma pneumonia (D) Rhinovirus" + target: Let's think step by step. We refer to Wikipedia articles on medicine for + help. The symptoms, especially the headache, suggest that the most likely cause + is Rhinovirus. Epstein-Barr virus will cause swollen lymph nodes but there is + no palpable cervical adenopathy. Lungs are clear to auscultation suggests it's + not Mycoplasma pneumonia. The answer is (D). + - question: 'A previously healthy 32-year-old woman comes to the physician 8 months + after her husband was killed in a car crash. Since that time, she has had a + decreased appetite and difficulty falling asleep. She states that she is often + sad and cries frequently. She has been rechecking the door lock five times before + leaving her house and has to count exactly five pieces of toilet paper before + she uses it. She says that she has always been a perfectionist but these urges + and rituals are new. Pharmacotherapy should be targeted to which of the following + neurotransmitters? + + (A) Dopamine (B) Glutamate (C) Norepinephrine (D) Serotonin' + target: Let's think step by step. We refer to Wikipedia articles on medicine for + help. The patient feels sad and among the options, only Dopamine and Serotonin + can help increase positive emotions. Serotonin also affects digestion and metabolism, + which can help the patient's decreased appetite and sleep difficulty. The answer + is (D). + - question: "A 42-year-old man comes to the office for preoperative evaluation prior\ + \ to undergoing adrenalectomy scheduled in 2 weeks. One month ago, he received\ + \ care in the emergency department for pain over his right flank following a\ + \ motor vehicle collision. At that time, blood pressure was 160/100 mm Hg and\ + \ CT scan of the abdomen showed an incidental 10-cm left adrenal mass. Results\ + \ of laboratory studies, including complete blood count, serum electrolyte concentrations,\ + \ and liver function tests, were within the reference ranges. The patient otherwise\ + \ had been healthy and had never been told that he had elevated blood pressure.\ + \ He takes no medications. A follow-up visit in the office 2 weeks ago disclosed\ + \ elevated urinary normetanephrine and metanephrine and plasma aldosterone concentrations.\ + \ The patient was referred to a surgeon, who recommended the adrenalectomy.\ + \ Today, vital signs are temperature 36.6\xB0C (97.9\xB0F), pulse 100/min, respirations\ + \ 14/min, and blood pressure 170/95 mm Hg. Physical examination discloses no\ + \ significant findings. Initial preoperative preparation should include treatment\ + \ with which of the following?\n(A) Labetalol (B) A loading dose of potassium\ + \ chloride (C) Nifedipine (D) Phenoxybenzamine" + target: 'Let''s think step by step. We refer to Wikipedia articles on medicine + for help. The symptoms and the adrenal mass suggested pheochromocytoma, and + the blood pressure indicates hypertension. Phenoxybenzamine is used to treat + hypertension caused by pheochromocytoma. The answer is (D).' +tag: mmlu_flan_cot_fewshot_other +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_professional_medicine diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_psychology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..60b5da683ff87d207304b894a5138a6c439a1c86 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_psychology.yaml @@ -0,0 +1,62 @@ +dataset_name: professional_psychology +description: The following are multiple choice questions (with answers) about professional + psychology. +fewshot_config: + sampler: first_n + samples: + - question: 'In the construction of a multiple regression equation for purposes of + prediction, the optimal combination of measures is one in which the predictors + + (A) are uncorrelated with each other but are moderately correlated with the + criterion (B) have low correlations with each other and low correlations with + the criterion (C) are highly intercorrelated with each other and moderately + correlated with the criterion (D) have low correlations with the criterion bur + are moderately correlated with each other' + target: Let's think step by step. We refer to Wikipedia articles on psychology + for help. The basis of multiple regression is to assess the relationship between + one continuous variable and a set of independent variables. So the predictors + should be uncorrelated with each other but are moderately correlated with the + criterion. The answer is (A). + - question: 'There are three ways to measure the Central Tendency: the Mean, the Median + and the Mode. From your knowledge about them, what is the mode? + + (A) less sensitive to extreme scores than the mean (B) more useful for skewed + distributions (C) sensitive to extreme values and highly skewed distributions + (D) the most frequently occurring number' + target: Let's think step by step. We refer to Wikipedia articles on psychology + for help. The definition of mode is the most frequently occurring number. The + answer is (D). + - question: "Carl Jung believed that a client's transference:\n(A) is a fantasy that\ + \ distracts the client from reality. (B) represents \u201Cmixed feelings\u201D\ + \ toward the therapist. (C) \"is a form of \"\"acting out.\"\"\" (D) reflects\ + \ the client\u2019s personal and collective unconscious." + target: Let's think step by step. We refer to Wikipedia articles on psychology + for help. Transference is a phenomenon that a person's feelings are unconsciously + redirected, so it reflects the client's personal and collective unconscious. + The answer is (D). + - question: "In terms of Hofstede\u2019s (1980) five cultural dimensions, the United\ + \ States scores at the top of the scale on:\n(A) individualism. (B) individualism\ + \ and power distance. (C) power distance and masculinity. (D) uncertainty avoidance." + target: Let's think step by step. We refer to Wikipedia articles on psychology + for help. US scores highest on individualism among the five cultural dimensions. + The answer is (A). + - question: 'One of your therapy clients asks your advice about a good weight- reduction + program. You have investigated the programs in the community and are enrolled + in the one you consider the best. This program offers a $50 bonus to its patrons + for each new person they bring into the program. Under these circumstances, + your most appropriate response would be to + + (A) tell your client the pros and cons of each program you know about except + for the one in which you are enrolled (B) recommend to your client the program + in which you are enrolled and explain the $50 bonus you will receive (C) recommend + to your client the program in which you are enrolled and offer to have the $50 + bonus credited to your client''s account in the program (D) tell your client + the pros and cons of each program you know about, but do not claim the $50 bonus + if your client enrolls in your program' + target: 'Let''s think step by step. We refer to Wikipedia articles on psychology + for help. Based on the circumstances, you should tell your client about the + pros and cons of each program, but it would be inappropriate to receive the + bonus, so you should not claim the $50 bonus. The answer is (D).' +tag: mmlu_flan_cot_fewshot_social_sciences +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_professional_psychology diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_public_relations.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe384b1e2b7d19c216f8344d5c249f2c16dc723b --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_public_relations.yaml @@ -0,0 +1,55 @@ +dataset_name: public_relations +description: The following are multiple choice questions (with answers) about public + relations. +fewshot_config: + sampler: first_n + samples: + - question: 'Earth Hour was a campaign launched by which organization? + + (A) Greenpeace (B) The UN (C) Oxfam (D) World Wildlife Fund' + target: Let's think step by step. We refer to Wikipedia articles on public relations + for help. Earth Hour is a worldwide movement oragnized launched by the World + Wildlife Fund. The answer is (D). + - question: 'In issues management, what is the most proactive approach to addressing + negative or misleading information posted online about your organization? + + (A) Buy domain names that could be used by opposition groups. (B) Post anonymous + comments on blogs to combat this information. (C) Prepare a news release that + discredits the inaccurate information. (D) Make policy changes to address complaints + highlighted on these sites.' + target: Let's think step by step. We refer to Wikipedia articles on public relations + for help. In issues management, the most proactive approach to addressing negative + or misleading information posted online is to make policy changes to address + complaints highlighted on those sites. The answer is (D). + - question: 'At which stage in the planning process would a situation analysis be carried + out? + + (A) Defining the program (B) Planning the program (C) Taking action and implementing + ideas (D) Evaluation of the program' + target: Let's think step by step. We refer to Wikipedia articles on public relations + for help. Situation analyses are typically carried out during the planning process + stage of defining the program. The answer is (A). + - question: 'Which of these statements is true of the Vatican in 2010 at the time of + the accusations of child abuse cover-ups? + + (A) There was a coordinated media response. (B) Consistent messages were communicated. + (C) Criticisms were taken as attacks on the Catholic Church. (D) The credibility + of the Vatican was upheld.' + target: Let's think step by step. We refer to Wikipedia articles on public relations + for help. In 2010 when there were accusations of child abuse cover-ups, the + Vatican took those criticisms as attacks on the Catholic Church. The answer + is (C). + - question: 'What should a public relations media practitioner do if she does not know + the answer to a reporter''s question? + + (A) Give the reporter other information she is certain is correct. (B) Say that + the information is ''off the record'' and will be disseminated later. (C) Say + ''I don''t know'' and promise to provide the information later. (D) Say ''no + comment,'' rather than appear uninformed.' + target: 'Let''s think step by step. We refer to Wikipedia articles on public relations + for help. If a public relations media practitioner does not know the answer + to a reporter''s question, they should say ''I don''t know'' and offer to provide + the information later. The answer is (C).' +tag: mmlu_flan_cot_fewshot_social_sciences +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_public_relations diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_security_studies.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b37e35b3bd4fefa0ca040f0d59ff2fcae156fb45 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_security_studies.yaml @@ -0,0 +1,104 @@ +dataset_name: security_studies +description: The following are multiple choice questions (with answers) about security + studies. +fewshot_config: + sampler: first_n + samples: + - question: 'What are the frameworks of analysis within which terrorism has been considered + (as of 2020)? + + (A) Competition between larger nations has resulted in some countries actively + supporting terrorist groups to undermine the strength of rival states. Terrorist + networks are extended patronage clubs maintained and paid for by their donor + states and are conceptualised as being like state actors, to be dealt with using + military force. (B) Globalization has enabled the internationalization of terrorist + activities by opening up their operational space, although coordination is still + managed from a geographical base. This suggests that terrorist groups are nationally + structured which means that terrorism cannot be considered in terms of a war + to be defeated militarily without having serious implications on the indigenous + population. (C) Terrorism can be viewed as a problem to be resolved by military + means (war on terrorism), by normal police techniques (terrorism as crime), + or as a medical problem with underlying causes and symptoms (terrorism as disease). + (D) Terrorism is viewed as a criminal problem. The criminalization of terrorism + has two important implications. Firstly, it suggests that terrorism can be eradicated + - terrorists can be caught and brought to trial by normal judicial proceedings + thereby removing the threat from society - and secondly, it suggests that preventative + crime techniques are applicable to prevent its development.' + target: "Let's think step by step. We refer to Wikipedia articles on security\ + \ studies for help. (A) is wrong because it is not competition between larger\ + \ nations that causes terrorism. \n(B) is wrong because globalization is not\ + \ the cause of terrorism.\n(C) is correct because the US undertook the war on\ + \ terrorism. \n(D) is wrong because preventative crime techniques will likely\ + \ not end terrorism. The answer is (C)." + - question: 'Which of the following is the best lens through which to investigate the + role of child soldiers? + + (A) Child soldiers are victims of combat that need re-education and rehabilitation. + (B) Children and their mothers are not active subjects in warfare and are best + considered as subjects in the private sphere. (C) Children are most often innocent + bystanders in war and are best used as signifiers of peace. (D) Children have + political subjecthood that is missed when they are considered as passive victims + of warfare.' + target: Let's think step by step. We refer to Wikipedia articles on security studies + for help. Child soliders as a political topic can be missed when they are considered + passive victims of warfare. The answer is (D). + - question: 'How can we best describe the relationship between the state-centric approach + and the concept of human security? + + (A) There are such wide divisions within the human security framework regarding + the nature of threats and referent objects that no widely applicable comparisons + between state-centric approaches and human security can be drawn. (B) By adopting + the framework of human security, the limitations of the realist state-centric + approach become evident. Whilst human security defines the referent object as + the person or population, state-centric approaches prioritise the security of + the state, de-prioritizing the pursuit of human security. (C) The state-centric + approach to security is a faction of human security, usually defined within + the broad school of human security. By being state-centric this approach prioritises + the individual as the referent object in security studies. (D) Both the state-centric + and human-centric approaches to security are mutually exclusive and offer a + sufficient analytic framework with which to understand the international security + system. It is therefore the role of security analysts to determine which of + these substantial concepts is correct, and which should be discarded.' + target: Let's think step by step. We refer to Wikipedia articles on security studies + for help. Human security focuses on a person or population whereas state-centric + approaches focus on the state while deprioritizing human security. The answer + is (B). + - question: 'In order to become securitized, a threat must be presented in which of + these ways? + + (A) As an existential threat that requires immediate and extraordinary action, + posing a threat to the survival of the state or to societal security. (B) As + requiring immediate and extraordinary action by the state, threatening the survival + of a referent object and therefore warranting the use of measures not normally + employed in the political realm. (C) As an urgent threat to the survival of + the referent object, so serious that it legitimises the employment of extraordinary + action in response. (D) As an urgent threat to the survival of the audience + that requires extraordinary or emergency measures.' + target: Let's think step by step. We refer to Wikipedia articles on security studies + for help. To be securitized, a threat must be an urgent threat to the survival + of the referent object. The answer is (C). + - question: 'What distinguishes coercive diplomacy from military force? + + (A) Compellence is another term for coercive diplomacy, but covering a narrower + set of criteria; compellence covers those threats aimed at initiating adversary + action. A threat to coerce a state to give up part of its territory would count + as coercive diplomacy, as long as that threat proactively initiates action before + reactive diplomacy is taken. (B) Coercive diplomacy constitutes the threats + of limited force to induce adversary''s incentive to comply with the coercer''s + demands. It is an influence strategy that is intended to obtain compliance: + the use of force to defeat an opponent first does not count. It leaves an element + of choice with the target to comply, or to continue. (C) Military force, or + the threat of military force, utilises fear to achieve strategic objectives. + Coercive diplomacy is differentiated from this approach, because it does not + use fear as a tool for coercing an adversary. (D) Coercive diplomacy is employed + to use force but to limit its effects on the international community. Coercive + diplomacy is an aggressive strategy that is intended to obtain compliance through + defeat. It does not leave an element of choice with the target, the target either + being forced to comply or engage in conflict. It seeks to control by imposing + compliance by removing any opportunity for negotiation or concession.' + target: 'Let''s think step by step. We refer to Wikipedia articles on security + studies for help. Coercive diplomacy uses the threat of force to induce the + opponent to comply with demands. The answer is (B).' +tag: mmlu_flan_cot_fewshot_social_sciences +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_security_studies diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_sociology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4229d64785ded8673d421a9fb1571d0cce705a93 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_sociology.yaml @@ -0,0 +1,58 @@ +dataset_name: sociology +description: The following are multiple choice questions (with answers) about sociology. +fewshot_config: + sampler: first_n + samples: + - question: 'Which of the following is not a problem associated with official statistics + on strike action? + + (A) most strikes go unnoticed by employers and the mass media (B) not all industrial + disputes will be reported by the employer (C) the definition of strikes excludes + those that involve fewer than ten workers or last less than one day (D) it is + hard to compare strikes that were measured in different ways' + target: Let's think step by step. We refer to Wikipedia articles on sociology + for help. Official statistics on strike action can be problematic because not + all industrial disputes will be reported by employers, the definition of strikes + excludes those that involves fewer than ten workers or last less than one day, + and it is hard to compare strikes that were measured in different ways. Thus, + (A) is not a problem associated with official statistics on strike action. The + answer is (A). + - question: 'What does Berger (1963) describe as a metaphor for social reality? + + (A) a fairground ride (B) a circus (C) a puppet theatre (D) a ballet' + target: Let's think step by step. We refer to Wikipedia articles on sociology + for help. Berger describes social reality using the metaphor of a puppet theatre. + The answer is (C). + - question: 'The term ''hegemony'' refers to: + + (A) the tendency for the working class not to realize their own interests (B) + a dominant ideology that legitimates economic, political and cultural power + (C) a form of dual consciousness based on ideology and everyday experiences + (D) a mode of payment given for outstanding topiary' + target: Let's think step by step. We refer to Wikipedia articles on sociology + for help. Hegemony refers to a dominant ideology that legitimates economic, + policital, and cultural power. The answer is (B). + - question: 'The shift from ''civil religion'' to ''common religion'' means that: + + (A) the increasing bureaucracy of the state has made religion only a marginal + part of our lives (B) despite the weakening of traditional authority, our everyday + lives and ''common sense'' remain shaped by religious beliefs and values (C) + religious participation in collective worship may have declined, but people + still practise their faiths in private (D) people are much more likely to discuss + their religious beliefs in public, informal settings' + target: Let's think step by step. We refer to Wikipedia articles on sociology + for help. The shift from civil religion to common religion means that despite + the weakening of traditional authority, our everyday lives and common sense + remain shaped by religious beliefs and values. The answer is (B). + - question: 'Which of the following did the post-war welfare state of 1948 not aim + to provide: + + (A) free health care and education for all (B) a minimum wage (C) full employment + (D) universal welfare' + target: 'Let''s think step by step. We refer to Wikipedia articles on sociology + for help. The post-war welfare state of 1948 aimed to provide free healthcare + and education, full employment, and universal welfare. But it did not aim to + provide a minimum wage. The answer is (B).' +tag: mmlu_flan_cot_fewshot_social_sciences +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_sociology diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_us_foreign_policy.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc86b7c88fa1b62d2f12deaa16394f43fc722225 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_us_foreign_policy.yaml @@ -0,0 +1,56 @@ +dataset_name: us_foreign_policy +description: The following are multiple choice questions (with answers) about us foreign + policy. +fewshot_config: + sampler: first_n + samples: + - question: 'How did Donald Trump attack globalization in the 2016 campaign? + + (A) Globalization had made men like him too rich (B) Globalization only benefited + certain American states, such as New York (C) Liberal elites had encouraged + globalization, while ''ordinary Americans'' lost jobs because of it (D) Globalization + encouraged damaging trade wars' + target: Let's think step by step. We refer to Wikipedia articles on us foreign + policy for help. Trump attacked globalization because he believed ordinary Americans + lost jobs due to it, and so he wanted to blame liberals who had encouraged it. + The answer is (C). + - question: 'How did NSC-68 change U.S. strategy? + + (A) It globalized containment. (B) It militarized containment. (C) It called + for the development of the hydrogen bomb. (D) All of the above' + target: Let's think step by step. We refer to Wikipedia articles on us foreign + policy for help. NSC-68 outlined a variety of courses of action, including globalization + of containment, militarization of contaiment, and the development of the hydrogen + bomb. The answer is (D). + - question: 'How do Defensive Realism and Offensive Realism differ in their explanation + of state behaviour? + + (A) Defensive realists place greater emphasis on the role of international institutions + (B) Defensive realists place less emphasis on geographical factors (C) Offensive + realists give more priority to the national interest than Defensive realists. + (D) Defensive realists believe states are security maximizers, while Offensive + realists believe states to be power maximizers' + target: Let's think step by step. We refer to Wikipedia articles on us foreign + policy for help. While defensive realism advocates that states are security + maximizers, offensive realists think of states as power maximizers. The answer + is (D). + - question: 'The realm of policy decisions concerned primarily with relations between + the United States and the rest of the world is known as + + (A) terrorism policy. (B) economic policy. (C) foreign policy. (D) international + policy.' + target: Let's think step by step. We refer to Wikipedia articles on us foreign + policy for help. The topic of policy decisions concerns with relations between + the US and the rest of the world is known as foreign policy. The answer is (C). + - question: 'How did the 2008 financial crisis affect America''s international reputation? + + (A) It damaged support for the US model of political economy and capitalism + (B) It created anger at the United States for exaggerating the crisis (C) It + increased support for American global leadership under President Obama (D) It + reduced global use of the US dollar' + target: 'Let''s think step by step. We refer to Wikipedia articles on us foreign + policy for help. The 2008 financial crisis damanged the international reputation + of the American model of political economy and capitalism. The answer is (A).' +tag: mmlu_flan_cot_fewshot_social_sciences +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_us_foreign_policy diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_virology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0e70f0ee24cc946a1cfbc51cb87d4fde20d8171c --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_virology.yaml @@ -0,0 +1,45 @@ +dataset_name: virology +description: The following are multiple choice questions (with answers) about virology. +fewshot_config: + sampler: first_n + samples: + - question: 'The median survival time to AIDS and death was established by following: + + (A) Seroprevalent HIV-infected individuals (B) Seronegatives (C) Seroconverters + (D) High-risk seronegatives' + target: Let's think step by step. We refer to Wikipedia articles on virology for + help. The median survival time to AIDS and death was established as a result + of the development of seroconverters. The answer is (C). + - question: 'Which of the following is a morphological characteristic of the paramyxoviruses. + + (A) Fragile viruses often visualised with RNA spewing from the inside (B) Elongate + viruses (C) Icosahedral viruses with envelope (D) Very large viruses' + target: Let's think step by step. We refer to Wikipedia articles on virology for + help. Paramyxoviruses are fragile viruses often visualised with RNA spewing + from the inside. The answer is (A). + - question: 'The most important goal of a behavioral intervention is: + + (A) Change in behavior (B) Comprehensive coverage (C) Effective use of behavioral + theory (D) Sustained behavior change' + target: Let's think step by step. We refer to Wikipedia articles on virology for + help. The prim goal of a behavioral intervention is to cause sustained behavior + change. The answer is (D). + - question: 'A key factor facilitating the application of nested case-control studies + from the MACS was: + + (A) Data collection (B) Establishment of a repository of biologic specimens + (C) Participant interest (D) Administration of the questionnaire by staff' + target: Let's think step by step. We refer to Wikipedia articles on virology for + help. The Multicenter AIDS Cohort Study's use of nested case-control studies + was facilitated by the establishment of a repository of biologic specimens. + The answer is (B). + - question: 'Why are parvoviruses a highly impactful parasite? + + (A) Because they have no nucleic acid (B) They require a helper virus (C) Only + replicate in dividing cells (D) Can integrate into host chromosomes' + target: 'Let''s think step by step. We refer to Wikipedia articles on virology + for help. Paroviruses are highly impactful because they do not have nucleic + acid. The answer is (A).' +tag: mmlu_flan_cot_fewshot_other +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_virology diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_world_religions.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..41502cc7a3a1318b7c6a0f2ac16cda86dda08486 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_world_religions.yaml @@ -0,0 +1,42 @@ +dataset_name: world_religions +description: The following are multiple choice questions (with answers) about world + religions. +fewshot_config: + sampler: first_n + samples: + - question: 'How can the Upanishads be characterized? + + (A) Ritual texts (B) Philosophical texts (C) Hymns (D) Origin stories' + target: Let's think step by step. We refer to Wikipedia articles on world religions + for help. The Upanishads are the most recent part of Vedas (the oldest scriptures + in Hinduism) and supplied the basis of later Hindu philosophy. So they are philosophical + texts. The answer is (B). + - question: 'What is the Second Gem in Buddhism? + + (A) The Dharma (B) The Sangha (C) The Buddha (D) The Bodhisattva' + target: Let's think step by step. We refer to Wikipedia articles on world religions + for help. The Second Gem in Buddhism is The Dharma. The answer is (A). + - question: 'Which Japanese government promoted a kind of national cult based on the + emperor and his associations with kami? + + (A) Honen (B) Tanaka (C) Tokugawa (D) Meiji' + target: Let's think step by step. We refer to Wikipedia articles on world religions + for help. The promotion of a national cult based on the emperor and his associations + with Kami happened during the reign of Emperor Meiji (1852-1912). The answer + is (D). + - question: 'In which dynasty was the "Mandate of Heaven" developed to legitimatize + the new rulers? + + (A) Shang (B) Zhou (C) Han (D) Xia' + target: Let's think step by step. We refer to Wikipedia articles on world religions + for help. The "Mandate of Heaven" was developed as an ancient Chinese philosophical + concept during the Zhou Dynasty (1046-256 BCE). The answer is (B). + - question: 'What is the sign of the covenant for Jewish males? + + (A) The rainbow (B) Circumcision (C) A son (D) Bar mitzvah' + target: 'Let''s think step by step. We refer to Wikipedia articles on world religions + for help. In Judaism, the most distinctive sign of the covenant is circumcision + (brit milah). The answer is (B).' +tag: mmlu_flan_cot_fewshot_humanities +include: _mmlu_flan_cot_fewshot_template_yaml +task: mmlu_flan_cot_fewshot_world_religions diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_clinical_knowledge.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c0e9d17db10f4e69d1c44d5a127f2bbe1f4e279 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_clinical_knowledge.yaml @@ -0,0 +1,6 @@ +"dataset_name": "clinical_knowledge" +"description": "The following are multiple choice questions (with answers) about clinical\ + \ knowledge.\n\n" +"tag": "mmlu_flan_cot_zeroshot_other" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_clinical_knowledge" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_medicine.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0450a068f4b763629e463d9882e4a3e99f86d726 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_medicine.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_medicine" +"description": "The following are multiple choice questions (with answers) about college\ + \ medicine.\n\n" +"tag": "mmlu_flan_cot_zeroshot_other" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_college_medicine" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_physics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c149ef083a87f6d3eb412f9e3fb2fbd131ec4c0e --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_physics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_physics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school physics.\n\n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_high_school_physics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_statistics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0f905569c82f31ec76a75505bfae64c28d72640 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_statistics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics.\n\n" +"tag": "mmlu_flan_cot_zeroshot_stem" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_high_school_statistics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_international_law.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..33766a464fa475a012d229c194c93fffb84942b6 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_international_law.yaml @@ -0,0 +1,6 @@ +"dataset_name": "international_law" +"description": "The following are multiple choice questions (with answers) about international\ + \ law.\n\n" +"tag": "mmlu_flan_cot_zeroshot_humanities" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_international_law" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_prehistory.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9db801a6a9f2d911e2bdbbe0084fd235c7572776 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_prehistory.yaml @@ -0,0 +1,6 @@ +"dataset_name": "prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory.\n\ + \n" +"tag": "mmlu_flan_cot_zeroshot_humanities" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_prehistory" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_professional_medicine.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f961bff89745dd8999c2ee497bdf9a7df88e04f --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_professional_medicine.yaml @@ -0,0 +1,6 @@ +"dataset_name": "professional_medicine" +"description": "The following are multiple choice questions (with answers) about professional\ + \ medicine.\n\n" +"tag": "mmlu_flan_cot_zeroshot_other" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_professional_medicine" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_security_studies.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..062f49630e82b66be1ea0e75ed9fe73c8d635215 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_security_studies.yaml @@ -0,0 +1,6 @@ +"dataset_name": "security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies.\n\n" +"tag": "mmlu_flan_cot_zeroshot_social_sciences" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_security_studies" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_us_foreign_policy.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c4afb8f84a193442cd98a856ada7e43f1515cbce --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_us_foreign_policy.yaml @@ -0,0 +1,6 @@ +"dataset_name": "us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy.\n\n" +"tag": "mmlu_flan_cot_zeroshot_social_sciences" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_us_foreign_policy" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_world_religions.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0eb04f31f0baaf6ac0f358de2897d5267e1a4357 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_world_religions.yaml @@ -0,0 +1,6 @@ +"dataset_name": "world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions.\n\n" +"tag": "mmlu_flan_cot_zeroshot_humanities" +"include": "_mmlu_flan_cot_zeroshot_template_yaml" +"task": "mmlu_flan_cot_zeroshot_world_religions" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/_mmlu.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/_mmlu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..14465ad6e5c5434974832399ea95903b59e4eaf5 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/_mmlu.yaml @@ -0,0 +1,32 @@ +group: mmlu_flan_n_shot_generative +group_alias: mmlu (flan style, generative) +task: + - group: stem + task: + - mmlu_flan_n_shot_generative_stem + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: other + task: + - mmlu_flan_n_shot_generative_other + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: social sciences + task: + - mmlu_flan_n_shot_generative_social_sciences + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: humanities + task: + - mmlu_flan_n_shot_generative_humanities + aggregate_metric_list: + - metric: acc + weight_by_size: True +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 2 diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/_mmlu_flan_generative_template_yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/_mmlu_flan_generative_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..a38a06969e2649d2fc0cf8e2be3efc60d91b3076 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/_mmlu_flan_generative_template_yaml @@ -0,0 +1,34 @@ +dataset_path: hails/mmlu_no_train # a copy of `cais/mmlu` with no auxiliary_train split +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: generate_until +doc_to_text: "Q: {{question.strip()}}\n(A) {{choices[0]}} (B) {{choices[1]}} (C) {{choices[2]}} (D) {{choices[3]}}\nA:" +doc_to_target: "{{['(A)', '(B)', '(C)', '(D)'][answer]}}" +filter_list: + - name: "strict-match" + filter: + - function: "take_first" + - name: "flexible-extract" + filter: + - function: "multi_choice_regex" + group_select: 0 + regex_pattern: "(\\([A-Z]\\))" + ignore_case: true + ignore_punctuation: true + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "<|im_end|>" + - "\n" +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true +metadata: + version: 3.0 +dataset_kwargs: + trust_remote_code: true diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_abstract_algebra.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e602ee8100ed612d89385532ea30004c3033c35 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_abstract_algebra.yaml @@ -0,0 +1,6 @@ +"dataset_name": "abstract_algebra" +"description": "The following are multiple choice questions (with answers) about abstract\ + \ algebra.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_abstract_algebra" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_anatomy.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa12cc8ef35b19f3b81dcc58a0107d424a3580cc --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_anatomy.yaml @@ -0,0 +1,6 @@ +"dataset_name": "anatomy" +"description": "The following are multiple choice questions (with answers) about anatomy.\n\ + \n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_anatomy" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_astronomy.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a4178654e0e6e7a053839319c7936967133cf756 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_astronomy.yaml @@ -0,0 +1,6 @@ +"dataset_name": "astronomy" +"description": "The following are multiple choice questions (with answers) about astronomy.\n\ + \n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_astronomy" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_business_ethics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4447d276b066ddec93b8f7efcf2d74d13810f458 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_business_ethics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "business_ethics" +"description": "The following are multiple choice questions (with answers) about business\ + \ ethics.\n\n" +"tag": "mmlu_flan_n_shot_generative_other" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_business_ethics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_clinical_knowledge.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..38f799060fa6901b890d3a87d8aa9b9444d34b57 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_clinical_knowledge.yaml @@ -0,0 +1,6 @@ +"dataset_name": "clinical_knowledge" +"description": "The following are multiple choice questions (with answers) about clinical\ + \ knowledge.\n\n" +"tag": "mmlu_flan_n_shot_generative_other" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_clinical_knowledge" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_biology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f36eb1f598f754154c2b15b24bbb650358c707c5 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_biology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_biology" +"description": "The following are multiple choice questions (with answers) about college\ + \ biology.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_college_biology" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_chemistry.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0eccce652fade13a319af78e06a7528b11814302 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_chemistry.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_chemistry" +"description": "The following are multiple choice questions (with answers) about college\ + \ chemistry.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_college_chemistry" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_computer_science.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fd415aa10efaf96331d9fef82c5b6a2bb538263a --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_computer_science.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_computer_science" +"description": "The following are multiple choice questions (with answers) about college\ + \ computer science.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_college_computer_science" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_mathematics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2d062721102c0f6e6c09574398a60db74c26b593 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_mathematics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_mathematics" +"description": "The following are multiple choice questions (with answers) about college\ + \ mathematics.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_college_mathematics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_medicine.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..edc660d9c30dfad6666f5e1b4c679489f62c5991 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_medicine.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_medicine" +"description": "The following are multiple choice questions (with answers) about college\ + \ medicine.\n\n" +"tag": "mmlu_flan_n_shot_generative_other" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_college_medicine" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_physics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aac8f400d1d9005376bfe3354753e87700a7bda8 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_physics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_college_physics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_computer_security.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..178c468346a5022a5d0031fd27c6b9a07ab24150 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_computer_security.yaml @@ -0,0 +1,6 @@ +"dataset_name": "computer_security" +"description": "The following are multiple choice questions (with answers) about computer\ + \ security.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_computer_security" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_conceptual_physics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3cfbe6250d19aab6e60c9089f0feb91eed37423 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_conceptual_physics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "conceptual_physics" +"description": "The following are multiple choice questions (with answers) about conceptual\ + \ physics.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_conceptual_physics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_econometrics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ad8704e4f8e3a60ee2ff7e370cf7394c0359aeb7 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_econometrics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "econometrics" +"description": "The following are multiple choice questions (with answers) about econometrics.\n\ + \n" +"tag": "mmlu_flan_n_shot_generative_social_sciences" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_econometrics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_electrical_engineering.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..56eeae0183ca0c087b0a16aa317f2b93d5f1b87b --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_electrical_engineering.yaml @@ -0,0 +1,6 @@ +"dataset_name": "electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_electrical_engineering" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_elementary_mathematics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..da3b3af2b5f310232cbd9c9ee63081acbb571638 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_elementary_mathematics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "elementary_mathematics" +"description": "The following are multiple choice questions (with answers) about elementary\ + \ mathematics.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_elementary_mathematics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_formal_logic.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2d3f4edc644842cbc3fae865c96f99322daaafbf --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_formal_logic.yaml @@ -0,0 +1,6 @@ +"dataset_name": "formal_logic" +"description": "The following are multiple choice questions (with answers) about formal\ + \ logic.\n\n" +"tag": "mmlu_flan_n_shot_generative_humanities" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_formal_logic" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_global_facts.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4feef1895254438bde19ebfc3d7a36aee87e61de --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_global_facts.yaml @@ -0,0 +1,6 @@ +"dataset_name": "global_facts" +"description": "The following are multiple choice questions (with answers) about global\ + \ facts.\n\n" +"tag": "mmlu_flan_n_shot_generative_other" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_global_facts" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_biology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..245d9be815c3644bf3298a0d093a76410b7487b6 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_biology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_biology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school biology.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_high_school_biology" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_chemistry.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..34eb30d32d5b6927d44d59a63f5a549587f414f1 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_chemistry.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_high_school_chemistry" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_computer_science.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..34250a6c61cb5e29acbb99f8a080d45f74a91d45 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_computer_science.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_computer_science" +"description": "The following are multiple choice questions (with answers) about high\ + \ school computer science.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_high_school_computer_science" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_european_history.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..42b7dd4d5aa2ab541b7f269c84845d262db452c5 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_european_history.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_european_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school european history.\n\n" +"tag": "mmlu_flan_n_shot_generative_humanities" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_high_school_european_history" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_geography.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e67277aa5480e1a9465169112755c3da70e12e6e --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_geography.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_geography" +"description": "The following are multiple choice questions (with answers) about high\ + \ school geography.\n\n" +"tag": "mmlu_flan_n_shot_generative_social_sciences" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_high_school_geography" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_government_and_politics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..84643a74239db620816f0d8a67575d0c8268e58f --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_government_and_politics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_government_and_politics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school government and politics.\n\n" +"tag": "mmlu_flan_n_shot_generative_social_sciences" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_high_school_government_and_politics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_macroeconomics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb08333804237ac3e0584db637d5c91477a6a93d --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_macroeconomics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_macroeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school macroeconomics.\n\n" +"tag": "mmlu_flan_n_shot_generative_social_sciences" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_high_school_macroeconomics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_mathematics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f1ca028d8262f22807eb591c3e498fecabd9887b --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_mathematics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_mathematics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school mathematics.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_high_school_mathematics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_microeconomics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c60982b78dab4866a6827fe5b1bf9f2b710ed8d3 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_microeconomics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_microeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school microeconomics.\n\n" +"tag": "mmlu_flan_n_shot_generative_social_sciences" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_high_school_microeconomics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_physics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..33b8d16739c9faf352ad242bd76b2bc33bc21aa6 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_physics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_physics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school physics.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_high_school_physics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_psychology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f47bbbb68c02a417e60e5b0a19f4f85c5723b41b --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_psychology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_psychology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school psychology.\n\n" +"tag": "mmlu_flan_n_shot_generative_social_sciences" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_high_school_psychology" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_statistics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..741971895ba27ad6651ac456def204a078ac5d3e --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_statistics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_high_school_statistics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_us_history.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..48696971c9e850a18baadd6c3e9f958851cc2a3e --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_us_history.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history.\n\n" +"tag": "mmlu_flan_n_shot_generative_humanities" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_high_school_us_history" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_world_history.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ae6cfcbba3f86dc0339edc3a361c898e6c8716fd --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_world_history.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_world_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school world history.\n\n" +"tag": "mmlu_flan_n_shot_generative_humanities" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_high_school_world_history" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_human_aging.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..677f119a754f0c671fae0f2285bb8ff29f2af85e --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_human_aging.yaml @@ -0,0 +1,6 @@ +"dataset_name": "human_aging" +"description": "The following are multiple choice questions (with answers) about human\ + \ aging.\n\n" +"tag": "mmlu_flan_n_shot_generative_other" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_human_aging" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_human_sexuality.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d4e33d7d607ef2f07ea0fdb67305b8f88a45d13a --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_human_sexuality.yaml @@ -0,0 +1,6 @@ +"dataset_name": "human_sexuality" +"description": "The following are multiple choice questions (with answers) about human\ + \ sexuality.\n\n" +"tag": "mmlu_flan_n_shot_generative_social_sciences" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_human_sexuality" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_international_law.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac5d9d5a46b7f4f1daafb7c7f0feb66933c4829d --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_international_law.yaml @@ -0,0 +1,6 @@ +"dataset_name": "international_law" +"description": "The following are multiple choice questions (with answers) about international\ + \ law.\n\n" +"tag": "mmlu_flan_n_shot_generative_humanities" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_international_law" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_jurisprudence.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c2f135869aca516492cd9dc8ce210838173a1d7a --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_jurisprudence.yaml @@ -0,0 +1,6 @@ +"dataset_name": "jurisprudence" +"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ + \n" +"tag": "mmlu_flan_n_shot_generative_humanities" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_jurisprudence" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_logical_fallacies.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6624e07743a432cc354ccff7af2363db2ec1ae11 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_logical_fallacies.yaml @@ -0,0 +1,6 @@ +"dataset_name": "logical_fallacies" +"description": "The following are multiple choice questions (with answers) about logical\ + \ fallacies.\n\n" +"tag": "mmlu_flan_n_shot_generative_humanities" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_logical_fallacies" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_machine_learning.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ab6c459ae50e7311dc9d8819ec753c69f6d9583b --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_machine_learning.yaml @@ -0,0 +1,6 @@ +"dataset_name": "machine_learning" +"description": "The following are multiple choice questions (with answers) about machine\ + \ learning.\n\n" +"tag": "mmlu_flan_n_shot_generative_stem" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_machine_learning" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_management.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4af9ded012e921feeb38d31cde98fef9888aba95 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_management.yaml @@ -0,0 +1,6 @@ +"dataset_name": "management" +"description": "The following are multiple choice questions (with answers) about management.\n\ + \n" +"tag": "mmlu_flan_n_shot_generative_other" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_management" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_marketing.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..22ef9d3fd49556afd4578685099abc0bb9b64c9e --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_marketing.yaml @@ -0,0 +1,6 @@ +"dataset_name": "marketing" +"description": "The following are multiple choice questions (with answers) about marketing.\n\ + \n" +"tag": "mmlu_flan_n_shot_generative_other" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_marketing" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_medical_genetics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c24da7938b431acdd991830424777e6645cf9bbb --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_medical_genetics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "medical_genetics" +"description": "The following are multiple choice questions (with answers) about medical\ + \ genetics.\n\n" +"tag": "mmlu_flan_n_shot_generative_other" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_medical_genetics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_miscellaneous.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5b90845321c954cc2e7875fdc084e5935444af7 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_miscellaneous.yaml @@ -0,0 +1,6 @@ +"dataset_name": "miscellaneous" +"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ + \n" +"tag": "mmlu_flan_n_shot_generative_other" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_miscellaneous" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_moral_disputes.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..295c39a6efce509983b01b18c20375866b08d3bc --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_moral_disputes.yaml @@ -0,0 +1,6 @@ +"dataset_name": "moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes.\n\n" +"tag": "mmlu_flan_n_shot_generative_humanities" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_moral_disputes" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_moral_scenarios.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f09f982f26462304a20420e9b61bf3ef941448a0 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_moral_scenarios.yaml @@ -0,0 +1,6 @@ +"dataset_name": "moral_scenarios" +"description": "The following are multiple choice questions (with answers) about moral\ + \ scenarios.\n\n" +"tag": "mmlu_flan_n_shot_generative_humanities" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_moral_scenarios" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_nutrition.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf633f270a6d9fbbaa0a793bc5d5e48731a31d57 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_nutrition.yaml @@ -0,0 +1,6 @@ +"dataset_name": "nutrition" +"description": "The following are multiple choice questions (with answers) about nutrition.\n\ + \n" +"tag": "mmlu_flan_n_shot_generative_other" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_nutrition" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_philosophy.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6a5fe27eefb47badf4c13e87ad0fbac96b08283e --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_philosophy.yaml @@ -0,0 +1,6 @@ +"dataset_name": "philosophy" +"description": "The following are multiple choice questions (with answers) about philosophy.\n\ + \n" +"tag": "mmlu_flan_n_shot_generative_humanities" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_philosophy" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_prehistory.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..60788fc6c201bf316398f48adc9575dcb806b649 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_prehistory.yaml @@ -0,0 +1,6 @@ +"dataset_name": "prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory.\n\ + \n" +"tag": "mmlu_flan_n_shot_generative_humanities" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_prehistory" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_accounting.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f312af231f28d9343f7a0e2353cec110fda1f9a4 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_accounting.yaml @@ -0,0 +1,6 @@ +"dataset_name": "professional_accounting" +"description": "The following are multiple choice questions (with answers) about professional\ + \ accounting.\n\n" +"tag": "mmlu_flan_n_shot_generative_other" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_professional_accounting" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_law.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..be0533f0d8b90fc9f82226579ec849ac3f24be15 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_law.yaml @@ -0,0 +1,6 @@ +"dataset_name": "professional_law" +"description": "The following are multiple choice questions (with answers) about professional\ + \ law.\n\n" +"tag": "mmlu_flan_n_shot_generative_humanities" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_professional_law" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_medicine.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9cae6f8a5ec27d73bcf9b57e8597b377aee62835 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_medicine.yaml @@ -0,0 +1,6 @@ +"dataset_name": "professional_medicine" +"description": "The following are multiple choice questions (with answers) about professional\ + \ medicine.\n\n" +"tag": "mmlu_flan_n_shot_generative_other" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_professional_medicine" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_psychology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21a39c51b7d246c3dd49e47ee0f5dd1865059c36 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_psychology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology.\n\n" +"tag": "mmlu_flan_n_shot_generative_social_sciences" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_professional_psychology" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_public_relations.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b2687d99a279caac3f322ff178a1ea1ac7ea44f8 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_public_relations.yaml @@ -0,0 +1,6 @@ +"dataset_name": "public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations.\n\n" +"tag": "mmlu_flan_n_shot_generative_social_sciences" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_public_relations" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_security_studies.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6c36a5522d3c0d6f165dbd5eaac9f5208822fb9d --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_security_studies.yaml @@ -0,0 +1,6 @@ +"dataset_name": "security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies.\n\n" +"tag": "mmlu_flan_n_shot_generative_social_sciences" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_security_studies" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_sociology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ce0809907575855a8680ec1db533688ad42de46 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_sociology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "sociology" +"description": "The following are multiple choice questions (with answers) about sociology.\n\ + \n" +"tag": "mmlu_flan_n_shot_generative_social_sciences" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_sociology" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_us_foreign_policy.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..56ed5e16281b6aca3720868538c93d2877d438b6 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_us_foreign_policy.yaml @@ -0,0 +1,6 @@ +"dataset_name": "us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy.\n\n" +"tag": "mmlu_flan_n_shot_generative_social_sciences" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_us_foreign_policy" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_virology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..257dcfbf8a18c96d836d6db1214e8ff69ec63278 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_virology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "virology" +"description": "The following are multiple choice questions (with answers) about virology.\n\ + \n" +"tag": "mmlu_flan_n_shot_generative_other" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_virology" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_world_religions.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..39b64d03d3983f5c692a1a762c8457175dbf5408 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_world_religions.yaml @@ -0,0 +1,6 @@ +"dataset_name": "world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions.\n\n" +"tag": "mmlu_flan_n_shot_generative_humanities" +"include": "_mmlu_flan_generative_template_yaml" +"task": "mmlu_flan_n_shot_generative_world_religions" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/utils.py b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..72246935de8cf0cf8b256fd1e6c87dfbbb90a2ad --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/generative/utils.py @@ -0,0 +1,112 @@ +import re +import sys +import unicodedata + +from lm_eval.filters.extraction import RegexFilter + + +class MultiChoiceRegexFilter(RegexFilter): + """ """ + + def __init__( + self, + regex_pattern: str = r"#### (\-?[0-9\.\,]+)", + group_select=0, + fallback: str = "[invalid]", + ignore_case=False, + ignore_punctuation=False, + regexes_to_ignore=None, + ) -> None: + """ + regex_pattern: The basic regex pattern to use. If fails to match, we will use the customized match procedure + - step 1 : We parse the choices between ([A-Z])s then try to find these choices in the response. + - step 2 : We parse the choice with regex :[\s]*([A-?]), where ? varies by number of choices. + group_select: Selects the (group_select)th match from the findall result. + ignore_case: Ignores the case during step 1 matching + ignore_punctuation: Remove the punctuation during step 1 matching + regexes_to_ignore: Remove these regexes during step 1 matching + """ + super().__init__(regex_pattern, group_select, fallback) + self.ignore_case = ignore_case + self.ignore_punctuation = ignore_punctuation + self.regexes_to_ignore = regexes_to_ignore + + def apply(self, resps, docs): + # here, we assume we have a list, in which each element is + # a list of model responses for some particular input/target pair. + # so we process each of these (same input/target response sets) + # independently (and keep them a list.) + + def find_match(regex, resp, convert_dict={}): + match = regex.findall(resp) + if match: + match = match[self.group_select] + if isinstance(match, tuple): + match = [m for m in match if m][0] + match = match.strip() + if match and match in convert_dict: + match = convert_dict[match] + return match + + punct_tbl = dict.fromkeys( + i + for i in range(sys.maxunicode) + if unicodedata.category(chr(i)).startswith("P") + ) + + def filter_ignores(st): + if self.regexes_to_ignore is not None: + for s in self.regexes_to_ignore: + st = re.sub(s, "", st) + + if self.ignore_case: + st = st.lower() + + if self.ignore_punctuation: + # https://stackoverflow.com/a/266162 + st = st.translate(punct_tbl) + return st + + filtered_resps = [] + + for r, doc in zip(resps, docs): + fallback_regexes = [] + choice_to_alpha = {} + next_alpha = "A" + + without_paren_fallback_regexes = [] + without_paren_to_target = {} + + choices = doc["choices"] + for c in choices: + m = filter_ignores(c.strip()) + fallback_regexes.append(f"{re.escape(m)}") + choice_to_alpha[m] = f"({next_alpha})" + + without_paren_fallback_regexes.append(next_alpha) + without_paren_to_target[next_alpha] = f"({next_alpha})" + + next_alpha = chr(ord(next_alpha) + 1) + fallback_regex = re.compile("|".join(fallback_regexes)) + without_paren_fallback_regex = "|".join(without_paren_fallback_regexes) + without_paren_fallback_regex = re.compile( + f":[\s]*({without_paren_fallback_regex})" + ) + + filtered = [] + for resp in r: + match = find_match(self.regex, resp) + if not match: + match = find_match( + fallback_regex, filter_ignores(resp), choice_to_alpha + ) + if not match: + match = find_match( + without_paren_fallback_regex, resp, without_paren_to_target + ) + if not match: + match = self.fallback + filtered.append(match) + filtered_resps.append(filtered) + + return filtered_resps diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/_mmlu.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/_mmlu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2cfa0fb9c30451fa79f6b8b038a01692c830f1a7 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/_mmlu.yaml @@ -0,0 +1,32 @@ +group: mmlu_flan_n_shot_loglikelihood +group_alias: mmlu (flan style, loglikelihood) +task: + - group: stem + task: + - mmlu_flan_n_shot_loglikelihood_stem + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: other + task: + - mmlu_flan_n_shot_loglikelihood_other + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: social sciences + task: + - mmlu_flan_n_shot_loglikelihood_social_sciences + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: humanities + task: + - mmlu_flan_n_shot_loglikelihood_humanities + aggregate_metric_list: + - metric: acc + weight_by_size: True +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 2 diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/_mmlu_flan_loglikelihood_template_yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/_mmlu_flan_loglikelihood_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..4605a4a15f2e84c4572388192fc1e51d717f70b1 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/_mmlu_flan_loglikelihood_template_yaml @@ -0,0 +1,17 @@ +dataset_path: hails/mmlu_no_train # a copy of `cais/mmlu` with no auxiliary_train split +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "Q: {{question.strip()}}\n(A) {{choices[0]}} (B) {{choices[1]}} (C) {{choices[2]}} (D) {{choices[3]}}\nA:" +doc_to_choice: ["(A)", "(B)", "(C)", "(D)"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 2.0 +dataset_kwargs: + trust_remote_code: true diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_abstract_algebra.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f5dfa65ded384d6e1299b8e5564f5a655f2ced79 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_abstract_algebra.yaml @@ -0,0 +1,6 @@ +"dataset_name": "abstract_algebra" +"description": "The following are multiple choice questions (with answers) about abstract\ + \ algebra.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_abstract_algebra" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_anatomy.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e837e5d8fd3e1577af4d23d2120d1b55029f052f --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_anatomy.yaml @@ -0,0 +1,6 @@ +"dataset_name": "anatomy" +"description": "The following are multiple choice questions (with answers) about anatomy.\n\ + \n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_anatomy" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_astronomy.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..43b9bc7ed89429c2d08cc74cc4472ebea28f67a2 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_astronomy.yaml @@ -0,0 +1,6 @@ +"dataset_name": "astronomy" +"description": "The following are multiple choice questions (with answers) about astronomy.\n\ + \n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_astronomy" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_business_ethics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2438e6678be07c008922d83ea5016efab56ebc78 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_business_ethics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "business_ethics" +"description": "The following are multiple choice questions (with answers) about business\ + \ ethics.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_other" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_business_ethics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_clinical_knowledge.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..82d66adda5d600a94d5f6e36544dd63d2de3fece --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_clinical_knowledge.yaml @@ -0,0 +1,6 @@ +"dataset_name": "clinical_knowledge" +"description": "The following are multiple choice questions (with answers) about clinical\ + \ knowledge.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_other" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_clinical_knowledge" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_biology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..15e6e75d3491dfd034df789a3481fb3a39dcaa02 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_biology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_biology" +"description": "The following are multiple choice questions (with answers) about college\ + \ biology.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_college_biology" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_chemistry.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2b8c1bd3a8de310698082f738d287743d3731c23 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_chemistry.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_chemistry" +"description": "The following are multiple choice questions (with answers) about college\ + \ chemistry.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_college_chemistry" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_computer_science.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1178c7b072f82bebdd4281a371d6105514a686e8 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_computer_science.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_computer_science" +"description": "The following are multiple choice questions (with answers) about college\ + \ computer science.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_college_computer_science" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_mathematics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9776889b514c04c6c93aeedfd0ced7c620d11493 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_mathematics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_mathematics" +"description": "The following are multiple choice questions (with answers) about college\ + \ mathematics.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_college_mathematics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_medicine.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c8fdad90bd103ff616b4b14c2a3e9024208e149a --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_medicine.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_medicine" +"description": "The following are multiple choice questions (with answers) about college\ + \ medicine.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_other" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_college_medicine" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_physics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..77a89689127b4ca129b9434653198b051324fc0a --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_physics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_college_physics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_computer_security.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e787e51745218e2465b739ee82b51c456bd228ab --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_computer_security.yaml @@ -0,0 +1,6 @@ +"dataset_name": "computer_security" +"description": "The following are multiple choice questions (with answers) about computer\ + \ security.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_computer_security" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_conceptual_physics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..859e88e48a5cea7114b85c31c594f832520bacb0 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_conceptual_physics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "conceptual_physics" +"description": "The following are multiple choice questions (with answers) about conceptual\ + \ physics.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_conceptual_physics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_econometrics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0455a515eab5e3102a659d917758b942c00b952d --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_econometrics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "econometrics" +"description": "The following are multiple choice questions (with answers) about econometrics.\n\ + \n" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_econometrics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_electrical_engineering.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b63e06172ec302a916f3be4b0a2ea0f1efa86674 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_electrical_engineering.yaml @@ -0,0 +1,6 @@ +"dataset_name": "electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_electrical_engineering" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_elementary_mathematics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..79771d21543868dd73bf6ff84201ef07d79c89a2 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_elementary_mathematics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "elementary_mathematics" +"description": "The following are multiple choice questions (with answers) about elementary\ + \ mathematics.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_elementary_mathematics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_formal_logic.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e46d8e21c62ff03a6f47bbbc7a6d085840049a4 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_formal_logic.yaml @@ -0,0 +1,6 @@ +"dataset_name": "formal_logic" +"description": "The following are multiple choice questions (with answers) about formal\ + \ logic.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_formal_logic" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_global_facts.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9e7aff59325d7dab9a02c4eda3a886d062fe3b4a --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_global_facts.yaml @@ -0,0 +1,6 @@ +"dataset_name": "global_facts" +"description": "The following are multiple choice questions (with answers) about global\ + \ facts.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_other" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_global_facts" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_biology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dfe33de2be1d2f821c92fc46111150e1ac366b7e --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_biology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_biology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school biology.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_high_school_biology" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_chemistry.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..661ea0ca2f72242eb4daf520f6683a9de3a7c32c --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_chemistry.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_high_school_chemistry" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_computer_science.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b271a661f943fdd6d364833c9f994c19ee10cd22 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_computer_science.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_computer_science" +"description": "The following are multiple choice questions (with answers) about high\ + \ school computer science.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_high_school_computer_science" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_european_history.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f1a329ebb24804c92690b5210cb27f6ec47be93d --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_european_history.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_european_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school european history.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_high_school_european_history" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_geography.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe681101f6704e7f058e27350b37838ba63fcd07 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_geography.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_geography" +"description": "The following are multiple choice questions (with answers) about high\ + \ school geography.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_high_school_geography" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_government_and_politics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d8a8f279fc5bfaa8b610f3ff5dcd1c2be0c88e07 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_government_and_politics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_government_and_politics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school government and politics.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_high_school_government_and_politics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_macroeconomics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..45664135facb151e9b6f91347bbc135297880acb --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_macroeconomics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_macroeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school macroeconomics.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_high_school_macroeconomics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_mathematics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..49903260ceff13c03070606e04beb45d99d660f7 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_mathematics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_mathematics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school mathematics.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_high_school_mathematics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_microeconomics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..394c1d77e553a24820ba5db934bfa8fd95a8a269 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_microeconomics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_microeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school microeconomics.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_high_school_microeconomics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_physics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7f32ef2fcc4bf03e34b43c5a3d1135431742db71 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_physics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_physics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school physics.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_high_school_physics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_psychology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a9aac0736a9610469c70b925b70b3f384ca9777 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_psychology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_psychology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school psychology.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_high_school_psychology" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_statistics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e7e02afb94aebb1676c5c395c51e37d4f149a39 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_statistics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_high_school_statistics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_us_history.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7bc84ea9dd78e87166f1e7b67c248d242cb98d83 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_us_history.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_high_school_us_history" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_world_history.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f25cf646bebbca23ed23ea421473e6c2461dda8a --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_world_history.yaml @@ -0,0 +1,6 @@ +"dataset_name": "high_school_world_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school world history.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_high_school_world_history" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_human_aging.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c258f919041775e1d2bf1226264a10b1133802db --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_human_aging.yaml @@ -0,0 +1,6 @@ +"dataset_name": "human_aging" +"description": "The following are multiple choice questions (with answers) about human\ + \ aging.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_other" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_human_aging" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_human_sexuality.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e192a78b48bd37e4dc37efc5783b527f84c3e55 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_human_sexuality.yaml @@ -0,0 +1,6 @@ +"dataset_name": "human_sexuality" +"description": "The following are multiple choice questions (with answers) about human\ + \ sexuality.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_human_sexuality" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_international_law.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..662bf6eb35157889356a6be7ded31d5f6f2a39ac --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_international_law.yaml @@ -0,0 +1,6 @@ +"dataset_name": "international_law" +"description": "The following are multiple choice questions (with answers) about international\ + \ law.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_international_law" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_jurisprudence.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..82036dc1da79c464f21f90b46e4681b061fe5ea1 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_jurisprudence.yaml @@ -0,0 +1,6 @@ +"dataset_name": "jurisprudence" +"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ + \n" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_jurisprudence" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_logical_fallacies.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..346e4b669771f23d7a3a805b329e96e711cd367e --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_logical_fallacies.yaml @@ -0,0 +1,6 @@ +"dataset_name": "logical_fallacies" +"description": "The following are multiple choice questions (with answers) about logical\ + \ fallacies.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_logical_fallacies" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_machine_learning.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d7c280155ae7302b0bed56715c5ea92191e3faf --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_machine_learning.yaml @@ -0,0 +1,6 @@ +"dataset_name": "machine_learning" +"description": "The following are multiple choice questions (with answers) about machine\ + \ learning.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_machine_learning" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_management.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7a732a778fb85eac5467fe2744e51340bce0c302 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_management.yaml @@ -0,0 +1,6 @@ +"dataset_name": "management" +"description": "The following are multiple choice questions (with answers) about management.\n\ + \n" +"tag": "mmlu_flan_n_shot_loglikelihood_other" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_management" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_marketing.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..56760226dba043ba37a110cf7065bbd52c3e9c93 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_marketing.yaml @@ -0,0 +1,6 @@ +"dataset_name": "marketing" +"description": "The following are multiple choice questions (with answers) about marketing.\n\ + \n" +"tag": "mmlu_flan_n_shot_loglikelihood_other" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_marketing" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_medical_genetics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6635c9613155a7c23bf67329b4be950e57fe2d30 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_medical_genetics.yaml @@ -0,0 +1,6 @@ +"dataset_name": "medical_genetics" +"description": "The following are multiple choice questions (with answers) about medical\ + \ genetics.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_other" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_medical_genetics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_miscellaneous.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ce8dff42a80057d6557f81e5aead49b4e93e4ef3 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_miscellaneous.yaml @@ -0,0 +1,6 @@ +"dataset_name": "miscellaneous" +"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ + \n" +"tag": "mmlu_flan_n_shot_loglikelihood_other" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_miscellaneous" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_moral_disputes.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..62460e82f3386022679443efe3c989c2ffb59abf --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_moral_disputes.yaml @@ -0,0 +1,6 @@ +"dataset_name": "moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_moral_disputes" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_moral_scenarios.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..408c69f11630d4c237079e327b1b4c9fe3971dc9 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_moral_scenarios.yaml @@ -0,0 +1,6 @@ +"dataset_name": "moral_scenarios" +"description": "The following are multiple choice questions (with answers) about moral\ + \ scenarios.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_moral_scenarios" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_nutrition.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5494f9dc462494e198dfc7ad86d63a186637bf5c --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_nutrition.yaml @@ -0,0 +1,6 @@ +"dataset_name": "nutrition" +"description": "The following are multiple choice questions (with answers) about nutrition.\n\ + \n" +"tag": "mmlu_flan_n_shot_loglikelihood_other" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_nutrition" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_philosophy.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4fd1f01a1707126eaf93e6a668d681408c8c7fe6 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_philosophy.yaml @@ -0,0 +1,6 @@ +"dataset_name": "philosophy" +"description": "The following are multiple choice questions (with answers) about philosophy.\n\ + \n" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_philosophy" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_prehistory.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1eb08bfbeb44dc8279ab6796e673b3b271517548 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_prehistory.yaml @@ -0,0 +1,6 @@ +"dataset_name": "prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory.\n\ + \n" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_prehistory" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_accounting.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5a23a990afe0abf5b354a15dfec3b5bbd2775fc9 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_accounting.yaml @@ -0,0 +1,6 @@ +"dataset_name": "professional_accounting" +"description": "The following are multiple choice questions (with answers) about professional\ + \ accounting.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_other" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_professional_accounting" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_law.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0a80f2baacbe17445f4a1ea564c7a72174b1c445 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_law.yaml @@ -0,0 +1,6 @@ +"dataset_name": "professional_law" +"description": "The following are multiple choice questions (with answers) about professional\ + \ law.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_professional_law" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_medicine.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..da9e30e118445c2a796eb4145f5c308e9e33215f --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_medicine.yaml @@ -0,0 +1,6 @@ +"dataset_name": "professional_medicine" +"description": "The following are multiple choice questions (with answers) about professional\ + \ medicine.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_other" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_professional_medicine" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_psychology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ce7043a07273fe781cb56733ab02b0b1cb4bf059 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_psychology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_professional_psychology" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_public_relations.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..debace7ca0d0c1dbdfad6ad1621dcc5d9a1469eb --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_public_relations.yaml @@ -0,0 +1,6 @@ +"dataset_name": "public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_public_relations" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_security_studies.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb1f585ce890b5a4fcccc72c3066691509330c49 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_security_studies.yaml @@ -0,0 +1,6 @@ +"dataset_name": "security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_security_studies" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_sociology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0580f7ae31687590598a77fe950d282020d9be16 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_sociology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "sociology" +"description": "The following are multiple choice questions (with answers) about sociology.\n\ + \n" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_sociology" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_us_foreign_policy.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3ff2d9ea791abf7bad56784b804bcadd5c82c077 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_us_foreign_policy.yaml @@ -0,0 +1,6 @@ +"dataset_name": "us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_us_foreign_policy" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_virology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c3edfd9528eed5d4199ebb0ba06a328ed8c50dd8 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_virology.yaml @@ -0,0 +1,6 @@ +"dataset_name": "virology" +"description": "The following are multiple choice questions (with answers) about virology.\n\ + \n" +"tag": "mmlu_flan_n_shot_loglikelihood_other" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_virology" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_world_religions.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..765e70c8fc22dbd75ba495a6490ec788d4e44b7e --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_world_religions.yaml @@ -0,0 +1,6 @@ +"dataset_name": "world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions.\n\n" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" +"include": "_mmlu_flan_loglikelihood_template_yaml" +"task": "mmlu_flan_n_shot_loglikelihood_world_religions" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/_default_template_yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/_default_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..71371402e1a1d74c961492c26b9f08d08d9acd28 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/_default_template_yaml @@ -0,0 +1,29 @@ +dataset_path: hails/mmlu_no_train # a copy of `cais/mmlu` with no auxiliary_train split +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: generate_until +doc_to_text: "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:" +doc_to_target: "{{['A', 'B', 'C', 'D'][answer]}}" +generation_kwargs: + until: + - "" + - "\n" +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_punctuation: true + ignore_case: true +filter_list: + - name: get_response + filter: + # Filter everything after the first letter + - function: "regex" + regex_pattern: "^([A-Z]).*" + - function: take_first +metadata: + version: 3.0 +dataset_kwargs: + trust_remote_code: true diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/_mmlu.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/_mmlu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..550caa37606f975110f5e4f425d27e594014c116 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/_mmlu.yaml @@ -0,0 +1,38 @@ +group: mmlu_generative +group_alias: mmlu (generative) +task: + - group: stem + task: + - mmlu_stem_generative + aggregate_metric_list: + - metric: exact_match + weight_by_size: true + filter_list: get_response + - group: other + task: + - mmlu_other_generative + aggregate_metric_list: + - metric: exact_match + weight_by_size: true + filter_list: get_response + - group: social sciences + task: + - mmlu_social_sciences_generative + aggregate_metric_list: + - metric: exact_match + weight_by_size: true + filter_list: get_response + - group: humanities + task: + - mmlu_humanities_generative + aggregate_metric_list: + - metric: exact_match + weight_by_size: true + filter_list: get_response +aggregate_metric_list: + - aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: get_response +metadata: + version: 3 diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_abstract_algebra.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..17bfcafb79b113cffe93f6e90c68562b7eae7c95 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_abstract_algebra.yaml @@ -0,0 +1,7 @@ +"dataset_name": "abstract_algebra" +"description": "The following are multiple choice questions (with answers) about abstract\ + \ algebra.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_abstract_algebra_generative" +"task_alias": "abstract_algebra" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_anatomy.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..72afc359a495af12d3dcb2b062c6442d92d45c88 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_anatomy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "anatomy" +"description": "The following are multiple choice questions (with answers) about anatomy.\n\ + \n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_anatomy_generative" +"task_alias": "anatomy" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_astronomy.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0b41447e74a2b95732b102bfe5ed642d3d208d2b --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_astronomy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "astronomy" +"description": "The following are multiple choice questions (with answers) about astronomy.\n\ + \n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_astronomy_generative" +"task_alias": "astronomy" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_business_ethics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e7c15d443691af36dcdc761eb41b8673f3782d0b --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_business_ethics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "business_ethics" +"description": "The following are multiple choice questions (with answers) about business\ + \ ethics.\n\n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_business_ethics_generative" +"task_alias": "business_ethics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_clinical_knowledge.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..24cd0b72d3f68fb00da90397979816b85ea1c76c --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_clinical_knowledge.yaml @@ -0,0 +1,7 @@ +"dataset_name": "clinical_knowledge" +"description": "The following are multiple choice questions (with answers) about clinical\ + \ knowledge.\n\n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_clinical_knowledge_generative" +"task_alias": "clinical_knowledge" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_college_biology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ff9cc284007337e30369dd4864b2b723e8e6768 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_college_biology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "college_biology" +"description": "The following are multiple choice questions (with answers) about college\ + \ biology.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_college_biology_generative" +"task_alias": "college_biology" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_college_chemistry.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12d9ce3eab1332fa202cf6f99a52785865aed1a7 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_college_chemistry.yaml @@ -0,0 +1,7 @@ +"dataset_name": "college_chemistry" +"description": "The following are multiple choice questions (with answers) about college\ + \ chemistry.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_college_chemistry_generative" +"task_alias": "college_chemistry" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_college_computer_science.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..73d91c52acd76bf99ce1869296257d25143ad149 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_college_computer_science.yaml @@ -0,0 +1,7 @@ +"dataset_name": "college_computer_science" +"description": "The following are multiple choice questions (with answers) about college\ + \ computer science.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_college_computer_science_generative" +"task_alias": "college_computer_science" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_college_mathematics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..15ae9dded855610af45a15bab8aa56596bfaddd4 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_college_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "college_mathematics" +"description": "The following are multiple choice questions (with answers) about college\ + \ mathematics.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_college_mathematics_generative" +"task_alias": "college_mathematics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_college_medicine.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0461ab7ae7dab9df6b10591fd14791a2cc3eff0f --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_college_medicine.yaml @@ -0,0 +1,7 @@ +"dataset_name": "college_medicine" +"description": "The following are multiple choice questions (with answers) about college\ + \ medicine.\n\n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_college_medicine_generative" +"task_alias": "college_medicine" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_college_physics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0d997d8974c99a549a2216a9bd9237f05a619e21 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_college_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_college_physics_generative" +"task_alias": "college_physics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_computer_security.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ee64d20100e25fc4bcf7f446b1e98acf042c4ab8 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_computer_security.yaml @@ -0,0 +1,7 @@ +"dataset_name": "computer_security" +"description": "The following are multiple choice questions (with answers) about computer\ + \ security.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_computer_security_generative" +"task_alias": "computer_security" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_conceptual_physics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..75764a2cbf542ba09a99ae252c76a103bf534a9f --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_conceptual_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "conceptual_physics" +"description": "The following are multiple choice questions (with answers) about conceptual\ + \ physics.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_conceptual_physics_generative" +"task_alias": "conceptual_physics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_econometrics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..43fec80ad3f505bedb810df609a8c6e8d2c2c0ed --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_econometrics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "econometrics" +"description": "The following are multiple choice questions (with answers) about econometrics.\n\ + \n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_econometrics_generative" +"task_alias": "econometrics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_electrical_engineering.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..130ec2b2aa2210322c1e2f86cdf6be31dd72bffc --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_electrical_engineering.yaml @@ -0,0 +1,7 @@ +"dataset_name": "electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_electrical_engineering_generative" +"task_alias": "electrical_engineering" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_elementary_mathematics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4afd087dc47f27653b54ff48a27a187bc9af07bc --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_elementary_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "elementary_mathematics" +"description": "The following are multiple choice questions (with answers) about elementary\ + \ mathematics.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_elementary_mathematics_generative" +"task_alias": "elementary_mathematics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_formal_logic.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..72c28c0b188b8b8fd69ba9ed79595f0d173f71cf --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_formal_logic.yaml @@ -0,0 +1,7 @@ +"dataset_name": "formal_logic" +"description": "The following are multiple choice questions (with answers) about formal\ + \ logic.\n\n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_formal_logic_generative" +"task_alias": "formal_logic" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_global_facts.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b788025ad5ddf0d859fc12a0d0f139c0975b16ba --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_global_facts.yaml @@ -0,0 +1,7 @@ +"dataset_name": "global_facts" +"description": "The following are multiple choice questions (with answers) about global\ + \ facts.\n\n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_global_facts_generative" +"task_alias": "global_facts" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_biology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3677842dcfc091bb28525889479a48096cbb854d --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_biology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_biology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school biology.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_biology_generative" +"task_alias": "high_school_biology" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_chemistry.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2df93cab2a999a7d6d8e78d3ac9c3ce9aeddcf12 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_chemistry.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_chemistry_generative" +"task_alias": "high_school_chemistry" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_computer_science.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ec5dc7f89abd7ddc57438c71e0502fce1ac47279 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_computer_science.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_computer_science" +"description": "The following are multiple choice questions (with answers) about high\ + \ school computer science.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_computer_science_generative" +"task_alias": "high_school_computer_science" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_european_history.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9732754bbd7352957dbe299494083e17b960c1bc --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_european_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_european_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school european history.\n\n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_european_history_generative" +"task_alias": "high_school_european_history" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_geography.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..66b1a3c97a64f9ee7db414ab13d3146efba5612d --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_geography.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_geography" +"description": "The following are multiple choice questions (with answers) about high\ + \ school geography.\n\n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_geography_generative" +"task_alias": "high_school_geography" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_government_and_politics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..46861fdc1149b72d4ac3f347c0e09f679f6c6e54 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_government_and_politics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_government_and_politics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school government and politics.\n\n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_government_and_politics_generative" +"task_alias": "high_school_government_and_politics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_macroeconomics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ada415922b2b777f153cf387f9095cce9c75304b --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_macroeconomics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_macroeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school macroeconomics.\n\n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_macroeconomics_generative" +"task_alias": "high_school_macroeconomics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_mathematics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b22a5888e61be187f5bbbca1e38171eecd6252d --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_mathematics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school mathematics.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_mathematics_generative" +"task_alias": "high_school_mathematics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_microeconomics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c59ff16270084981614d6f01065851c005039413 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_microeconomics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_microeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school microeconomics.\n\n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_microeconomics_generative" +"task_alias": "high_school_microeconomics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_physics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21d846afb9c8c6b372d59ee462561bb8f67ae83e --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_physics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school physics.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_physics_generative" +"task_alias": "high_school_physics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_psychology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cd1321a5f17efca463edbc6711c197fb18c3a81d --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_psychology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_psychology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school psychology.\n\n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_psychology_generative" +"task_alias": "high_school_psychology" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_statistics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f1442fb8df4168606151af5cc1dfd769bb2e70e3 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_statistics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_statistics_generative" +"task_alias": "high_school_statistics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_us_history.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4552a560f38e3ed5db503fa677548a11766873c2 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_us_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history.\n\n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_us_history_generative" +"task_alias": "high_school_us_history" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_world_history.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d510f22ff39219829e6a9030cb39dc2c43062ca4 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_high_school_world_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "high_school_world_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school world history.\n\n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_high_school_world_history_generative" +"task_alias": "high_school_world_history" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_human_aging.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..56352f4a8c86966853cdbafd68453d1ee85dbabb --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_human_aging.yaml @@ -0,0 +1,7 @@ +"dataset_name": "human_aging" +"description": "The following are multiple choice questions (with answers) about human\ + \ aging.\n\n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_human_aging_generative" +"task_alias": "human_aging" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_human_sexuality.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a23559cfb36a380131573f46b30bbdb5f4656b42 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_human_sexuality.yaml @@ -0,0 +1,7 @@ +"dataset_name": "human_sexuality" +"description": "The following are multiple choice questions (with answers) about human\ + \ sexuality.\n\n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_human_sexuality_generative" +"task_alias": "human_sexuality" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_international_law.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..878df6f3cacb299a51afacca461204fdc4e3a782 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_international_law.yaml @@ -0,0 +1,7 @@ +"dataset_name": "international_law" +"description": "The following are multiple choice questions (with answers) about international\ + \ law.\n\n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_international_law_generative" +"task_alias": "international_law" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_jurisprudence.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5782d81551072a0ff03d79c930f02edb64488f3 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_jurisprudence.yaml @@ -0,0 +1,7 @@ +"dataset_name": "jurisprudence" +"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ + \n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_jurisprudence_generative" +"task_alias": "jurisprudence" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_logical_fallacies.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..43e8e0168b9f4638cc80b76ff1a4edc8893212b4 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_logical_fallacies.yaml @@ -0,0 +1,7 @@ +"dataset_name": "logical_fallacies" +"description": "The following are multiple choice questions (with answers) about logical\ + \ fallacies.\n\n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_logical_fallacies_generative" +"task_alias": "logical_fallacies" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_machine_learning.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d39a4b53164ce8bb641c99fa50f24ace308d3f4 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_machine_learning.yaml @@ -0,0 +1,7 @@ +"dataset_name": "machine_learning" +"description": "The following are multiple choice questions (with answers) about machine\ + \ learning.\n\n" +"tag": "mmlu_stem_generative" +"include": "_default_template_yaml" +"task": "mmlu_machine_learning_generative" +"task_alias": "machine_learning" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_management.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6d51ea0d0aa41fb4b2579162111aa8ebd8ce8f6d --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_management.yaml @@ -0,0 +1,7 @@ +"dataset_name": "management" +"description": "The following are multiple choice questions (with answers) about management.\n\ + \n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_management_generative" +"task_alias": "management" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_marketing.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..744385a2ea524d6f651851856e15aaf190eb847e --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_marketing.yaml @@ -0,0 +1,7 @@ +"dataset_name": "marketing" +"description": "The following are multiple choice questions (with answers) about marketing.\n\ + \n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_marketing_generative" +"task_alias": "marketing" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_medical_genetics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7fea57959818525acdada5bf8a327b0ce96fefb0 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_medical_genetics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "medical_genetics" +"description": "The following are multiple choice questions (with answers) about medical\ + \ genetics.\n\n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_medical_genetics_generative" +"task_alias": "medical_genetics" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_miscellaneous.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e7e0fabc2536d4894526b680deba9a382ff9c3ff --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_miscellaneous.yaml @@ -0,0 +1,7 @@ +"dataset_name": "miscellaneous" +"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ + \n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_miscellaneous_generative" +"task_alias": "miscellaneous" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_moral_disputes.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..61d2feee6a9cf4ed4d71b7c2f9aa68f5219c270a --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_moral_disputes.yaml @@ -0,0 +1,7 @@ +"dataset_name": "moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes.\n\n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_moral_disputes_generative" +"task_alias": "moral_disputes" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_moral_scenarios.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2aeb93f967f0811d3a2f1d886aedfb334a96714e --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_moral_scenarios.yaml @@ -0,0 +1,7 @@ +"dataset_name": "moral_scenarios" +"description": "The following are multiple choice questions (with answers) about moral\ + \ scenarios.\n\n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_moral_scenarios_generative" +"task_alias": "moral_scenarios" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_nutrition.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..638ac8100b6f918ccaa0a3dc13946512d3c97b33 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_nutrition.yaml @@ -0,0 +1,7 @@ +"dataset_name": "nutrition" +"description": "The following are multiple choice questions (with answers) about nutrition.\n\ + \n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_nutrition_generative" +"task_alias": "nutrition" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_philosophy.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..149894b8484cb1fad9ddad1fc5cb2c07a659aea1 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_philosophy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "philosophy" +"description": "The following are multiple choice questions (with answers) about philosophy.\n\ + \n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_philosophy_generative" +"task_alias": "philosophy" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_prehistory.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e130e1baacc3f8a8f558b568336896668e84dd4f --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_prehistory.yaml @@ -0,0 +1,7 @@ +"dataset_name": "prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory.\n\ + \n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_prehistory_generative" +"task_alias": "prehistory" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_professional_accounting.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a46792ec22d84ee3193996653f536084b9ab7861 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_professional_accounting.yaml @@ -0,0 +1,7 @@ +"dataset_name": "professional_accounting" +"description": "The following are multiple choice questions (with answers) about professional\ + \ accounting.\n\n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_professional_accounting_generative" +"task_alias": "professional_accounting" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_professional_law.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f087657e579524b35bf7de4c0f81cb5b697caed4 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_professional_law.yaml @@ -0,0 +1,7 @@ +"dataset_name": "professional_law" +"description": "The following are multiple choice questions (with answers) about professional\ + \ law.\n\n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_professional_law_generative" +"task_alias": "professional_law" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_professional_medicine.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc80878980195f58ac5ae26a0a70589a47b325d5 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_professional_medicine.yaml @@ -0,0 +1,7 @@ +"dataset_name": "professional_medicine" +"description": "The following are multiple choice questions (with answers) about professional\ + \ medicine.\n\n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_professional_medicine_generative" +"task_alias": "professional_medicine" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_professional_psychology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d0b36ccde61e7edc33464a676d4fe0fcc25f3304 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_professional_psychology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology.\n\n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_professional_psychology_generative" +"task_alias": "professional_psychology" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_public_relations.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..37cdccba9b7cebbaa34c5f1e9da01655367477f6 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_public_relations.yaml @@ -0,0 +1,7 @@ +"dataset_name": "public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations.\n\n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_public_relations_generative" +"task_alias": "public_relations" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_security_studies.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..36c235feefd1548320400e7e8d9f3e03f2d478d0 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_security_studies.yaml @@ -0,0 +1,7 @@ +"dataset_name": "security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies.\n\n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_security_studies_generative" +"task_alias": "security_studies" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_sociology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b7e2e592e4457118c9458ccb757b823f9adbb193 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_sociology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "sociology" +"description": "The following are multiple choice questions (with answers) about sociology.\n\ + \n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_sociology_generative" +"task_alias": "sociology" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_us_foreign_policy.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d5fb95366245eae638918270bff4353024195d5f --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_us_foreign_policy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy.\n\n" +"tag": "mmlu_social_sciences_generative" +"include": "_default_template_yaml" +"task": "mmlu_us_foreign_policy_generative" +"task_alias": "us_foreign_policy" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_virology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9954dc182f1bbd5030b94d2a08b2ddf4a135a6cf --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_virology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "virology" +"description": "The following are multiple choice questions (with answers) about virology.\n\ + \n" +"tag": "mmlu_other_generative" +"include": "_default_template_yaml" +"task": "mmlu_virology_generative" +"task_alias": "virology" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_world_religions.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1db5128b43e615d0fc41f9c7448db3b5ea39942c --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu/generative/mmlu_world_religions.yaml @@ -0,0 +1,7 @@ +"dataset_name": "world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions.\n\n" +"tag": "mmlu_humanities_generative" +"include": "_default_template_yaml" +"task": "mmlu_world_religions_generative" +"task_alias": "world_religions" diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/README.md b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/README.md new file mode 100644 index 0000000000000000000000000000000000000000..8fddbef266f755ae611c38b6ec2aea05ff6aa033 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/README.md @@ -0,0 +1,64 @@ +# mmlu_pro + +### Paper + +Title: `MMLU-Pro: A More Robust and Challenging Multi-Task Language Understanding Benchmark` + +Abstract: `In the age of large-scale language models, benchmarks like the Massive Multitask Language Understanding (MMLU) have been pivotal in pushing the boundaries of what AI can achieve in language comprehension and reasoning across diverse domains. However, as models continue to improve, their performance on these benchmarks has begun to plateau, making it increasingly difficult to discern differences in model capabilities. This paper introduces MMLU-Pro, an enhanced dataset designed to extend the mostly knowledge-driven MMLU benchmark by integrating more challenging, reasoning-focused questions and expanding the choice set from four to ten options. Additionally, MMLU-Pro eliminates the trivial and noisy questions in MMLU. Our experimental results show that MMLU-Pro not only raises the challenge, causing a significant drop in accuracy by 16% to 33% compared to MMLU but also demonstrates greater stability under varying prompts. With 24 different prompt styles tested, the sensitivity of model scores to prompt variations decreased from 4-5% in MMLU to just 2% in MMLU-Pro. Additionally, we found that models utilizing Chain of Thought (CoT) reasoning achieved better performance on MMLU-Pro compared to direct answering, which is in stark contrast to the findings on the original MMLU, indicating that MMLU-Pro includes more complex reasoning questions. Our assessments confirm that MMLU-Pro is a more discriminative benchmark to better track progress in the field.` + +Homepage: https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro + +### Citation + +```bibtex +@misc{wang2024mmlupro, + title={MMLU-Pro: A More Robust and Challenging Multi-Task Language Understanding Benchmark}, + author={Yubo Wang and Xueguang Ma and Ge Zhang and Yuansheng Ni and Abhranil Chandra and Shiguang Guo and Weiming Ren and Aaran Arulraj and Xuan He and Ziyan Jiang and Tianle Li and Max Ku and Kai Wang and Alex Zhuang and Rongqi Fan and Xiang Yue and Wenhu Chen}, + year={2024}, + eprint={2406.01574}, + archivePrefix={arXiv}, + primaryClass={id='cs.CL' full_name='Computation and Language' is_active=True alt_name='cmp-lg' in_archive='cs' is_general=False description='Covers natural language processing. Roughly includes material in ACM Subject Class I.2.7. Note that work on artificial languages (programming languages, logics, formal systems) that does not explicitly address natural-language issues broadly construed (natural-language processing, computational linguistics, speech, text retrieval, etc.) is not appropriate for this area.'} +} +``` + +### Groups and Tasks + +#### Groups + +* `mmlu_pro`: 'All 14 subjects of the mmlu_pro dataset, evaluated following the methodology in mmlu's original implementation' + +#### Tasks + +The following tasks evaluate subjects in the mmlu_pro dataset +- `mmlu_pro_biology` +- `mmlu_pro_business` +- `mmlu_pro_chemistry` +- `mmlu_pro_computer_science` +- `mmlu_pro_economics` +- `mmlu_pro_engineering` +- `mmlu_pro_health` +- `mmlu_pro_history` +- `mmlu_pro_law` +- `mmlu_pro_math` +- `mmlu_pro_other` +- `mmlu_pro_philosophy` +- `mmlu_pro_physics` +- `mmlu_pro_psychology` + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? + +### Changelog + +* (tasks, group) 2024-09-23 -- (version 1 --> version 2) + * Added one newline to task description(s) as per [reference implementation](https://github.com/TIGER-AI-Lab/MMLU-Pro/blob/47b9891aacb8bd7cda29d5c5ba17b9434dd333bc/evaluate_from_local.py#L93) diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/_default_template_yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/_default_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..9c4f44b5c96769d70215403b06bccab33f1bbfb7 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/_default_template_yaml @@ -0,0 +1,33 @@ +dataset_path: TIGER-Lab/MMLU-Pro +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'answer is \(?([ABCDEFGHIJ])\)?' + # regex_pattern: r".*[aA]nswer:\s*([A-J])", + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/_mmlu_pro.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/_mmlu_pro.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fc3204127604d6eac759299f77d63ce9ef49d24e --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/_mmlu_pro.yaml @@ -0,0 +1,23 @@ +group: mmlu_pro +task: + - mmlu_pro_biology + - mmlu_pro_business + - mmlu_pro_chemistry + - mmlu_pro_computer_science + - mmlu_pro_economics + - mmlu_pro_engineering + - mmlu_pro_health + - mmlu_pro_history + - mmlu_pro_law + - mmlu_pro_math + - mmlu_pro_other + - mmlu_pro_philosophy + - mmlu_pro_physics + - mmlu_pro_psychology +aggregate_metric_list: + - aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 2.0 diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_biology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..80aee85108ce8ac9452c0d0d07341e05b11b0b7a --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_biology.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about biology. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_biology" +task_alias: "biology" +process_docs: !function utils.process_biology diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_business.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..daf871f6bb5abd614c2058c0552389d41cbced50 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_business.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about business. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_business" +task_alias: "business" +process_docs: !function utils.process_business diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_chemistry.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5baf354ec202e66647bacdc9e9008617cd2d4244 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_chemistry.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about chemistry. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_chemistry" +task_alias: "chemistry" +process_docs: !function utils.process_chemistry diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_computer_science.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7de347373e7b92d8eb9ba33bbf8ec4fb2a3dbc49 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_computer_science.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about computer science. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_computer_science" +task_alias: "computer_science" +process_docs: !function utils.process_computer_science diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_economics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..274612783fb749aad10ec67698c3abf5dce13ebf --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_economics.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about economics. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_economics" +task_alias: "economics" +process_docs: !function utils.process_economics diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_engineering.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dcf02f5029823ab83c1cebfbbf9819ac5ec4f1d0 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_engineering.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about engineering. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_engineering" +task_alias: "engineering" +process_docs: !function utils.process_engineering diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_health.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d161d1d81a29a06a9425bafc9e86211a2669b9be --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_health.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about health. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_health" +task_alias: "health" +process_docs: !function utils.process_health diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_history.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d28efd3e77ef31dc2df96847c90e7c76920007bc --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_history.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about history. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_history" +task_alias: "history" +process_docs: !function utils.process_history diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_law.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba99f16dbb8df07ab9272d8b47d23839a919bc9c --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_law.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about law. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_law" +task_alias: "law" +process_docs: !function utils.process_law diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_math.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0526f11f975f556f94c21acbc7074c0b4b76dd42 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_math.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about math. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_math" +task_alias: "math" +process_docs: !function utils.process_math diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_other.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..beb2ec9da7011dd1437828fc442ad21733cdf1a8 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_other.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about other. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_other" +task_alias: "other" +process_docs: !function utils.process_other diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_philosophy.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..99e5d65b4c0af9a2a35589ab104cb07be100cabc --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_philosophy.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about philosophy. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_philosophy" +task_alias: "philosophy" +process_docs: !function utils.process_philosophy diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_physics.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7e7fa740bd58ebd55382bd8464abd9abaee3d96e --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_physics.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about physics. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_physics" +task_alias: "physics" +process_docs: !function utils.process_physics diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_psychology.yaml b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b28fb72d329d4d9755e45201275604d257851754 --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/mmlu_pro_psychology.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about psychology. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_psychology" +task_alias: "psychology" +process_docs: !function utils.process_psychology diff --git a/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/utils.py b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..03117be5f165fd7edf40404bf9934b3753039f1d --- /dev/null +++ b/Prism/Dream/Dream_Prism/eval_instruct/lm_eval/tasks/mmlu_pro/utils.py @@ -0,0 +1,63 @@ +from functools import partial + + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + + +def format_cot_example(example, including_answer=True): + prompt = "Question:\n" + question = example["question"] + options = example["options"] + prompt += question + "\n" + prompt += "Options:\n" + for i, opt in enumerate(options): + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace( + "A: Let's think step by step.", "Answer: Let's think step by step." + ) + prompt += cot_content + "\n\n" + else: + prompt += "Answer: Let's think step by step." + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/Prism/Dream/Dream_Prism/src/diffllm/__init__.py b/Prism/Dream/Dream_Prism/src/diffllm/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/Dream/Dream_Prism/src/diffllm/gen_utils.py b/Prism/Dream/Dream_Prism/src/diffllm/gen_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..6e572f8cdabb75ced4b104df2e0dec071b234b77 --- /dev/null +++ b/Prism/Dream/Dream_Prism/src/diffllm/gen_utils.py @@ -0,0 +1,110 @@ +import torch +import torch.nn.functional as F + + +def q_sample( + input_ids, + maskable_mask, + mask_token_id, + min=0.0, + max=1.0, + eos_token_id=None, + t=None, + t_mask=None, +): + x_0 = input_ids + + if t_mask is None: + if t is None: + t = torch.rand((x_0.shape[0],), dtype=torch.float, device=input_ids.device) + t = min + (max - min) * t + u = torch.rand_like(x_0, dtype=torch.float) # t/T prob to mask + t_mask = (u < t[:, None]) & maskable_mask + + x_t = x_0.masked_fill(t_mask, mask_token_id) + + if eos_token_id is not None: + # get the last non-eos token index + last_non_eos_token_idx = ((input_ids != eos_token_id) | (~maskable_mask)).sum( + dim=-1 + ) - 1 + seq_len = x_0.shape[1] + + for i in range(x_0.shape[0]): + if last_non_eos_token_idx[i] < seq_len - 1: # with eos tokens + t_mask_at_eos = t_mask[ + i, last_non_eos_token_idx[i] + 1 + ] # use arbitrary eos token + # t_mask[i, last_non_eos_token_idx[i] + 2:] = False # only learn the first eos token + if t_mask_at_eos: + x_t[i, last_non_eos_token_idx[i] + 1 :] = mask_token_id + t_mask[i, last_non_eos_token_idx[i] + 1 :] = True + else: + x_t[i, last_non_eos_token_idx[i] + 1 :] = eos_token_id + t_mask[i, last_non_eos_token_idx[i] + 1 :] = False + + return x_t, t, t_mask # True means it's "MASK" token and should have loss + + +def top_p_logits(logits, top_p=None): + sorted_logits, sorted_indices = torch.sort(logits, descending=True) + cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1) + sorted_indices_to_remove = cumulative_probs > top_p + # Shift the indices to the right to keep the first token above the threshold + sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone() + sorted_indices_to_remove[..., 0] = 0 + + mask = torch.zeros_like(logits, dtype=torch.bool, device=logits.device) + mask = mask.scatter_(-1, sorted_indices, sorted_indices_to_remove) + logits = logits.masked_fill(mask, torch.finfo(logits.dtype).min) + return logits + + +def top_k_logits(logits, top_k=None): + top_k = min(top_k, logits.size(-1)) # Safety check + # Remove all tokens with a probability less than the last token of the top-k + indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, None] + logits = logits.masked_fill(indices_to_remove, torch.finfo(logits.dtype).min) + return logits + + +def sample_tokens( + logits, + temperature=0.0, + top_p=None, + top_k=None, + margin_confidence=False, + neg_entropy=False, +): + + if temperature > 0: + logits = logits / temperature + if top_p is not None and top_p < 1: + logits = top_p_logits(logits, top_p) + if top_k is not None: + logits = top_k_logits(logits, top_k) + probs = torch.softmax(logits, dim=-1) + + if temperature > 0: + try: + x0 = torch.multinomial(probs, num_samples=1).squeeze(-1) + confidence = torch.gather(probs, -1, x0.unsqueeze(-1)).squeeze(-1) + except: + confidence, x0 = probs.max(dim=-1) + else: + confidence, x0 = probs.max(dim=-1) + + if margin_confidence: + sorted_probs, _ = torch.sort(probs, dim=-1, descending=True) + # Extract top1 and top2 probabilities + top1_probs = sorted_probs[:, 0] + top2_probs = sorted_probs[:, 1] + # Calculate confidence as top1 - top2 + confidence = top1_probs - top2_probs + + if neg_entropy: + epsilon = 1e-10 + log_probs = torch.log(probs + epsilon) + confidence = torch.sum(probs * log_probs, dim=-1) + + return confidence, x0 diff --git a/Prism/Dream/Dream_Prism/src/trainer/__init__.py b/Prism/Dream/Dream_Prism/src/trainer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/Dream/Dream_Prism/src/trainer/config/sft_trainer.yaml b/Prism/Dream/Dream_Prism/src/trainer/config/sft_trainer.yaml new file mode 100644 index 0000000000000000000000000000000000000000..72b1d49d5d38a1afa5fc6eb82a4cb85e1aab737a --- /dev/null +++ b/Prism/Dream/Dream_Prism/src/trainer/config/sft_trainer.yaml @@ -0,0 +1,57 @@ +data: + perbatch_cutoff: False + perbatch_cutoff_type: random + collate_fn_type: optimized + train_batch_size: 256 + micro_batch_size_per_gpu: 4 # this is also val batch size + train_files: ~/data/gsm8k/train.parquet + val_files: ~/data/gsm8k/test.parquet + tokenized: False + prompt_key: question + response_key: answer + max_length: 1024 + truncation: error + chat_template: null + resp_cutoff_ratio: 0.0 + pad_token_id: null +model: + partial_pretrain: ~/models/gemma-1.1-7b-it + fsdp_config: + wrap_policy: + min_num_params: 0 + cpu_offload: False + offload_params: False + external_lib: null + enable_gradient_checkpointing: False + trust_remote_code: False + lora_rank: 0 # Set to positive value to enable LoRA (e.g., 32) + lora_alpha: 16 # LoRA scaling factor + target_modules: all-linear # Target modules for LoRA adaptation + use_liger: False + attention_dropout: 0.0 +optim: + lr: 1e-5 + betas: [0.9, 0.95] + weight_decay: 0.01 + warmup_steps_ratio: 0.1 + clip_grad: 1.0 +ulysses_sequence_parallel_size: 1 +use_remove_padding: False +trainer: + default_local_dir: /tmp/sft_model + default_hdfs_dir: hdfs://tmp/experiments/gsm8k/gemma-1.1-7b-it/ # change the hdfs path here + resume_path: null + resume_training: False + project_name: gsm8k-sft + experiment_name: test + total_epochs: 3 + total_training_steps: null + logger: ['console'] + seed: 1 + save_checkpoint_steps: 1000 +diffusion: + token_reweighting: false # use focal loss for token-level reweighting + alpha: 0.25 # for focal loss + gamma: 2 # for focal loss + time_reweighting: original # time-level reweighting strategy + cart_p: 0.1 diff --git a/Prism/Dream/Dream_Prism/src/trainer/fsdp_sft_trainer.py b/Prism/Dream/Dream_Prism/src/trainer/fsdp_sft_trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..a46307dbf8bfb2a5082aa34c89ba76cd6b095ea1 --- /dev/null +++ b/Prism/Dream/Dream_Prism/src/trainer/fsdp_sft_trainer.py @@ -0,0 +1,1159 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +A lightweight one-file FSDP SFT Trainer +TODO(zhangchi.usc1992) +- Add calculation of mfu +- Add validation +""" + +import os + +os.environ["NCCL_DEBUG"] = "WARN" +os.environ["TOKENIZERS_PARALLELISM"] = "true" + +import logging +import math +import random +import re +from contextlib import nullcontext +from typing import List + +import hydra +import numpy as np +import torch +import torch.distributed +import verl.utils.hdfs_io as hdfs_io +from peft import LoraConfig, TaskType, get_peft_model +from tensordict import TensorDict +from torch import nn, optim +from torch.distributed.device_mesh import DeviceMesh, init_device_mesh +from torch.distributed.fsdp import CPUOffload +from torch.distributed.fsdp import FullyShardedDataParallel as FSDP +from torch.distributed.fsdp import MixedPrecision, ShardingStrategy +from torch.utils.data import DataLoader, DistributedSampler +from tqdm import tqdm +from transformers import AutoConfig, AutoModel, PreTrainedModel +from verl.trainer.fsdp_sft_trainer import FSDPSFTTrainer +from verl.utils.debug import log_gpu_memory_usage +from verl.utils.distributed import initialize_global_process_group +from verl.utils.fs import copy_local_path_from_hdfs +from verl.utils.fsdp_utils import ( + get_fsdp_wrap_policy, + get_init_weight_context_manager, + init_fn, +) +from verl.utils.model import compute_position_id_with_mask +from verl.utils.torch_functional import get_cosine_schedule_with_warmup +from verl.utils.tracking import Tracking +from verl.workers.sharding_manager import FSDPUlyssesShardingManager + +from src.diffllm.gen_utils import q_sample +from src.trainer.sft_dataset import SFTDataset, TokenizedSFTDataset + +logger = logging.getLogger(__file__) +logger.setLevel(os.getenv("VERL_SFT_LOGGING_LEVEL", "WARN")) + + +def extract_step(path): + match = re.search(r"global_step_(\d+)", path) + if match: + return int(match.group(1)) + return None + + +def convert_to_regular_types(obj): + """Convert Hydra configs and other special types to regular Python types.""" + from omegaconf import DictConfig, ListConfig + + if isinstance(obj, (ListConfig, DictConfig)): + return ( + {k: convert_to_regular_types(v) for k, v in obj.items()} + if isinstance(obj, DictConfig) + else list(obj) + ) + elif isinstance(obj, (list, tuple)): + return [convert_to_regular_types(x) for x in obj] + elif isinstance(obj, dict): + return {k: convert_to_regular_types(v) for k, v in obj.items()} + return obj + + +def context_adaptive_reweight(seq_len, distribution="symmetric-geometric", **kwargs): + position_ids_l = np.arange(seq_len).reshape(-1, 1) + position_ids_r = np.arange(seq_len).reshape(1, -1) + distance = position_ids_l - position_ids_r + distance = torch.from_numpy(distance) + + def geometric_distribution(k, cart_p=0.8, **kwargs): + if not 0 < cart_p <= 1: + raise ValueError("p must be between 0 and 1") + + res = (math.log(cart_p) + (k.abs() - 1) * math.log(1 - cart_p)).exp() * 0.5 + res.masked_fill_(k == 0, 0) # ignore distance=0 + return res + + if distribution == "symmetric-geometric": + matrix = geometric_distribution(distance, **kwargs) + else: + raise ValueError(f"Unknown distribution {distribution}") + + return matrix + + +class OptimizedCollateFunction: + """ + Optimized collate function that completes preprocessing during data loading + Reduces GPU computation overhead and improves training efficiency + """ + + def __init__(self, config, tokenizer): + self.config = config + self.tokenizer = tokenizer + self.pad_eos_token_id = ( + config.data.pad_token_id + if config.data.pad_token_id is not None + else tokenizer.pad_token_id + ) + + # Cache configuration items to avoid repeated access + self.enable_perbatch_cutoff = getattr(config.data, "perbatch_cutoff", False) + self.perbatch_cutoff_type = getattr(config.data, "perbatch_cutoff_type", None) + self.resp_cutoff_ratio = getattr(config.data, "resp_cutoff_ratio", 0.0) + + self.random = random.Random(42) + self.np_random = np.random.RandomState(42) + + def __call__(self, batch_samples): + """ + Efficient collate function implementation + Args: + batch_samples: List of samples from dataset + Returns: + Preprocessed batch dict + """ + # Use default collate to merge samples into batch tensors + from torch.utils.data.dataloader import default_collate + + batch = default_collate(batch_samples) + + # Extract tensors + input_ids = batch["input_ids"] + attention_mask = batch["attention_mask"].bool() + position_ids = batch["position_ids"] + loss_mask = batch["loss_mask"].bool() + + # 1. Handle perbatch_cutoff related logic + if self.enable_perbatch_cutoff: + input_ids, attention_mask, position_ids, loss_mask = ( + self._apply_perbatch_cutoff( + input_ids, attention_mask, position_ids, loss_mask + ) + ) + + # 2. Handle response truncation + if self.resp_cutoff_ratio > 0.0: + input_ids, attention_mask, position_ids, loss_mask = ( + self._apply_resp_cutoff( + input_ids, attention_mask, position_ids, loss_mask + ) + ) + + return { + "input_ids": input_ids, + "attention_mask": attention_mask, + "position_ids": position_ids, + "loss_mask": loss_mask, + } + + def _apply_perbatch_cutoff( + self, input_ids, attention_mask, position_ids, loss_mask + ): + """Apply perbatch cutoff logic""" + if self.perbatch_cutoff_type == "random": + non_pad_lens = (input_ids != self.pad_eos_token_id).sum(-1).cpu() + + # randomly choose a cutoff length from non_pad_lens + cutoff_seq_len = np.random.choice(non_pad_lens) + + # cutoff + input_ids = input_ids[:, :cutoff_seq_len] + attention_mask = attention_mask[:, :cutoff_seq_len] + position_ids = position_ids[:, :cutoff_seq_len] + loss_mask = loss_mask[:, :cutoff_seq_len] + elif self.perbatch_cutoff_type == "random_with_input_pad": + prompt_mask = loss_mask == 0 + response_mask = (loss_mask == 1) & (input_ids != self.pad_eos_token_id) + + prompt_lens = prompt_mask.sum(-1) + response_lens = response_mask.sum(-1) + max_prompt_len = prompt_lens.max() + pad_lens = max_prompt_len - prompt_lens + + # randomly choose a response length from response_lens + kept_response_len = np.random.choice(response_lens.cpu()) + + # rebuild input_ids, attention_mask, loss_mask + new_input_ids = ( + torch.ones( + input_ids.shape[0], + max_prompt_len + kept_response_len, + dtype=input_ids.dtype, + device=input_ids.device, + ) + * self.pad_eos_token_id + ) + new_attention_mask = torch.ones_like( + new_input_ids, dtype=attention_mask.dtype + ) + new_loss_mask = torch.ones_like(new_input_ids, dtype=loss_mask.dtype) + + for i in range(input_ids.shape[0]): + kept_response_len_i = min(kept_response_len, response_lens[i]) + new_input_ids[i, pad_lens[i] : pad_lens[i] + prompt_lens[i]] = ( + input_ids[i][prompt_mask[i]] + ) + new_input_ids[ + i, + pad_lens[i] + + prompt_lens[i] : pad_lens[i] + + prompt_lens[i] + + kept_response_len_i, + ] = input_ids[i][response_mask[i]][:kept_response_len_i] + + new_attention_mask[i, : pad_lens[i]] = 0 + new_loss_mask[i, : pad_lens[i] + prompt_lens[i]] = 0 + + input_ids = new_input_ids + attention_mask = new_attention_mask + position_ids = compute_position_id_with_mask(new_attention_mask) + loss_mask = new_loss_mask + else: + pad_lens = (input_ids == self.pad_eos_token_id).sum(-1) + # cutoff_len = eos_lens.min() - 1 + # assert cutoff_len > 0, input_ids + cutoff_len = pad_lens.min() + assert cutoff_len >= 0 + + # cutoff + seq_len = input_ids.shape[-1] + input_ids = input_ids[:, : seq_len - cutoff_len] + attention_mask = attention_mask[:, : seq_len - cutoff_len] + position_ids = position_ids[:, : seq_len - cutoff_len] + loss_mask = loss_mask[:, : seq_len - cutoff_len] + + return input_ids, attention_mask, position_ids, loss_mask + + def _apply_resp_cutoff(self, input_ids, attention_mask, position_ids, loss_mask): + """Apply response truncation logic""" + import numpy as np + + if self.np_random.rand() < self.resp_cutoff_ratio: + # Calculate response length for each sample (loss_mask True portion) + resp_lens = loss_mask.sum(-1) + min_resp_len = resp_lens.min().item() + + if min_resp_len > 1: + # Randomly select truncation length + cutoff_len = self.np_random.randint(1, min_resp_len) + + # Truncate from the end of sequence + new_seq_len = input_ids.shape[-1] - cutoff_len + input_ids = input_ids[:, :new_seq_len].contiguous() + attention_mask = attention_mask[:, :new_seq_len].contiguous() + position_ids = position_ids[:, :new_seq_len].contiguous() + loss_mask = loss_mask[:, :new_seq_len].contiguous() + + return input_ids, attention_mask, position_ids, loss_mask + + +class StreamingCollateFunction(OptimizedCollateFunction): + """ + Streaming version of collate function + Suitable for large batches or long sequences, reduces memory peaks + """ + + def __call__(self, batch_samples): + """ + Streaming version, processes samples one by one then concatenates + """ + if len(batch_samples) == 0: + return {} + + # Process samples one by one to reduce memory usage + processed_samples = [] + for sample in batch_samples: + processed_sample = self._preprocess_single_sample(sample) + processed_samples.append(processed_sample) + + # Concatenate all processed samples + from torch.utils.data.dataloader import default_collate + + batch = default_collate(processed_samples) + + # Apply batch-level preprocessing (inherited from parent class logic) + return self._apply_batch_preprocessing(batch) + + def _preprocess_single_sample(self, sample): + """Preprocess single sample""" + # Sample-level preprocessing can be implemented here + # e.g., dynamic truncation, special token processing, etc. + return sample + + def _apply_batch_preprocessing(self, batch): + """Apply batch-level preprocessing logic""" + # Extract tensors + input_ids = batch["input_ids"] + attention_mask = batch["attention_mask"].bool() + position_ids = batch["position_ids"] + loss_mask = batch["loss_mask"].bool() + + if self.enable_perbatch_cutoff: + input_ids, attention_mask, position_ids, loss_mask = ( + self._apply_perbatch_cutoff( + input_ids, attention_mask, position_ids, loss_mask + ) + ) + + if self.resp_cutoff_ratio > 0.0: + input_ids, attention_mask, position_ids, loss_mask = ( + self._apply_resp_cutoff( + input_ids, attention_mask, position_ids, loss_mask + ) + ) + + return { + "input_ids": input_ids, + "attention_mask": attention_mask, + "position_ids": position_ids, + "loss_mask": loss_mask, + } + + +class FSDPSFTTrainer(object): + + def __init__( + self, config, device_mesh: DeviceMesh, ulysses_device_mesh: DeviceMesh + ): + self.config = config + self.device_mesh = device_mesh + self.ulysses_device_mesh = ulysses_device_mesh + self.sharding_manager = FSDPUlyssesShardingManager(self.ulysses_device_mesh) + + # Add tracking for current epoch + self.current_epoch = 0 + + # Check if resuming training + self.resume_training = getattr(self.config.trainer, "resume_training", False) + self.resume_checkpoint_path = getattr(self.config.trainer, "resume_path", None) + + # build tokenizer first + if self.resume_training and self.resume_checkpoint_path: + # If resuming from specific checkpoint, use that path for tokenizer + local_model_path = copy_local_path_from_hdfs( + src=self.resume_checkpoint_path, verbose=True + ) + else: + local_model_path = copy_local_path_from_hdfs( + src=self.config.model.partial_pretrain, verbose=True + ) + + from verl.utils import hf_tokenizer + + self.tokenizer = hf_tokenizer( + local_model_path, trust_remote_code=self.config.model.trust_remote_code + ) + if self.config.data.chat_template is not None: + raise ValueError("Apply Chat template from config is not supported yet.") + + # normalize dp size + self._normalize_config_bsz() + + # Set sequence parallel size + self.config.ulysses_sequence_parallel_size = getattr( + self.config, "ulysses_sequence_parallel_size", 1 + ) + self.use_remove_padding = getattr(self.config, "use_remove_padding", False) + if self.device_mesh.get_rank() == 0: + print( + f"Using sequence parallel size: {self.config.ulysses_sequence_parallel_size}" + ) + print(f"Using remove padding: {self.use_remove_padding}") + + self._build_dataloader() + # build model + self._build_model_optimizer() + + # TODO: add checkpoint manager + if self.device_mesh.get_rank() == 0: + print(self.config) + + def _normalize_config_bsz(self): + dp_size = ( + self.device_mesh.size(0) + if not self.ulysses_device_mesh + else self.ulysses_device_mesh.size(0) + ) + if self.device_mesh.get_rank() == 0: + print(f"Normalize batch size by dp {dp_size}") + + assert ( + self.config.data.train_batch_size % dp_size == 0 + ), f"Global batch size {self.config.data.train_batch_size} is not divisible by dp size {dp_size}" + + self.config.data.train_batch_size //= dp_size + + assert ( + self.config.data.train_batch_size + % self.config.data.micro_batch_size_per_gpu + == 0 + ) + + def _build_dataloader(self): + config = self.config + # build dataset + if config.data.tokenized: + self.train_dataset = TokenizedSFTDataset( + parquet_files=config.data.train_files, + ) + self.val_dataset = TokenizedSFTDataset( + parquet_files=config.data.val_files, + ) + else: + self.train_dataset = SFTDataset( + parquet_files=config.data.train_files, + tokenizer=self.tokenizer, + prompt_key=config.data.prompt_key, + response_key=config.data.response_key, + max_length=config.data.max_length, + truncation=config.data.truncation, + pad_token_id=config.data.pad_token_id, + ) + self.val_dataset = SFTDataset( + parquet_files=config.data.val_files, + tokenizer=self.tokenizer, + prompt_key=config.data.prompt_key, + response_key=config.data.response_key, + max_length=config.data.max_length, + truncation=config.data.truncation, + pad_token_id=config.data.pad_token_id, + ) + + # build dataloader + # Use data parallel rank and size instead of global rank and world size + + # If doing SP, we need to use the local rank and size + if self.config.ulysses_sequence_parallel_size > 1: + rank = self.ulysses_device_mesh.get_local_rank("dp") + world_size = self.ulysses_device_mesh.size(0) + if self.ulysses_device_mesh.get_rank() == 0: + print( + f"Using SP rank {rank} and size {world_size} for data distribution" + ) + print( + f"Each SP rank gets different data, but the same data WITHIN the same rank" + ) + else: + rank = self.device_mesh.get_rank() + world_size = self.device_mesh.size() + if self.device_mesh.get_rank() == 0: + print(f"Using FSDP rank {rank} and size {world_size} for data distribution") + + self.train_sampler = DistributedSampler( + self.train_dataset, + shuffle=True, + num_replicas=world_size, + rank=rank, + drop_last=True, + ) + + # Create optimized collate function + collate_fn_type = getattr(config.data, "collate_fn_type", "optimized") + if collate_fn_type == "streaming": + train_collate_fn = StreamingCollateFunction(config, self.tokenizer) + val_collate_fn = StreamingCollateFunction(config, self.tokenizer) + else: + train_collate_fn = OptimizedCollateFunction(config, self.tokenizer) + val_collate_fn = OptimizedCollateFunction(config, self.tokenizer) + + if self.device_mesh.get_rank() == 0: + print(f"Using {collate_fn_type} collate function for data preprocessing") + + self.train_dataloader = DataLoader( + dataset=self.train_dataset, + batch_size=config.data.train_batch_size, + sampler=self.train_sampler, + num_workers=8, + pin_memory=True, + drop_last=True, + prefetch_factor=4, + persistent_workers=True, + collate_fn=train_collate_fn, # Use optimized collate function + ) + + self.val_sampler = DistributedSampler( + self.val_dataset, + shuffle=True, + num_replicas=world_size, + rank=rank, + drop_last=True, + ) + self.val_dataloader = DataLoader( + dataset=self.val_dataset, + batch_size=config.data.micro_batch_size_per_gpu, + sampler=self.val_sampler, + num_workers=8, + pin_memory=True, + drop_last=True, + prefetch_factor=4, + persistent_workers=True, + collate_fn=val_collate_fn, # Use optimized collate function + ) + + def _build_model_optimizer(self, checkpoint_path=None): + """Build model and optimizer, optionally from a checkpoint.""" + # Determine which path to load from + if checkpoint_path: + local_model_path = checkpoint_path + else: + local_model_path = copy_local_path_from_hdfs( + src=self.config.model.partial_pretrain, verbose=True + ) + + if self.config.model.get("external_lib", None) is not None: + # This is used to import external_lib into the huggingface systems + import importlib + + importlib.import_module(self.config.model.external_lib) + + log_gpu_memory_usage("Before model allocation", logger=logger) + + trust_remote_code = self.config.model.trust_remote_code + # load config first + config = AutoConfig.from_pretrained( + local_model_path, + trust_remote_code=trust_remote_code, + attention_dropout=self.config.model.attention_dropout, + ) + if self.config.ulysses_sequence_parallel_size > 1: + assert ( + self.use_remove_padding + ), "Sequence parallel is only supported when remove_padding is enabled" + from verl.models.registry import check_model_support_rmpad + + check_model_support_rmpad(config.model_type) + + if self.use_remove_padding and self.config.ulysses_sequence_parallel_size > 1: + from verl.models.transformers.monkey_patch import apply_monkey_patch + + apply_monkey_patch(config, verbose=True) + + # This may be very large + init_context = get_init_weight_context_manager( + use_meta_tensor=not config.tie_word_embeddings + ) + + with init_context(): + self.model: PreTrainedModel = AutoModel.from_pretrained( + local_model_path, + config=config, + torch_dtype=torch.float32, + attn_implementation="flash_attention_2", + trust_remote_code=trust_remote_code, + ) + + # Apply Liger kernel if use_liger is enabled + if self.config.model.get("use_liger", False): + from liger_kernel.transformers.monkey_patch import ( + _apply_liger_kernel_to_instance, + ) + + _apply_liger_kernel_to_instance(model=self.model) + + if self.config.model.get("lora_rank", 0) > 0: + self.model.enable_input_require_grads() + # Convert config to regular Python types before creating PEFT model + lora_config = { + "task_type": TaskType.CAUSAL_LM, + "r": self.config.model.lora_rank, + "lora_alpha": self.config.model.lora_alpha, + "target_modules": convert_to_regular_types( + self.config.model.target_modules + ), + "bias": "none", + } + self.model = get_peft_model(self.model, LoraConfig(**lora_config)) + + if self.config.model.enable_gradient_checkpointing: + self.model.gradient_checkpointing_enable( + gradient_checkpointing_kwargs={"use_reentrant": False} + ) + + log_gpu_memory_usage("After model allocation", logger=logger) + + mixed_precision = MixedPrecision( + param_dtype=torch.bfloat16, + reduce_dtype=torch.float32, + buffer_dtype=torch.float32, + ) + + auto_wrap_policy = get_fsdp_wrap_policy( + self.model, + config=self.config.model.fsdp_config.wrap_policy, + is_lora=self.config.model.get("lora_rank", 0) > 0, + ) + if self.device_mesh.get_rank() == 0: + print(auto_wrap_policy) + + if not self.config.model.fsdp_config.cpu_offload: + cpu_offload = None + else: + cpu_offload = CPUOffload( + offload_params=self.config.model.fsdp_config.offload_params + ) + + self.fsdp_model = FSDP( + module=self.model, + auto_wrap_policy=auto_wrap_policy, + param_init_fn=init_fn, + sharding_strategy=ShardingStrategy.FULL_SHARD, + mixed_precision=mixed_precision, + device_mesh=self.device_mesh, + sync_module_states=True, + device_id=torch.cuda.current_device(), + cpu_offload=cpu_offload, + use_orig_params=False, + ) + + log_gpu_memory_usage("After FSDP wrapping", logger=logger) + + self.optimizer = optim.AdamW( + self.fsdp_model.parameters(), + lr=self.config.optim.lr, + betas=self.config.optim.betas, + weight_decay=self.config.optim.weight_decay, + ) + + log_gpu_memory_usage("After initialize optimizer", logger=logger) + + self.steps_per_epoch = len(self.train_dataloader) + self.total_steps = self.steps_per_epoch * self.config.trainer.total_epochs + + if self.device_mesh.get_rank() == 0: + print( + f"Number of steps/epoch {self.steps_per_epoch}, number of epochs {self.config.trainer.total_epochs}, total number of steps {self.total_steps}" + ) + + num_warmup_steps = int(self.total_steps * self.config.optim.warmup_steps_ratio) + + self.lr_scheduler = get_cosine_schedule_with_warmup( + optimizer=self.optimizer, + num_warmup_steps=num_warmup_steps, + num_training_steps=self.total_steps, + ) + + def _load_from_checkpoint(self, checkpoint_path): + """Initialize training state from checkpoint.""" + from torch.distributed.fsdp import FullStateDictConfig, StateDictType + + if self.device_mesh.get_rank() == 0: + print(f"Resuming from checkpoint: {checkpoint_path}") + + # Load training state + training_state_path = os.path.join(checkpoint_path, "training_state.pt") + optimizer_state_path = os.path.join(checkpoint_path, "optimizer_state.pt") + + # Only rank 0 loads the full state initially + if self.device_mesh.get_rank() == 0 and os.path.exists(training_state_path): + training_state = torch.load(training_state_path) + epoch = training_state["epoch"] + global_step = training_state["global_step"] + + # Load scheduler state + self.lr_scheduler.load_state_dict(training_state["lr_scheduler"]) + else: + # For other ranks or if file is missing, get step from path + epoch = 0 + global_step = extract_step(checkpoint_path) or 0 + + # Broadcast values to all ranks + if torch.distributed.get_world_size() > 1: + tensor = torch.tensor([epoch, global_step], device="cuda") + torch.distributed.broadcast(tensor, src=0) + if self.device_mesh.get_rank() != 0: + epoch, global_step = tensor.tolist() + + # Load optimizer state if exists + if os.path.exists(optimizer_state_path): + from torch.distributed.fsdp import FullStateDictConfig + from torch.distributed.fsdp.api import FullOptimStateDictConfig + + with FSDP.state_dict_type( + self.fsdp_model, + StateDictType.FULL_STATE_DICT, + FullStateDictConfig(offload_to_cpu=True, rank0_only=True), + ): + # Load optimizer state - rank 0 loads, others receive broadcast + if self.device_mesh.get_rank() == 0: + optim_state = torch.load(optimizer_state_path) + else: + optim_state = None + + # Use FSDP utility to load optimizer state + optim_state_dict = FSDP.scatter_full_optim_state_dict( + optim_state, self.fsdp_model + ) + self.optimizer.load_state_dict(optim_state_dict) + + self.current_epoch = epoch + return global_step + + def _compute_loss_and_backward(self, batch, do_backward=True): + """Compute loss with optional sequence parallelism and remove padding features""" + use_sp = ( + self.use_remove_padding and self.config.ulysses_sequence_parallel_size > 1 + ) + + input_ids = batch["input_ids"].cuda(non_blocking=True) + attention_mask = batch["attention_mask"].cuda(non_blocking=True).bool() + position_ids = batch["position_ids"].cuda(non_blocking=True) + loss_mask = batch["loss_mask"].cuda(non_blocking=True).bool() + + loss_fct = nn.CrossEntropyLoss(reduction="none") + pad_eos_token_id = ( + self.config.data.pad_token_id + if self.config.data.pad_token_id is not None + else self.tokenizer.pad_token_id + ) + + # Context manager for sequence parallel if needed + context = self.sharding_manager if use_sp else nullcontext() + with context: + with torch.autocast(device_type="cuda", dtype=torch.bfloat16): + if not use_sp: + # Standard forward pass without sequence parallel + labels = input_ids.contiguous() + + # Forward pass + # NOTE: loss_mask is of size (batch_size, seq_len - 1) + batch_size = input_ids.shape[0] + masked_input_ids, t, loss_mask_nonflatten = q_sample( + input_ids, + maskable_mask=loss_mask, + mask_token_id=self.tokenizer.mask_token_id, + eos_token_id=( + pad_eos_token_id + if self.config.data.get("treat_eos_as_one", False) + else None + ), + ) + loss_mask = loss_mask_nonflatten.reshape(-1) + + # 2d -> 4d conversion for attention_mask + attention_mask = torch.logical_and( + attention_mask.unsqueeze(1).unsqueeze(-2), + attention_mask.unsqueeze(1).unsqueeze(-1), + ) + + output = self.fsdp_model( + input_ids=masked_input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + use_cache=False, + ) + logits = output.logits + + shift_logits = torch.cat( + [logits[:, 0:1], logits[:, :-1]], dim=1 + ).contiguous() + shift_labels = labels.contiguous() + # Flatten the tokens + shift_logits = shift_logits.view(-1, self.model.config.vocab_size) + shift_labels = shift_labels.view(-1) + # Enable model parallelism + shift_labels = shift_labels.to(shift_logits.device) + loss = loss_fct(shift_logits, shift_labels) + + # We use weighted loss + loss_mask = loss_mask.to(loss.device) + loss = loss.masked_fill(~loss_mask, 0) + if self.config.diffusion.token_reweighting: + loss = ( + self.config.diffusion.alpha + * (1 - torch.exp(-loss)) ** self.config.diffusion.gamma + * loss + ) + + if self.config.diffusion.time_reweighting == "original": + weight = 1 / t[:, None].float().expand(labels.size()) + elif self.config.diffusion.time_reweighting == "linear": + weight = 1 - t[:, None].float().expand(labels.size()) + elif self.config.diffusion.time_reweighting == "cart": + # seq_len = self.config.data.max_length + seq_len = input_ids.shape[-1] + weight_matrix = context_adaptive_reweight( + seq_len, cart_p=self.config.diffusion.cart_p + ) + _weight_matrix = weight_matrix[:seq_len, :seq_len].to( + loss_mask.device + ) + non_mask = ~loss_mask_nonflatten.to( + loss.device + ) # loss_mask indicates where is mask + weight = ( + non_mask.type_as(_weight_matrix) + .matmul(_weight_matrix) + .masked_fill(non_mask, 0) + ) + else: + weight = ( + t.new_ones((batch_size, 1)).float().expand(labels.size()) + ) + + loss = loss * weight.reshape(-1) + else: + raise NotImplementedError( + "Sequence parallel is not implemented yet" + ) + + valid_token_this_rank = torch.sum(loss_mask) + loss = torch.sum(loss) / valid_token_this_rank + + if do_backward: + loss.backward() + return loss + + def training_step(self, batch: TensorDict): + self.fsdp_model.train() + + log_gpu_memory_usage("Before optimizer zero_grad", logger=logger) + + self.optimizer.zero_grad() + + log_gpu_memory_usage("After optimizer zero_grad", logger=logger) + + micro_batches = batch.split(self.config.data.micro_batch_size_per_gpu) + n_micro_batches = len(micro_batches) + step_loss = 0 + for micro_batch in micro_batches: + loss = ( + self._compute_loss_and_backward(batch=micro_batch, do_backward=False) + / n_micro_batches + ) + loss.backward() + step_loss += loss.item() + + grad_norm = self.fsdp_model.clip_grad_norm_( + max_norm=self.config.optim.clip_grad + ) + + log_gpu_memory_usage("Before optimizer step", logger=logger) + + self.optimizer.step() + + log_gpu_memory_usage("After optimizer step", logger=logger) + + self.lr_scheduler.step() + + # reduce loss across dp ranks + lr = self.lr_scheduler.get_last_lr()[0] + + log_gpu_memory_usage("After offload weights", logger=logger) + + step_loss = torch.tensor(step_loss, device="cuda") + torch.distributed.all_reduce(step_loss, op=torch.distributed.ReduceOp.AVG) + + # record how many tokens are padded as EOS + all_num_eos = [] + for micro_batch in micro_batches: + loss_mask = batch["loss_mask"] + num_eos = loss_mask.sum(-1) + num_eos -= num_eos.min() - 1 + all_num_eos.append(num_eos) + all_num_eos = torch.cat(all_num_eos) + + return { + "train/loss": step_loss.detach().item(), + "train/lr(1e-3)": lr * 1e3, + "train/grad_norm": grad_norm, + "train/num_eos_mean": all_num_eos.float().mean().item(), + "train/num_eos_max": all_num_eos.float().max().item(), + } + + def validation_step(self, batch: TensorDict): + self.fsdp_model.eval() + with torch.no_grad(): + loss = self._compute_loss_and_backward(batch, do_backward=False) + torch.distributed.all_reduce(loss, op=torch.distributed.ReduceOp.AVG) + return loss + + def save_checkpoint(self, step): + """Save model, optimizer, and training state.""" + from torch.distributed.fsdp import FullStateDictConfig, StateDictType + from torch.distributed.fsdp.api import FullOptimStateDictConfig + + # Create checkpoint directory + path = os.path.join( + self.config.trainer.default_local_dir, f"global_step_{step}" + ) + + # Save model state + model_cfg = FullStateDictConfig(offload_to_cpu=True, rank0_only=True) + + with FSDP.state_dict_type( + self.fsdp_model, StateDictType.FULL_STATE_DICT, model_cfg + ): + model_state = self.fsdp_model.state_dict() + optim_state = FSDP.full_optim_state_dict(self.fsdp_model, self.optimizer) + + # Save training state + training_state = { + "lr_scheduler": self.lr_scheduler.state_dict(), + "global_step": step, + "epoch": self.current_epoch, + } + + # Save on rank 0 only + if self.device_mesh.get_rank() == 0: + os.makedirs(path, exist_ok=True) + + # Save model using HF's save_pretrained + self.model.save_pretrained(path, state_dict=model_state) + self.tokenizer.save_pretrained(path) + + # Save optimizer and training state + torch.save(optim_state, os.path.join(path, "optimizer_state.pt")) + torch.save(training_state, os.path.join(path, "training_state.pt")) + + # Copy to HDFS if configured + if self.config.trainer.default_hdfs_dir: + hdfs_io.makedirs(self.config.trainer.default_hdfs_dir, exist_ok=True) + hdfs_io.copy( + src=path, + dst=self.config.trainer.default_hdfs_dir, + dirs_exist_ok=True, + ) + torch.distributed.barrier() + + def _find_latest_checkpoint(self): + """Find the latest checkpoint in checkpoint directories.""" + latest_checkpoint = None + latest_step = -1 + + # Check local directory first + local_dir = self.config.trainer.default_local_dir + if os.path.exists(local_dir): + checkpoints = [ + d + for d in os.listdir(local_dir) + if os.path.isdir(os.path.join(local_dir, d)) + and d.startswith("global_step_") + ] + + for ckpt in checkpoints: + step = extract_step(ckpt) + if step is not None and step > latest_step: + latest_step = step + latest_checkpoint = os.path.join(local_dir, ckpt) + + # If not found locally and HDFS is configured, check there + if latest_checkpoint is None and self.config.trainer.default_hdfs_dir: + try: + if hdfs_io.exists(self.config.trainer.default_hdfs_dir): + checkpoints = [ + d + for d in hdfs_io.listdir(self.config.trainer.default_hdfs_dir) + if d.startswith("global_step_") + ] + for ckpt in checkpoints: + step = extract_step(ckpt) + if step is not None and step > latest_step: + latest_step = step + remote_path = os.path.join( + self.config.trainer.default_hdfs_dir, ckpt + ) + + # Copy from HDFS to local + local_path = os.path.join(local_dir, ckpt) + os.makedirs(local_dir, exist_ok=True) + hdfs_io.copy( + src=remote_path, dst=local_path, dirs_exist_ok=True + ) + latest_checkpoint = local_path + except Exception as e: + if self.device_mesh.get_rank() == 0: + print(f"Error checking HDFS for checkpoints: {e}") + + return latest_checkpoint + + def fit(self): + rank = self.device_mesh.get_rank() + + # TODO: add a unified tracking + if rank == 0: + tracking = Tracking( + project_name=self.config.trainer.project_name, + experiment_name=self.config.trainer.experiment_name, + default_backend=self.config.trainer.logger, + ) + + global_step = 0 + + # Handle resuming training + if self.resume_training: + # Find latest checkpoint if not specified + if not self.resume_checkpoint_path: + self.resume_checkpoint_path = self._find_latest_checkpoint() + + if self.resume_checkpoint_path: + global_step = self._load_from_checkpoint(self.resume_checkpoint_path) + if rank == 0: + print( + f"Resumed training from step {global_step}, epoch {self.current_epoch}" + ) + elif rank == 0: + print("No checkpoint found, starting training from scratch") + + # Compute total training steps + total_training_steps = ( + len(self.train_dataloader) * self.config.trainer.total_epochs + ) + + if self.config.trainer.total_training_steps is not None: + total_training_steps = self.config.trainer.total_training_steps + + self.total_training_steps = total_training_steps + if rank == 0: + print(f"Total training steps: {self.total_training_steps}") + + # Begin training from the current epoch + for epoch in range(self.current_epoch, self.config.trainer.total_epochs): + self.current_epoch = epoch + self.train_sampler.set_epoch(epoch=epoch) + + # Create a data iterator + dataloader_iter = iter(self.train_dataloader) + + # If resuming mid-epoch, skip to the right position + if epoch == self.current_epoch and global_step > 0 and self.resume_training: + steps_in_epoch = global_step % self.steps_per_epoch + if steps_in_epoch > 0: + if rank == 0: + print( + f"Skipping {steps_in_epoch} steps to resume at the right position" + ) + for _ in range(steps_in_epoch): + try: + next(dataloader_iter) + except StopIteration: + dataloader_iter = iter(self.train_dataloader) + + # Calculate remaining steps in this epoch + remaining_steps = self.steps_per_epoch + if epoch == self.current_epoch and global_step > 0 and self.resume_training: + remaining_steps -= global_step % self.steps_per_epoch + + for data in tqdm( + dataloader_iter, + initial=self.steps_per_epoch - remaining_steps, + total=self.steps_per_epoch, + desc=f"Epoch {epoch+1}/{self.config.trainer.total_epochs}", + ): + data = TensorDict( + data, batch_size=self.config.data.train_batch_size + ).cuda() + metric = self.training_step(data) + if rank == 0: + tracking.log(data=metric, step=global_step) + global_step += 1 + + # for early exit validation + if global_step >= self.total_training_steps: + # Perform final validation + val_losses = [] + for val_data in self.val_dataloader: + val_data = TensorDict( + val_data, + batch_size=self.config.data.micro_batch_size_per_gpu, + ).cuda() + val_loss = self.validation_step(val_data) + val_losses.append(val_loss) + if rank == 0: + avg_val_loss = torch.mean(torch.stack(val_losses)) + metric = {"val/loss": avg_val_loss.detach().item()} + tracking.log(data=metric, step=global_step) + torch.distributed.barrier() + + # Save final checkpoint + self.save_checkpoint(step=global_step) + return + + if global_step % self.config.trainer.save_checkpoint_steps == 0: + # Perform validation + val_losses = [] + for val_data in self.val_dataloader: + val_data = TensorDict( + val_data, + batch_size=self.config.data.micro_batch_size_per_gpu, + ).cuda() + val_loss = self.validation_step(val_data) + val_losses.append(val_loss) + if rank == 0: + avg_val_loss = torch.mean(torch.stack(val_losses)) + metric = {"val/loss": avg_val_loss.detach().item()} + tracking.log(data=metric, step=global_step) + torch.distributed.barrier() + + # Save checkpoint + self.save_checkpoint(step=global_step) + + # validation + val_losses = [] + for data in self.val_dataloader: + data = TensorDict( + data, batch_size=self.config.data.micro_batch_size_per_gpu + ).cuda() + val_loss = self.validation_step(data) + val_losses.append(val_loss) + if rank == 0: + val_loss = torch.mean(torch.stack(val_losses)) + metric = {"val/loss": val_loss.detach().item()} + tracking.log(data=metric, step=global_step) + torch.distributed.barrier() + + # save checkpoint + self.save_checkpoint(step=global_step) + + +@hydra.main(config_path="config", config_name="sft_trainer", version_base=None) +def main(config): + local_rank, rank, world_size = initialize_global_process_group() + + device_mesh = init_device_mesh( + device_type="cuda", mesh_shape=(world_size,), mesh_dim_names=("fsdp",) + ) + dp_size = world_size // config.ulysses_sequence_parallel_size + ulysses_device_mesh = init_device_mesh( + device_type="cuda", + mesh_shape=(dp_size, config.ulysses_sequence_parallel_size), + mesh_dim_names=("dp", "sp"), + ) + trainer = FSDPSFTTrainer( + config=config, device_mesh=device_mesh, ulysses_device_mesh=ulysses_device_mesh + ) + trainer.fit() + + +if __name__ == "__main__": + main() diff --git a/Prism/Dream/Dream_Prism/src/trainer/sft_dataset.py b/Prism/Dream/Dream_Prism/src/trainer/sft_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..a1c3e549f2a1e35d8f9e3236843e4652a7775c5f --- /dev/null +++ b/Prism/Dream/Dream_Prism/src/trainer/sft_dataset.py @@ -0,0 +1,255 @@ +# Copyright 2024 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +SFT dataset +- We assume user pass a single parquet file. +- We load all the data into the memory. +- **NOTE**: We support multi-turn prompts. +Each parquet file contains +""" + +from typing import List, Union + +import pandas as pd + +import torch +from datasets import Dataset as HFDataset +from torch.utils.data import Dataset +from transformers import PreTrainedTokenizer +from functools import partial + +from verl.utils.fs import copy_local_path_from_hdfs +from verl.utils.model import compute_position_id_with_mask +from verl.utils import hf_tokenizer + + +class SFTDataset(Dataset): + """ + This is an in-memory SFTDataset + """ + + def __init__( + self, + parquet_files: Union[str, List[str]], + tokenizer, + prompt_key="prompt", + response_key="response", + max_length=1024, + truncation="error", + pad_token_id=None, + pad_input=False, + ): + assert truncation in ["error", "left", "right"] + self.truncation = truncation + + if not isinstance(parquet_files, List): + parquet_files = [parquet_files] + + self.parquet_files = parquet_files + if isinstance(tokenizer, str): + tokenizer = hf_tokenizer(tokenizer) + self.tokenizer: PreTrainedTokenizer = tokenizer + + self.prompt_key = prompt_key + self.response_key = response_key + + self.max_length = max_length + self.pad_token_id = ( + pad_token_id if pad_token_id is not None else self.tokenizer.pad_token_id + ) + self.pad_input = pad_input + self._download() + self._read_files_and_tokenize() + + def _download(self): + for i, parquet_file in enumerate(self.parquet_files): + self.parquet_files[i] = copy_local_path_from_hdfs( + parquet_file, verbose=True + ) + + def _read_files_and_tokenize(self): + + def series_to_item(ls): + import pandas, numpy + + while ( + isinstance(ls, (pandas.core.series.Series, numpy.ndarray)) + and len(ls) == 1 + ): + ls = ls[0] + return ls + + dataframes = [] + for parquet_file in self.parquet_files: + # read parquet files and cache + dataframe = pd.read_parquet(parquet_file) + dataframes.append(dataframe) + self.dataframe = pd.concat(dataframes) + + @staticmethod + def _tokenize_static(example, tokenizer, prompt_key, response_key, max_length, truncation, pad_token_id): + prompt = example[prompt_key] + response = example[response_key] + + # apply chat template + if not isinstance(prompt, str): + prompt_chat = list(prompt) + else: + prompt_chat = [{"role": "user", "content": prompt}] + + # string + prompt_chat_str = tokenizer.apply_chat_template( + prompt_chat, add_generation_prompt=True, tokenize=False + ) + response_chat_str = response + tokenizer.eos_token + + # tokenize + prompt_ids_output = tokenizer( + prompt_chat_str, return_tensors="pt", add_special_tokens=False + ) + prompt_ids = prompt_ids_output["input_ids"][0] + prompt_attention_mask = prompt_ids_output["attention_mask"][0] + + response_ids_output = tokenizer( + response_chat_str, return_tensors="pt", add_special_tokens=False + ) + response_ids = response_ids_output["input_ids"][0] + response_attention_mask = response_ids_output["attention_mask"][0] + + prompt_length = prompt_ids.shape[0] + response_length = response_ids.shape[0] + + input_ids = torch.cat((prompt_ids, response_ids), dim=-1) + attention_mask = torch.cat( + (prompt_attention_mask, response_attention_mask), dim=-1 + ) + + # padding to max length + sequence_length = input_ids.shape[0] + if sequence_length < max_length: + padded_input_ids = ( + torch.ones( + size=(max_length - sequence_length,), dtype=input_ids.dtype + ) + * pad_token_id + ) + padded_attention_mask = torch.ones( # NOTE: we use 1 here + size=(max_length - sequence_length,), dtype=attention_mask.dtype + ) + + input_ids = torch.cat((input_ids, padded_input_ids)) + attention_mask = torch.cat((attention_mask, padded_attention_mask)) + elif sequence_length > max_length: + if truncation == "left": + # actually, left truncation may not be reasonable + input_ids = input_ids[-max_length :] + attention_mask = attention_mask[-max_length :] + elif truncation == "right": + input_ids = input_ids[: max_length] + attention_mask = attention_mask[: max_length] + elif truncation == "error": + raise NotImplementedError( + f"{sequence_length=} is larger than {max_length=}" + ) + else: + raise NotImplementedError( + f"Unknown truncation method {truncation}" + ) + + position_ids = compute_position_id_with_mask(attention_mask) + + loss_mask = attention_mask.clone() + if prompt_length > 1: + # mask out prompt for SFT. + loss_mask[: min(prompt_length, loss_mask.size(0))] = 0 + + return { + "input_ids": input_ids.numpy(), + "attention_mask": attention_mask.numpy(), + "position_ids": position_ids.numpy(), + "loss_mask": loss_mask.numpy(), + } + + def _tokenize(self, example): + return self._tokenize_static( + example, + self.tokenizer, + self.prompt_key, + self.response_key, + self.max_length, + self.truncation, + self.pad_token_id + ) + + def __len__(self): + return len(self.dataframe) + + def __getitem__(self, item): + example = self.dataframe.iloc[item] + data = self._tokenize(example) + return { + "input_ids": torch.tensor(data["input_ids"]), + "attention_mask": torch.tensor(data["attention_mask"]), + "position_ids": torch.tensor(data["position_ids"]), + "loss_mask": torch.tensor(data["loss_mask"]), + } + + def save_tokenized(self, path, num_proc=16): + hf_dataset = HFDataset.from_pandas(self.dataframe) + tokenize_fn = partial( + self._tokenize_static, + tokenizer=self.tokenizer, + prompt_key=self.prompt_key, + response_key=self.response_key, + max_length=self.max_length, + truncation=self.truncation, + pad_token_id=self.pad_token_id + ) + hf_dataset = hf_dataset.map(tokenize_fn, num_proc=num_proc) + hf_dataset.to_pandas().to_parquet(path) + + +class TokenizedSFTDataset(Dataset): + """ + This is an in-memory tokenized SFTDataset + """ + + def __init__( + self, + parquet_files: Union[str, List[str]], + ): + if not isinstance(parquet_files, List): + parquet_files = [parquet_files] + + self.parquet_files = parquet_files + self._read_files() + + def _read_files(self): + dataframes = [] + for parquet_file in self.parquet_files: + # read parquet files and cache + dataframe = pd.read_parquet(parquet_file) + dataframes.append(dataframe) + dataframe = pd.concat(dataframes) + self.hf_dataset = HFDataset.from_pandas(dataframe) + self.hf_dataset.set_format( + type="torch", + columns=["input_ids", "attention_mask", "position_ids", "loss_mask"], + ) + + def __len__(self): + return len(self.hf_dataset) + + def __getitem__(self, item): + return self.hf_dataset[item] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/aiosignal/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/aiosignal/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d82f51b4c1739bc7f26c653a57d37a27a1603843 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/aiosignal/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama-0.4.6.dist-info/licenses/LICENSE.txt b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama-0.4.6.dist-info/licenses/LICENSE.txt new file mode 100644 index 0000000000000000000000000000000000000000..3105888ec149d10cad51c11d332779e94b548661 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama-0.4.6.dist-info/licenses/LICENSE.txt @@ -0,0 +1,27 @@ +Copyright (c) 2010 Jonathan Hartley +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holders, nor those of its contributors + may be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda_bindings-12.9.4.dist-info/licenses/LICENSE b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda_bindings-12.9.4.dist-info/licenses/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..b7d042fcee35d11ffb37ffa7b8fb7d2bee3f7999 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda_bindings-12.9.4.dist-info/licenses/LICENSE @@ -0,0 +1,48 @@ +NVIDIA SOFTWARE LICENSE + +This license is a legal agreement between you and NVIDIA Corporation ("NVIDIA") and governs your use of the NVIDIA CUDA Python software and materials provided hereunder ("SOFTWARE"). + +This license can be accepted only by an adult of legal age of majority in the country in which the SOFTWARE is used. If you are under the legal age of majority, you must ask your parent or legal guardian to consent to this license. By taking delivery of the SOFTWARE, you affirm that you have reached the legal age of majority, you accept the terms of this license, and you take legal and financial responsibility for the actions of your permitted users. + +You agree to use the SOFTWARE only for purposes that are permitted by (a) this license, and (b) any applicable law, regulation or generally accepted practices or guidelines in the relevant jurisdictions. + +1. LICENSE. Subject to the terms of this license, NVIDIA grants you a non-exclusive limited license to: (a) install and use the SOFTWARE, and (b) distribute the SOFTWARE subject to the distribution requirements described in this license. NVIDIA reserves all rights, title and interest in and to the SOFTWARE not expressly granted to you under this license. + +2. DISTRIBUTION REQUIREMENTS. These are the distribution requirements for you to exercise the distribution grant: +a. The terms under which you distribute the SOFTWARE must be consistent with the terms of this license, including (without limitation) terms relating to the license grant and license restrictions and protection of NVIDIA's intellectual property rights. +b. You agree to notify NVIDIA in writing of any known or suspected distribution or use of the SOFTWARE not in compliance with the requirements of this license, and to enforce the terms of your agreements with respect to distributed SOFTWARE. + +3. LIMITATIONS. Your license to use the SOFTWARE is restricted as follows: +a. The SOFTWARE is licensed for you to develop applications only for use in systems with NVIDIA GPUs. +b. You may not reverse engineer, decompile or disassemble, or remove copyright or other proprietary notices from any portion of the SOFTWARE or copies of the SOFTWARE. +c. You may not modify or create derivative works of any portion of the SOFTWARE. +d. You may not bypass, disable, or circumvent any technical measure, encryption, security, digital rights management or authentication mechanism in the SOFTWARE. +e. You may not use the SOFTWARE in any manner that would cause it to become subject to an open source software license. As examples, licenses that require as a condition of use, modification, and/or distribution that the SOFTWARE be (i) disclosed or distributed in source code form; (ii) licensed for the purpose of making derivative works; or (iii) redistributable at no charge. +f. Unless you have an agreement with NVIDIA for this purpose, you may not use the SOFTWARE with any system or application where the use or failure of the system or application can reasonably be expected to threaten or result in personal injury, death, or catastrophic loss. Examples include use in avionics, navigation, military, medical, life support or other life critical applications. NVIDIA does not design, test or manufacture the SOFTWARE for these critical uses and NVIDIA shall not be liable to you or any third party, in whole or in part, for any claims or damages arising from such uses. +g. You agree to defend, indemnify and hold harmless NVIDIA and its affiliates, and their respective employees, contractors, agents, officers and directors, from and against any and all claims, damages, obligations, losses, liabilities, costs or debt, fines, restitutions and expenses (including but not limited to attorney's fees and costs incident to establishing the right of indemnification) arising out of or related to use of the SOFTWARE outside of the scope of this Agreement, or not in compliance with its terms. + +4. PRE-RELEASE. SOFTWARE versions identified as alpha, beta, preview, early access or otherwise as pre-release may not be fully functional, may contain errors or design flaws, and may have reduced or different security, privacy, availability, and reliability standards relative to commercial versions of NVIDIA software and materials. You may use a pre-release SOFTWARE version at your own risk, understanding that these versions are not intended for use in production or business-critical systems. + +5. OWNERSHIP. The SOFTWARE and the related intellectual property rights therein are and will remain the sole and exclusive property of NVIDIA or its licensors. The SOFTWARE is copyrighted and protected by the laws of the United States and other countries, and international treaty provisions. NVIDIA may make changes to the SOFTWARE, at any time without notice, but is not obligated to support or update the SOFTWARE. + +6. COMPONENTS UNDER OTHER LICENSES. The SOFTWARE may include NVIDIA or third-party components with separate legal notices or terms as may be described in proprietary notices accompanying the SOFTWARE. If and to the extent there is a conflict between the terms in this license and the license terms associated with a component, the license terms associated with the components control only to the extent necessary to resolve the conflict. + +7. FEEDBACK. You may, but don't have to, provide to NVIDIA any Feedback. "Feedback" means any suggestions, bug fixes, enhancements, modifications, feature requests or other feedback regarding the SOFTWARE. For any Feedback that you voluntarily provide, you hereby grant NVIDIA and its affiliates a perpetual, non-exclusive, worldwide, irrevocable license to use, reproduce, modify, license, sublicense (through multiple tiers of sublicensees), and distribute (through multiple tiers of distributors) the Feedback without the payment of any royalties or fees to you. NVIDIA will use Feedback at its choice. + +8. NO WARRANTIES. THE SOFTWARE IS PROVIDED "AS IS" WITHOUT ANY EXPRESS OR IMPLIED WARRANTY OF ANY KIND INCLUDING, BUT NOT LIMITED TO, WARRANTIES OF MERCHANTABILITY, NONINFRINGEMENT, OR FITNESS FOR A PARTICULAR PURPOSE. NVIDIA DOES NOT WARRANT THAT THE SOFTWARE WILL MEET YOUR REQUIREMENTS OR THAT THE OPERATION THEREOF WILL BE UNINTERRUPTED OR ERROR-FREE, OR THAT ALL ERRORS WILL BE CORRECTED. + +9. LIMITATIONS OF LIABILITY. TO THE MAXIMUM EXTENT PERMITTED BY LAW, NVIDIA AND ITS AFFILIATES SHALL NOT BE LIABLE FOR ANY SPECIAL, INCIDENTAL, PUNITIVE OR CONSEQUENTIAL DAMAGES, OR ANY LOST PROFITS, PROJECT DELAYS, LOSS OF USE, LOSS OF DATA OR LOSS OF GOODWILL, OR THE COSTS OF PROCURING SUBSTITUTE PRODUCTS, ARISING OUT OF OR IN CONNECTION WITH THIS LICENSE OR THE USE OR PERFORMANCE OF THE SOFTWARE, WHETHER SUCH LIABILITY ARISES FROM ANY CLAIM BASED UPON BREACH OF CONTRACT, BREACH OF WARRANTY, TORT (INCLUDING NEGLIGENCE), PRODUCT LIABILITY OR ANY OTHER CAUSE OF ACTION OR THEORY OF LIABILITY, EVEN IF NVIDIA HAS PREVIOUSLY BEEN ADVISED OF, OR COULD REASONABLY HAVE FORESEEN, THE POSSIBILITY OF SUCH DAMAGES. IN NO EVENT WILL NVIDIA'S AND ITS AFFILIATES TOTAL CUMULATIVE LIABILITY UNDER OR ARISING OUT OF THIS LICENSE EXCEED US$10.00. THE NATURE OF THE LIABILITY OR THE NUMBER OF CLAIMS OR SUITS SHALL NOT ENLARGE OR EXTEND THIS LIMIT. + +10. TERMINATION. Your rights under this license will terminate automatically without notice from NVIDIA if you fail to comply with any term and condition of this license or if you commence or participate in any legal proceeding against NVIDIA with respect to the SOFTWARE. NVIDIA may terminate this license with advance written notice to you if NVIDIA decides to no longer provide the SOFTWARE in a country or, in NVIDIA's sole discretion, the continued use of it is no longer commercially viable. Upon any termination of this license, you agree to promptly discontinue use of the SOFTWARE and destroy all copies in your possession or control. Your prior distributions in accordance with this license are not affected by the termination of this license. All provisions of this license will survive termination, except for the license granted to you. + +11. APPLICABLE LAW. This license will be governed in all respects by the laws of the United States and of the State of Delaware as those laws are applied to contracts entered into and performed entirely within Delaware by Delaware residents, without regard to the conflicts of laws principles. The United Nations Convention on Contracts for the International Sale of Goods is specifically disclaimed. You agree to all terms of this Agreement in the English language. The state or federal courts residing in Santa Clara County, California shall have exclusive jurisdiction over any dispute or claim arising out of this license. Notwithstanding this, you agree that NVIDIA shall still be allowed to apply for injunctive remedies or an equivalent type of urgent legal relief in any jurisdiction. + +12. NO ASSIGNMENT. This license and your rights and obligations thereunder may not be assigned by you by any means or operation of law without NVIDIA's permission. Any attempted assignment not approved by NVIDIA in writing shall be void and of no effect. + +13. EXPORT. The SOFTWARE is subject to United States export laws and regulations. You agree that you will not ship, transfer or export the SOFTWARE into any country, or use the SOFTWARE in any manner, prohibited by the United States Bureau of Industry and Security or economic sanctions regulations administered by the U.S. Department of Treasury's Office of Foreign Assets Control (OFAC), or any applicable export laws, restrictions or regulations. These laws include restrictions on destinations, end users and end use. By accepting this license, you confirm that you are not a resident or citizen of any country currently embargoed by the U.S. and that you are not otherwise prohibited from receiving the SOFTWARE. + +14. GOVERNMENT USE. The SOFTWARE has been developed entirely at private expense and is "commercial items" consisting of "commercial computer software" and "commercial computer software documentation" provided with RESTRICTED RIGHTS. Use, duplication or disclosure by the U.S. Government or a U.S. Government subcontractor is subject to the restrictions in this license pursuant to DFARS 227.7202-3(a) or as set forth in subparagraphs (b)(1) and (2) of the Commercial Computer Software - Restricted Rights clause at FAR 52.227-19, as applicable. Contractor/manufacturer is NVIDIA, 2788 San Tomas Expressway, Santa Clara, CA 95051. + +15. ENTIRE AGREEMENT. This license is the final, complete and exclusive agreement between the parties relating to the subject matter of this license and supersedes all prior or contemporaneous understandings and agreements relating to this subject matter, whether oral or written. If any court of competent jurisdiction determines that any provision of this license is illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect. This license may only be modified in a writing signed by an authorized representative of each party. + +(v. May 12, 2021) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8341e566f93a072b327a31b8e9f84904df597a71 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_commit_api.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_commit_api.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e9c0e54a584c17c8e2490d70049d5571102dc292 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_commit_api.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_commit_scheduler.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_commit_scheduler.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d267e49398d6020a9bf1bb46628dfd0ab300b5ad Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_commit_scheduler.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_inference_endpoints.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_inference_endpoints.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7f8a686c7813dcbf0a7fba2c8b2ed4a6743db3a2 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_inference_endpoints.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_jobs_api.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_jobs_api.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b44560c710f87b5afd085fc26beb7912a7f66f50 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_jobs_api.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_local_folder.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_local_folder.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fa26839c95f9ad46f896d5c6c443cf9f06c870df Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_local_folder.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_login.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_login.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b2611a8cc64d0042479284d96911e34c9a5a6974 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_login.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_oauth.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_oauth.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c62c2ff010ff94d9b71e7a3d362037bee942a970 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_oauth.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_snapshot_download.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_snapshot_download.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..786afb4519c3cea3bc77c74cb5205a49869c6c30 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_snapshot_download.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_space_api.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_space_api.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7d7b89ab22be78d8076a37db0d5a7aa1d7b47b5d Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_space_api.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_tensorboard_logger.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_tensorboard_logger.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3b630353d495bcbdf9e6c66e8bbb878d7d8760bd Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_tensorboard_logger.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_upload_large_folder.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_upload_large_folder.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d12d48e5eb77394632ee39138f3041657f851e2c Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_upload_large_folder.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_webhooks_payload.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_webhooks_payload.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7d9ffcecb69418e6256e7cd313602d74e82a132c Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_webhooks_payload.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_webhooks_server.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_webhooks_server.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..035e0fe0612ebce97d1d0df682112708a490739f Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/_webhooks_server.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/community.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/community.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dab104638d872e98fb6a0cd10e69c26f683640db Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/community.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/constants.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/constants.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..731afc3d476dd1800fc1bd1f2f07f1cd716436bb Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/constants.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/dataclasses.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/dataclasses.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cedd2f5e05b806b02b047b4bfdb859540ab6252d Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/dataclasses.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/errors.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/errors.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a50f12710013b34c7161b7b5a9a7dfa2cc21f8c9 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/errors.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/fastai_utils.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/fastai_utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7dcfdd26f1280606e53667e1cbcc9be455287733 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/fastai_utils.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/file_download.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/file_download.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3740b5dc1aa1d3f3997f2325b2e88cb1ba1f818f Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/file_download.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/hf_file_system.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/hf_file_system.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d1948f57077bee09234e1327a043caa8748db7ad Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/hf_file_system.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/hub_mixin.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/hub_mixin.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3307eb10ecb075ba8a97506b3b14c5b07ade097f Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/hub_mixin.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/inference_api.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/inference_api.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6b76323fe9c0ecd6be0d16decfc490590c446a8a Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/inference_api.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/keras_mixin.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/keras_mixin.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6356bd943cac251af135b13ffaa4dc55c4ca3977 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/keras_mixin.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/lfs.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/lfs.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7f20bb304c459c7e5bce6b658e7d586a72f9d01c Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/lfs.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/repocard.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/repocard.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..97e6f4dae7a3c5a2bf1303bfebfead3dc4f6536d Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/repocard.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/repocard_data.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/repocard_data.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d36de65509783e15539939d7210efec811f49d24 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/repocard_data.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/repository.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/repository.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c19f627b1495b2d9e33448e171ea8c1612b3c0c8 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/__pycache__/repository.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7a1a8d793b89e16e5fa46ec5d420ec96fe1d72fe --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/__init__.py @@ -0,0 +1,27 @@ +# Copyright 2025 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import ABC, abstractmethod +from argparse import _SubParsersAction + + +class BaseHuggingfaceCLICommand(ABC): + @staticmethod + @abstractmethod + def register_subcommand(parser: _SubParsersAction): + raise NotImplementedError() + + @abstractmethod + def run(self): + raise NotImplementedError() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/_cli_utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/_cli_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..bd56ad6896db2a257323e022896940c0ba0d68d3 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/_cli_utils.py @@ -0,0 +1,69 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains a utility for good-looking prints.""" + +import os +from typing import List, Union + + +class ANSI: + """ + Helper for en.wikipedia.org/wiki/ANSI_escape_code + """ + + _bold = "\u001b[1m" + _gray = "\u001b[90m" + _red = "\u001b[31m" + _reset = "\u001b[0m" + _yellow = "\u001b[33m" + + @classmethod + def bold(cls, s: str) -> str: + return cls._format(s, cls._bold) + + @classmethod + def gray(cls, s: str) -> str: + return cls._format(s, cls._gray) + + @classmethod + def red(cls, s: str) -> str: + return cls._format(s, cls._bold + cls._red) + + @classmethod + def yellow(cls, s: str) -> str: + return cls._format(s, cls._yellow) + + @classmethod + def _format(cls, s: str, code: str) -> str: + if os.environ.get("NO_COLOR"): + # See https://no-color.org/ + return s + return f"{code}{s}{cls._reset}" + + +def tabulate(rows: List[List[Union[str, int]]], headers: List[str]) -> str: + """ + Inspired by: + + - stackoverflow.com/a/8356620/593036 + - stackoverflow.com/questions/9535954/printing-lists-as-tabular-data + """ + col_widths = [max(len(str(x)) for x in col) for col in zip(*rows, headers)] + row_format = ("{{:{}}} " * len(headers)).format(*col_widths) + lines = [] + lines.append(row_format.format(*headers)) + lines.append(row_format.format(*["-" * w for w in col_widths])) + for row in rows: + lines.append(row_format.format(*row)) + return "\n".join(lines) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/auth.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/auth.py new file mode 100644 index 0000000000000000000000000000000000000000..bbf475a4f8785152b992b116a69b4b16293688f3 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/auth.py @@ -0,0 +1,213 @@ +# Copyright 2020 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains commands to authenticate to the Hugging Face Hub and interact with your repositories. + +Usage: + # login and save token locally. + hf auth login --token=hf_*** --add-to-git-credential + + # switch between tokens + hf auth switch + + # list all tokens + hf auth list + + # logout from all tokens + hf auth logout + + # check which account you are logged in as + hf auth whoami +""" + +from argparse import _SubParsersAction +from typing import List, Optional + +from requests.exceptions import HTTPError + +from huggingface_hub.commands import BaseHuggingfaceCLICommand +from huggingface_hub.constants import ENDPOINT +from huggingface_hub.hf_api import HfApi + +from .._login import auth_list, auth_switch, login, logout +from ..utils import get_stored_tokens, get_token, logging +from ._cli_utils import ANSI + + +logger = logging.get_logger(__name__) + +try: + from InquirerPy import inquirer + from InquirerPy.base.control import Choice + + _inquirer_py_available = True +except ImportError: + _inquirer_py_available = False + + +class AuthCommands(BaseHuggingfaceCLICommand): + @staticmethod + def register_subcommand(parser: _SubParsersAction): + # Create the main 'auth' command + auth_parser = parser.add_parser("auth", help="Manage authentication (login, logout, etc.).") + auth_subparsers = auth_parser.add_subparsers(help="Authentication subcommands") + + # Show help if no subcommand is provided + auth_parser.set_defaults(func=lambda args: auth_parser.print_help()) + + # Add 'login' as a subcommand of 'auth' + login_parser = auth_subparsers.add_parser( + "login", help="Log in using a token from huggingface.co/settings/tokens" + ) + login_parser.add_argument( + "--token", + type=str, + help="Token generated from https://huggingface.co/settings/tokens", + ) + login_parser.add_argument( + "--add-to-git-credential", + action="store_true", + help="Optional: Save token to git credential helper.", + ) + login_parser.set_defaults(func=lambda args: AuthLogin(args)) + + # Add 'logout' as a subcommand of 'auth' + logout_parser = auth_subparsers.add_parser("logout", help="Log out") + logout_parser.add_argument( + "--token-name", + type=str, + help="Optional: Name of the access token to log out from.", + ) + logout_parser.set_defaults(func=lambda args: AuthLogout(args)) + + # Add 'whoami' as a subcommand of 'auth' + whoami_parser = auth_subparsers.add_parser( + "whoami", help="Find out which huggingface.co account you are logged in as." + ) + whoami_parser.set_defaults(func=lambda args: AuthWhoami(args)) + + # Existing subcommands + auth_switch_parser = auth_subparsers.add_parser("switch", help="Switch between access tokens") + auth_switch_parser.add_argument( + "--token-name", + type=str, + help="Optional: Name of the access token to switch to.", + ) + auth_switch_parser.add_argument( + "--add-to-git-credential", + action="store_true", + help="Optional: Save token to git credential helper.", + ) + auth_switch_parser.set_defaults(func=lambda args: AuthSwitch(args)) + + auth_list_parser = auth_subparsers.add_parser("list", help="List all stored access tokens") + auth_list_parser.set_defaults(func=lambda args: AuthList(args)) + + +class BaseAuthCommand: + def __init__(self, args): + self.args = args + self._api = HfApi() + + +class AuthLogin(BaseAuthCommand): + def run(self): + logging.set_verbosity_info() + login( + token=self.args.token, + add_to_git_credential=self.args.add_to_git_credential, + ) + + +class AuthLogout(BaseAuthCommand): + def run(self): + logging.set_verbosity_info() + logout(token_name=self.args.token_name) + + +class AuthSwitch(BaseAuthCommand): + def run(self): + logging.set_verbosity_info() + token_name = self.args.token_name + if token_name is None: + token_name = self._select_token_name() + + if token_name is None: + print("No token name provided. Aborting.") + exit() + auth_switch(token_name, add_to_git_credential=self.args.add_to_git_credential) + + def _select_token_name(self) -> Optional[str]: + token_names = list(get_stored_tokens().keys()) + + if not token_names: + logger.error("No stored tokens found. Please login first.") + return None + + if _inquirer_py_available: + return self._select_token_name_tui(token_names) + # if inquirer is not available, use a simpler terminal UI + print("Available stored tokens:") + for i, token_name in enumerate(token_names, 1): + print(f"{i}. {token_name}") + while True: + try: + choice = input("Enter the number of the token to switch to (or 'q' to quit): ") + if choice.lower() == "q": + return None + index = int(choice) - 1 + if 0 <= index < len(token_names): + return token_names[index] + else: + print("Invalid selection. Please try again.") + except ValueError: + print("Invalid input. Please enter a number or 'q' to quit.") + + def _select_token_name_tui(self, token_names: List[str]) -> Optional[str]: + choices = [Choice(token_name, name=token_name) for token_name in token_names] + try: + return inquirer.select( + message="Select a token to switch to:", + choices=choices, + default=None, + ).execute() + except KeyboardInterrupt: + logger.info("Token selection cancelled.") + return None + + +class AuthList(BaseAuthCommand): + def run(self): + logging.set_verbosity_info() + auth_list() + + +class AuthWhoami(BaseAuthCommand): + def run(self): + token = get_token() + if token is None: + print("Not logged in") + exit() + try: + info = self._api.whoami(token) + print(ANSI.bold("user: "), info["name"]) + orgs = [org["name"] for org in info["orgs"]] + if orgs: + print(ANSI.bold("orgs: "), ",".join(orgs)) + + if ENDPOINT != "https://huggingface.co": + print(f"Authenticated through private endpoint: {ENDPOINT}") + except HTTPError as e: + print(e) + print(ANSI.red(e.response.text)) + exit(1) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/cache.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/cache.py new file mode 100644 index 0000000000000000000000000000000000000000..cc36ef5efd2508bcc5e32b1fbe222bb55358777c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/cache.py @@ -0,0 +1,403 @@ +# coding=utf-8 +# Copyright 2025-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains the 'hf cache' command group with 'scan' and 'delete' subcommands.""" + +import os +import time +from argparse import Namespace, _SubParsersAction +from functools import wraps +from tempfile import mkstemp +from typing import Any, Callable, Iterable, List, Literal, Optional, Union + +from ..utils import CachedRepoInfo, CachedRevisionInfo, CacheNotFound, HFCacheInfo, scan_cache_dir +from . import BaseHuggingfaceCLICommand +from ._cli_utils import ANSI, tabulate + + +# --- DELETE helpers (from delete_cache.py) --- +try: + from InquirerPy import inquirer + from InquirerPy.base.control import Choice + from InquirerPy.separator import Separator + + _inquirer_py_available = True +except ImportError: + _inquirer_py_available = False + +SortingOption_T = Literal["alphabetical", "lastUpdated", "lastUsed", "size"] +_CANCEL_DELETION_STR = "CANCEL_DELETION" + + +def require_inquirer_py(fn: Callable) -> Callable: + @wraps(fn) + def _inner(*args, **kwargs): + if not _inquirer_py_available: + raise ImportError( + "The 'cache delete' command requires extra dependencies for the TUI.\n" + "Please run 'pip install \"huggingface_hub[cli]\"' to install them.\n" + "Otherwise, disable TUI using the '--disable-tui' flag." + ) + return fn(*args, **kwargs) + + return _inner + + +class CacheCommand(BaseHuggingfaceCLICommand): + @staticmethod + def register_subcommand(parser: _SubParsersAction): + cache_parser = parser.add_parser("cache", help="Manage local cache directory.") + cache_subparsers = cache_parser.add_subparsers(dest="cache_command", help="Cache subcommands") + + # Show help if no subcommand is provided + cache_parser.set_defaults(func=lambda args: cache_parser.print_help()) + + # Scan subcommand + scan_parser = cache_subparsers.add_parser("scan", help="Scan cache directory.") + scan_parser.add_argument( + "--dir", + type=str, + default=None, + help="cache directory to scan (optional). Default to the default HuggingFace cache.", + ) + scan_parser.add_argument( + "-v", + "--verbose", + action="count", + default=0, + help="show a more verbose output", + ) + scan_parser.set_defaults(func=CacheCommand, cache_command="scan") + # Delete subcommand + delete_parser = cache_subparsers.add_parser("delete", help="Delete revisions from the cache directory.") + delete_parser.add_argument( + "--dir", + type=str, + default=None, + help="cache directory (optional). Default to the default HuggingFace cache.", + ) + delete_parser.add_argument( + "--disable-tui", + action="store_true", + help=( + "Disable Terminal User Interface (TUI) mode. Useful if your platform/terminal doesn't support the multiselect menu." + ), + ) + delete_parser.add_argument( + "--sort", + nargs="?", + choices=["alphabetical", "lastUpdated", "lastUsed", "size"], + help=( + "Sort repositories by the specified criteria. Options: " + "'alphabetical' (A-Z), " + "'lastUpdated' (newest first), " + "'lastUsed' (most recent first), " + "'size' (largest first)." + ), + ) + delete_parser.set_defaults(func=CacheCommand, cache_command="delete") + + def __init__(self, args: Namespace) -> None: + self.args = args + self.verbosity: int = getattr(args, "verbose", 0) + self.cache_dir: Optional[str] = getattr(args, "dir", None) + self.disable_tui: bool = getattr(args, "disable_tui", False) + self.sort_by: Optional[SortingOption_T] = getattr(args, "sort", None) + self.cache_command: Optional[str] = getattr(args, "cache_command", None) + + def run(self): + if self.cache_command == "scan": + self._run_scan() + elif self.cache_command == "delete": + self._run_delete() + else: + print("Please specify a cache subcommand (scan or delete). Use -h for help.") + + def _run_scan(self): + try: + t0 = time.time() + hf_cache_info = scan_cache_dir(self.cache_dir) + t1 = time.time() + except CacheNotFound as exc: + cache_dir = exc.cache_dir + print(f"Cache directory not found: {cache_dir}") + return + print(get_table(hf_cache_info, verbosity=self.verbosity)) + print( + f"\nDone in {round(t1 - t0, 1)}s. Scanned {len(hf_cache_info.repos)} repo(s)" + f" for a total of {ANSI.red(hf_cache_info.size_on_disk_str)}." + ) + if len(hf_cache_info.warnings) > 0: + message = f"Got {len(hf_cache_info.warnings)} warning(s) while scanning." + if self.verbosity >= 3: + print(ANSI.gray(message)) + for warning in hf_cache_info.warnings: + print(ANSI.gray(str(warning))) + else: + print(ANSI.gray(message + " Use -vvv to print details.")) + + def _run_delete(self): + hf_cache_info = scan_cache_dir(self.cache_dir) + if self.disable_tui: + selected_hashes = _manual_review_no_tui(hf_cache_info, preselected=[], sort_by=self.sort_by) + else: + selected_hashes = _manual_review_tui(hf_cache_info, preselected=[], sort_by=self.sort_by) + if len(selected_hashes) > 0 and _CANCEL_DELETION_STR not in selected_hashes: + confirm_message = _get_expectations_str(hf_cache_info, selected_hashes) + " Confirm deletion ?" + if self.disable_tui: + confirmed = _ask_for_confirmation_no_tui(confirm_message) + else: + confirmed = _ask_for_confirmation_tui(confirm_message) + if confirmed: + strategy = hf_cache_info.delete_revisions(*selected_hashes) + print("Start deletion.") + strategy.execute() + print( + f"Done. Deleted {len(strategy.repos)} repo(s) and" + f" {len(strategy.snapshots)} revision(s) for a total of" + f" {strategy.expected_freed_size_str}." + ) + return + print("Deletion is cancelled. Do nothing.") + + +def get_table(hf_cache_info: HFCacheInfo, *, verbosity: int = 0) -> str: + if verbosity == 0: + return tabulate( + rows=[ + [ + repo.repo_id, + repo.repo_type, + "{:>12}".format(repo.size_on_disk_str), + repo.nb_files, + repo.last_accessed_str, + repo.last_modified_str, + ", ".join(sorted(repo.refs)), + str(repo.repo_path), + ] + for repo in sorted(hf_cache_info.repos, key=lambda repo: repo.repo_path) + ], + headers=[ + "REPO ID", + "REPO TYPE", + "SIZE ON DISK", + "NB FILES", + "LAST_ACCESSED", + "LAST_MODIFIED", + "REFS", + "LOCAL PATH", + ], + ) + else: + return tabulate( + rows=[ + [ + repo.repo_id, + repo.repo_type, + revision.commit_hash, + "{:>12}".format(revision.size_on_disk_str), + revision.nb_files, + revision.last_modified_str, + ", ".join(sorted(revision.refs)), + str(revision.snapshot_path), + ] + for repo in sorted(hf_cache_info.repos, key=lambda repo: repo.repo_path) + for revision in sorted(repo.revisions, key=lambda revision: revision.commit_hash) + ], + headers=[ + "REPO ID", + "REPO TYPE", + "REVISION", + "SIZE ON DISK", + "NB FILES", + "LAST_MODIFIED", + "REFS", + "LOCAL PATH", + ], + ) + + +def _get_repo_sorting_key(repo: CachedRepoInfo, sort_by: Optional[SortingOption_T] = None): + if sort_by == "alphabetical": + return (repo.repo_type, repo.repo_id.lower()) + elif sort_by == "lastUpdated": + return -max(rev.last_modified for rev in repo.revisions) + elif sort_by == "lastUsed": + return -repo.last_accessed + elif sort_by == "size": + return -repo.size_on_disk + else: + return (repo.repo_type, repo.repo_id) + + +@require_inquirer_py +def _manual_review_tui( + hf_cache_info: HFCacheInfo, preselected: List[str], sort_by: Optional[SortingOption_T] = None +) -> List[str]: + choices = _get_tui_choices_from_scan(repos=hf_cache_info.repos, preselected=preselected, sort_by=sort_by) + checkbox = inquirer.checkbox( + message="Select revisions to delete:", + choices=choices, + cycle=False, + height=100, + instruction=_get_expectations_str( + hf_cache_info, selected_hashes=[c.value for c in choices if isinstance(c, Choice) and c.enabled] + ), + long_instruction="Press to select, to validate and to quit without modification.", + transformer=lambda result: f"{len(result)} revision(s) selected.", + ) + + def _update_expectations(_): + checkbox._instruction = _get_expectations_str( + hf_cache_info, + selected_hashes=[choice["value"] for choice in checkbox.content_control.choices if choice["enabled"]], + ) + + checkbox.kb_func_lookup["toggle"].append({"func": _update_expectations}) + try: + return checkbox.execute() + except KeyboardInterrupt: + return [] + + +@require_inquirer_py +def _ask_for_confirmation_tui(message: str, default: bool = True) -> bool: + return inquirer.confirm(message, default=default).execute() + + +def _get_tui_choices_from_scan( + repos: Iterable[CachedRepoInfo], preselected: List[str], sort_by: Optional[SortingOption_T] = None +) -> List: + choices: List[Union["Choice", "Separator"]] = [] + choices.append( + Choice( + _CANCEL_DELETION_STR, name="None of the following (if selected, nothing will be deleted).", enabled=False + ) + ) + sorted_repos = sorted(repos, key=lambda repo: _get_repo_sorting_key(repo, sort_by)) + for repo in sorted_repos: + choices.append( + Separator( + f"\n{repo.repo_type.capitalize()} {repo.repo_id} ({repo.size_on_disk_str}, used {repo.last_accessed_str})" + ) + ) + for revision in sorted(repo.revisions, key=_revision_sorting_order): + choices.append( + Choice( + revision.commit_hash, + name=( + f"{revision.commit_hash[:8]}: {', '.join(sorted(revision.refs)) or '(detached)'} # modified {revision.last_modified_str}" + ), + enabled=revision.commit_hash in preselected, + ) + ) + return choices + + +def _manual_review_no_tui( + hf_cache_info: HFCacheInfo, preselected: List[str], sort_by: Optional[SortingOption_T] = None +) -> List[str]: + fd, tmp_path = mkstemp(suffix=".txt") + os.close(fd) + lines = [] + sorted_repos = sorted(hf_cache_info.repos, key=lambda repo: _get_repo_sorting_key(repo, sort_by)) + for repo in sorted_repos: + lines.append( + f"\n# {repo.repo_type.capitalize()} {repo.repo_id} ({repo.size_on_disk_str}, used {repo.last_accessed_str})" + ) + for revision in sorted(repo.revisions, key=_revision_sorting_order): + lines.append( + f"{'' if revision.commit_hash in preselected else '#'} {revision.commit_hash} # Refs: {', '.join(sorted(revision.refs)) or '(detached)'} # modified {revision.last_modified_str}" + ) + with open(tmp_path, "w") as f: + f.write(_MANUAL_REVIEW_NO_TUI_INSTRUCTIONS) + f.write("\n".join(lines)) + instructions = f""" + TUI is disabled. In order to select which revisions you want to delete, please edit + the following file using the text editor of your choice. Instructions for manual + editing are located at the beginning of the file. Edit the file, save it and confirm + to continue. + File to edit: {ANSI.bold(tmp_path)} + """ + print("\n".join(line.strip() for line in instructions.strip().split("\n"))) + while True: + selected_hashes = _read_manual_review_tmp_file(tmp_path) + if _ask_for_confirmation_no_tui( + _get_expectations_str(hf_cache_info, selected_hashes) + " Continue ?", default=False + ): + break + os.remove(tmp_path) + return sorted(selected_hashes) + + +def _ask_for_confirmation_no_tui(message: str, default: bool = True) -> bool: + YES = ("y", "yes", "1") + NO = ("n", "no", "0") + DEFAULT = "" + ALL = YES + NO + (DEFAULT,) + full_message = message + (" (Y/n) " if default else " (y/N) ") + while True: + answer = input(full_message).lower() + if answer == DEFAULT: + return default + if answer in YES: + return True + if answer in NO: + return False + print(f"Invalid input. Must be one of {ALL}") + + +def _get_expectations_str(hf_cache_info: HFCacheInfo, selected_hashes: List[str]) -> str: + if _CANCEL_DELETION_STR in selected_hashes: + return "Nothing will be deleted." + strategy = hf_cache_info.delete_revisions(*selected_hashes) + return f"{len(selected_hashes)} revisions selected counting for {strategy.expected_freed_size_str}." + + +def _read_manual_review_tmp_file(tmp_path: str) -> List[str]: + with open(tmp_path) as f: + content = f.read() + lines = [line.strip() for line in content.split("\n")] + selected_lines = [line for line in lines if not line.startswith("#")] + selected_hashes = [line.split("#")[0].strip() for line in selected_lines] + return [hash for hash in selected_hashes if len(hash) > 0] + + +_MANUAL_REVIEW_NO_TUI_INSTRUCTIONS = f""" +# INSTRUCTIONS +# ------------ +# This is a temporary file created by running `hf cache delete --disable-tui`. It contains a set of revisions that can be deleted from your local cache directory. +# +# Please manually review the revisions you want to delete: +# - Revision hashes can be commented out with '#'. +# - Only non-commented revisions in this file will be deleted. +# - Revision hashes that are removed from this file are ignored as well. +# - If `{_CANCEL_DELETION_STR}` line is uncommented, the all cache deletion is cancelled and no changes will be applied. +# +# Once you've manually reviewed this file, please confirm deletion in the terminal. This file will be automatically removed once done. +# ------------ + +# KILL SWITCH +# ------------ +# Un-comment following line to completely cancel the deletion process +# {_CANCEL_DELETION_STR} +# ------------ + +# REVISIONS +# ------------ +""".strip() + + +def _revision_sorting_order(revision: CachedRevisionInfo) -> Any: + return revision.last_modified diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/download.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/download.py new file mode 100644 index 0000000000000000000000000000000000000000..2660644e62955952f010701a823d7a8bdce1803b --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/download.py @@ -0,0 +1,181 @@ +# coding=utf-8 +# Copyright 202-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains command to download files from the Hub with the CLI. + +Usage: + hf download --help + + # Download file + hf download gpt2 config.json + + # Download entire repo + hf download fffiloni/zeroscope --repo-type=space --revision=refs/pr/78 + + # Download repo with filters + hf download gpt2 --include="*.safetensors" + + # Download with token + hf download Wauplin/private-model --token=hf_*** + + # Download quietly (no progress bar, no warnings, only the returned path) + hf download gpt2 config.json --quiet + + # Download to local dir + hf download gpt2 --local-dir=./models/gpt2 +""" + +import warnings +from argparse import Namespace, _SubParsersAction +from typing import List, Optional + +from huggingface_hub import logging +from huggingface_hub._snapshot_download import snapshot_download +from huggingface_hub.commands import BaseHuggingfaceCLICommand +from huggingface_hub.file_download import hf_hub_download +from huggingface_hub.utils import disable_progress_bars, enable_progress_bars + + +logger = logging.get_logger(__name__) + + +class DownloadCommand(BaseHuggingfaceCLICommand): + @staticmethod + def register_subcommand(parser: _SubParsersAction): + download_parser = parser.add_parser("download", help="Download files from the Hub") + download_parser.add_argument( + "repo_id", type=str, help="ID of the repo to download from (e.g. `username/repo-name`)." + ) + download_parser.add_argument( + "filenames", type=str, nargs="*", help="Files to download (e.g. `config.json`, `data/metadata.jsonl`)." + ) + download_parser.add_argument( + "--repo-type", + choices=["model", "dataset", "space"], + default="model", + help="Type of repo to download from (defaults to 'model').", + ) + download_parser.add_argument( + "--revision", + type=str, + help="An optional Git revision id which can be a branch name, a tag, or a commit hash.", + ) + download_parser.add_argument( + "--include", nargs="*", type=str, help="Glob patterns to match files to download." + ) + download_parser.add_argument( + "--exclude", nargs="*", type=str, help="Glob patterns to exclude from files to download." + ) + download_parser.add_argument( + "--cache-dir", type=str, help="Path to the directory where to save the downloaded files." + ) + download_parser.add_argument( + "--local-dir", + type=str, + help=( + "If set, the downloaded file will be placed under this directory. Check out" + " https://huggingface.co/docs/huggingface_hub/guides/download#download-files-to-local-folder for more" + " details." + ), + ) + download_parser.add_argument( + "--force-download", + action="store_true", + help="If True, the files will be downloaded even if they are already cached.", + ) + download_parser.add_argument( + "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens" + ) + download_parser.add_argument( + "--quiet", + action="store_true", + help="If True, progress bars are disabled and only the path to the download files is printed.", + ) + download_parser.add_argument( + "--max-workers", + type=int, + default=8, + help="Maximum number of workers to use for downloading files. Default is 8.", + ) + download_parser.set_defaults(func=DownloadCommand) + + def __init__(self, args: Namespace) -> None: + self.token = args.token + self.repo_id: str = args.repo_id + self.filenames: List[str] = args.filenames + self.repo_type: str = args.repo_type + self.revision: Optional[str] = args.revision + self.include: Optional[List[str]] = args.include + self.exclude: Optional[List[str]] = args.exclude + self.cache_dir: Optional[str] = args.cache_dir + self.local_dir: Optional[str] = args.local_dir + self.force_download: bool = args.force_download + self.quiet: bool = args.quiet + self.max_workers: int = args.max_workers + + def run(self) -> None: + if self.quiet: + disable_progress_bars() + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + print(self._download()) # Print path to downloaded files + enable_progress_bars() + else: + logging.set_verbosity_info() + print(self._download()) # Print path to downloaded files + logging.set_verbosity_warning() + + def _download(self) -> str: + # Warn user if patterns are ignored + if len(self.filenames) > 0: + if self.include is not None and len(self.include) > 0: + warnings.warn("Ignoring `--include` since filenames have being explicitly set.") + if self.exclude is not None and len(self.exclude) > 0: + warnings.warn("Ignoring `--exclude` since filenames have being explicitly set.") + + # Single file to download: use `hf_hub_download` + if len(self.filenames) == 1: + return hf_hub_download( + repo_id=self.repo_id, + repo_type=self.repo_type, + revision=self.revision, + filename=self.filenames[0], + cache_dir=self.cache_dir, + force_download=self.force_download, + token=self.token, + local_dir=self.local_dir, + library_name="huggingface-cli", + ) + + # Otherwise: use `snapshot_download` to ensure all files comes from same revision + elif len(self.filenames) == 0: + allow_patterns = self.include + ignore_patterns = self.exclude + else: + allow_patterns = self.filenames + ignore_patterns = None + + return snapshot_download( + repo_id=self.repo_id, + repo_type=self.repo_type, + revision=self.revision, + allow_patterns=allow_patterns, + ignore_patterns=ignore_patterns, + force_download=self.force_download, + cache_dir=self.cache_dir, + token=self.token, + local_dir=self.local_dir, + library_name="huggingface-cli", + max_workers=self.max_workers, + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/hf.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/hf.py new file mode 100644 index 0000000000000000000000000000000000000000..2587918b294b427fb8f3e0f990884826b66514a8 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/hf.py @@ -0,0 +1,63 @@ +# Copyright 2020 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from argparse import ArgumentParser + +from huggingface_hub.cli.auth import AuthCommands +from huggingface_hub.cli.cache import CacheCommand +from huggingface_hub.cli.download import DownloadCommand +from huggingface_hub.cli.jobs import JobsCommands +from huggingface_hub.cli.lfs import LfsCommands +from huggingface_hub.cli.repo import RepoCommands +from huggingface_hub.cli.repo_files import RepoFilesCommand +from huggingface_hub.cli.system import EnvironmentCommand, VersionCommand +from huggingface_hub.cli.upload import UploadCommand +from huggingface_hub.cli.upload_large_folder import UploadLargeFolderCommand + + +def main(): + parser = ArgumentParser("hf", usage="hf []") + commands_parser = parser.add_subparsers(help="hf command helpers") + + # Register commands + AuthCommands.register_subcommand(commands_parser) + CacheCommand.register_subcommand(commands_parser) + DownloadCommand.register_subcommand(commands_parser) + JobsCommands.register_subcommand(commands_parser) + RepoCommands.register_subcommand(commands_parser) + RepoFilesCommand.register_subcommand(commands_parser) + UploadCommand.register_subcommand(commands_parser) + UploadLargeFolderCommand.register_subcommand(commands_parser) + + # System commands + EnvironmentCommand.register_subcommand(commands_parser) + VersionCommand.register_subcommand(commands_parser) + + # LFS commands (hidden in --help) + LfsCommands.register_subcommand(commands_parser) + + # Let's go + args = parser.parse_args() + if not hasattr(args, "func"): + parser.print_help() + exit(1) + + # Run + service = args.func(args) + if service is not None: + service.run() + + +if __name__ == "__main__": + main() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/jobs.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/jobs.py new file mode 100644 index 0000000000000000000000000000000000000000..3a661c7df7d65813dbb1b2a8f449ca8410e320e0 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/jobs.py @@ -0,0 +1,1100 @@ +# Copyright 2025 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains commands to interact with jobs on the Hugging Face Hub. + +Usage: + # run a job + hf jobs run + + # List running or completed jobs + hf jobs ps [-a] [-f key=value] [--format TEMPLATE] + + # Stream logs from a job + hf jobs logs + + # Inspect detailed information about a job + hf jobs inspect + + # Cancel a running job + hf jobs cancel +""" + +import json +import os +import re +from argparse import Namespace, _SubParsersAction +from dataclasses import asdict +from pathlib import Path +from typing import Dict, List, Optional, Union + +import requests + +from huggingface_hub import HfApi, SpaceHardware, get_token +from huggingface_hub.utils import logging +from huggingface_hub.utils._dotenv import load_dotenv + +from . import BaseHuggingfaceCLICommand + + +logger = logging.get_logger(__name__) + +SUGGESTED_FLAVORS = [item.value for item in SpaceHardware if item.value != "zero-a10g"] + + +class JobsCommands(BaseHuggingfaceCLICommand): + @staticmethod + def register_subcommand(parser: _SubParsersAction): + jobs_parser = parser.add_parser("jobs", help="Run and manage Jobs on the Hub.") + jobs_subparsers = jobs_parser.add_subparsers(help="huggingface.co jobs related commands") + + # Show help if no subcommand is provided + jobs_parser.set_defaults(func=lambda args: jobs_parser.print_help()) + + # Register commands + InspectCommand.register_subcommand(jobs_subparsers) + LogsCommand.register_subcommand(jobs_subparsers) + PsCommand.register_subcommand(jobs_subparsers) + RunCommand.register_subcommand(jobs_subparsers) + CancelCommand.register_subcommand(jobs_subparsers) + UvCommand.register_subcommand(jobs_subparsers) + ScheduledJobsCommands.register_subcommand(jobs_subparsers) + + +class RunCommand(BaseHuggingfaceCLICommand): + @staticmethod + def register_subcommand(parser: _SubParsersAction) -> None: + run_parser = parser.add_parser("run", help="Run a Job") + run_parser.add_argument("image", type=str, help="The Docker image to use.") + run_parser.add_argument("-e", "--env", action="append", help="Set environment variables. E.g. --env ENV=value") + run_parser.add_argument( + "-s", + "--secrets", + action="append", + help=( + "Set secret environment variables. E.g. --secrets SECRET=value " + "or `--secrets HF_TOKEN` to pass your Hugging Face token." + ), + ) + run_parser.add_argument("--env-file", type=str, help="Read in a file of environment variables.") + run_parser.add_argument("--secrets-file", type=str, help="Read in a file of secret environment variables.") + run_parser.add_argument( + "--flavor", + type=str, + help=f"Flavor for the hardware, as in HF Spaces. Defaults to `cpu-basic`. Possible values: {', '.join(SUGGESTED_FLAVORS)}.", + ) + run_parser.add_argument( + "--timeout", + type=str, + help="Max duration: int/float with s (seconds, default), m (minutes), h (hours) or d (days).", + ) + run_parser.add_argument( + "-d", + "--detach", + action="store_true", + help="Run the Job in the background and print the Job ID.", + ) + run_parser.add_argument( + "--namespace", + type=str, + help="The namespace where the Job will be created. Defaults to the current user's namespace.", + ) + run_parser.add_argument( + "--token", + type=str, + help="A User Access Token generated from https://huggingface.co/settings/tokens", + ) + run_parser.add_argument("command", nargs="...", help="The command to run.") + run_parser.set_defaults(func=RunCommand) + + def __init__(self, args: Namespace) -> None: + self.image: str = args.image + self.command: List[str] = args.command + self.env: dict[str, Optional[str]] = {} + if args.env_file: + self.env.update(load_dotenv(Path(args.env_file).read_text(), environ=os.environ.copy())) + for env_value in args.env or []: + self.env.update(load_dotenv(env_value, environ=os.environ.copy())) + self.secrets: dict[str, Optional[str]] = {} + extended_environ = _get_extended_environ() + if args.secrets_file: + self.secrets.update(load_dotenv(Path(args.secrets_file).read_text(), environ=extended_environ)) + for secret in args.secrets or []: + self.secrets.update(load_dotenv(secret, environ=extended_environ)) + self.flavor: Optional[SpaceHardware] = args.flavor + self.timeout: Optional[str] = args.timeout + self.detach: bool = args.detach + self.namespace: Optional[str] = args.namespace + self.token: Optional[str] = args.token + + def run(self) -> None: + api = HfApi(token=self.token) + job = api.run_job( + image=self.image, + command=self.command, + env=self.env, + secrets=self.secrets, + flavor=self.flavor, + timeout=self.timeout, + namespace=self.namespace, + ) + # Always print the job ID to the user + print(f"Job started with ID: {job.id}") + print(f"View at: {job.url}") + + if self.detach: + return + + # Now let's stream the logs + for log in api.fetch_job_logs(job_id=job.id): + print(log) + + +class LogsCommand(BaseHuggingfaceCLICommand): + @staticmethod + def register_subcommand(parser: _SubParsersAction) -> None: + run_parser = parser.add_parser("logs", help="Fetch the logs of a Job") + run_parser.add_argument("job_id", type=str, help="Job ID") + run_parser.add_argument( + "--namespace", + type=str, + help="The namespace where the job is running. Defaults to the current user's namespace.", + ) + run_parser.add_argument( + "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens" + ) + run_parser.set_defaults(func=LogsCommand) + + def __init__(self, args: Namespace) -> None: + self.job_id: str = args.job_id + self.namespace: Optional[str] = args.namespace + self.token: Optional[str] = args.token + + def run(self) -> None: + api = HfApi(token=self.token) + for log in api.fetch_job_logs(job_id=self.job_id, namespace=self.namespace): + print(log) + + +def _tabulate(rows: List[List[Union[str, int]]], headers: List[str]) -> str: + """ + Inspired by: + + - stackoverflow.com/a/8356620/593036 + - stackoverflow.com/questions/9535954/printing-lists-as-tabular-data + """ + col_widths = [max(len(str(x)) for x in col) for col in zip(*rows, headers)] + terminal_width = max(os.get_terminal_size().columns, len(headers) * 12) + while len(headers) + sum(col_widths) > terminal_width: + col_to_minimize = col_widths.index(max(col_widths)) + col_widths[col_to_minimize] //= 2 + if len(headers) + sum(col_widths) <= terminal_width: + col_widths[col_to_minimize] = terminal_width - sum(col_widths) - len(headers) + col_widths[col_to_minimize] + row_format = ("{{:{}}} " * len(headers)).format(*col_widths) + lines = [] + lines.append(row_format.format(*headers)) + lines.append(row_format.format(*["-" * w for w in col_widths])) + for row in rows: + row_format_args = [ + str(x)[: col_width - 3] + "..." if len(str(x)) > col_width else str(x) + for x, col_width in zip(row, col_widths) + ] + lines.append(row_format.format(*row_format_args)) + return "\n".join(lines) + + +class PsCommand(BaseHuggingfaceCLICommand): + @staticmethod + def register_subcommand(parser: _SubParsersAction) -> None: + run_parser = parser.add_parser("ps", help="List Jobs") + run_parser.add_argument( + "-a", + "--all", + action="store_true", + help="Show all Jobs (default shows just running)", + ) + run_parser.add_argument( + "--namespace", + type=str, + help="The namespace from where it lists the jobs. Defaults to the current user's namespace.", + ) + run_parser.add_argument( + "--token", + type=str, + help="A User Access Token generated from https://huggingface.co/settings/tokens", + ) + # Add Docker-style filtering argument + run_parser.add_argument( + "-f", + "--filter", + action="append", + default=[], + help="Filter output based on conditions provided (format: key=value)", + ) + # Add option to format output + run_parser.add_argument( + "--format", + type=str, + help="Format output using a custom template", + ) + run_parser.set_defaults(func=PsCommand) + + def __init__(self, args: Namespace) -> None: + self.all: bool = args.all + self.namespace: Optional[str] = args.namespace + self.token: Optional[str] = args.token + self.format: Optional[str] = args.format + self.filters: Dict[str, str] = {} + + # Parse filter arguments (key=value pairs) + for f in args.filter: + if "=" in f: + key, value = f.split("=", 1) + self.filters[key.lower()] = value + else: + print(f"Warning: Ignoring invalid filter format '{f}'. Use key=value format.") + + def run(self) -> None: + """ + Fetch and display job information for the current user. + Uses Docker-style filtering with -f/--filter flag and key=value pairs. + """ + try: + api = HfApi(token=self.token) + + # Fetch jobs data + jobs = api.list_jobs(namespace=self.namespace) + + # Define table headers + table_headers = ["JOB ID", "IMAGE/SPACE", "COMMAND", "CREATED", "STATUS"] + + # Process jobs data + rows = [] + + for job in jobs: + # Extract job data for filtering + status = job.status.stage if job.status else "UNKNOWN" + + # Skip job if not all jobs should be shown and status doesn't match criteria + if not self.all and status not in ("RUNNING", "UPDATING"): + continue + + # Extract job ID + job_id = job.id + + # Extract image or space information + image_or_space = job.docker_image or "N/A" + + # Extract and format command + command = job.command or [] + command_str = " ".join(command) if command else "N/A" + + # Extract creation time + created_at = job.created_at.strftime("%Y-%m-%d %H:%M:%S") if job.created_at else "N/A" + + # Create a dict with all job properties for filtering + job_properties = { + "id": job_id, + "image": image_or_space, + "status": status.lower(), + "command": command_str, + } + + # Check if job matches all filters + if not self._matches_filters(job_properties): + continue + + # Create row + rows.append([job_id, image_or_space, command_str, created_at, status]) + + # Handle empty results + if not rows: + filters_msg = "" + if self.filters: + filters_msg = f" matching filters: {', '.join([f'{k}={v}' for k, v in self.filters.items()])}" + + print(f"No jobs found{filters_msg}") + return + + # Apply custom format if provided or use default tabular format + self._print_output(rows, table_headers) + + except requests.RequestException as e: + print(f"Error fetching jobs data: {e}") + except (KeyError, ValueError, TypeError) as e: + print(f"Error processing jobs data: {e}") + except Exception as e: + print(f"Unexpected error - {type(e).__name__}: {e}") + + def _matches_filters(self, job_properties: Dict[str, str]) -> bool: + """Check if job matches all specified filters.""" + for key, pattern in self.filters.items(): + # Check if property exists + if key not in job_properties: + return False + + # Support pattern matching with wildcards + if "*" in pattern or "?" in pattern: + # Convert glob pattern to regex + regex_pattern = pattern.replace("*", ".*").replace("?", ".") + if not re.search(f"^{regex_pattern}$", job_properties[key], re.IGNORECASE): + return False + # Simple substring matching + elif pattern.lower() not in job_properties[key].lower(): + return False + + return True + + def _print_output(self, rows, headers): + """Print output according to the chosen format.""" + if self.format: + # Custom template formatting (simplified) + template = self.format + for row in rows: + line = template + for i, field in enumerate(["id", "image", "command", "created", "status"]): + placeholder = f"{{{{.{field}}}}}" + if placeholder in line: + line = line.replace(placeholder, str(row[i])) + print(line) + else: + # Default tabular format + print( + _tabulate( + rows, + headers=headers, + ) + ) + + +class InspectCommand(BaseHuggingfaceCLICommand): + @staticmethod + def register_subcommand(parser: _SubParsersAction) -> None: + run_parser = parser.add_parser("inspect", help="Display detailed information on one or more Jobs") + run_parser.add_argument( + "--namespace", + type=str, + help="The namespace where the job is running. Defaults to the current user's namespace.", + ) + run_parser.add_argument( + "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens" + ) + run_parser.add_argument("job_ids", nargs="...", help="The jobs to inspect") + run_parser.set_defaults(func=InspectCommand) + + def __init__(self, args: Namespace) -> None: + self.namespace: Optional[str] = args.namespace + self.token: Optional[str] = args.token + self.job_ids: List[str] = args.job_ids + + def run(self) -> None: + api = HfApi(token=self.token) + jobs = [api.inspect_job(job_id=job_id, namespace=self.namespace) for job_id in self.job_ids] + print(json.dumps([asdict(job) for job in jobs], indent=4, default=str)) + + +class CancelCommand(BaseHuggingfaceCLICommand): + @staticmethod + def register_subcommand(parser: _SubParsersAction) -> None: + run_parser = parser.add_parser("cancel", help="Cancel a Job") + run_parser.add_argument("job_id", type=str, help="Job ID") + run_parser.add_argument( + "--namespace", + type=str, + help="The namespace where the job is running. Defaults to the current user's namespace.", + ) + run_parser.add_argument( + "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens" + ) + run_parser.set_defaults(func=CancelCommand) + + def __init__(self, args: Namespace) -> None: + self.job_id: str = args.job_id + self.namespace = args.namespace + self.token: Optional[str] = args.token + + def run(self) -> None: + api = HfApi(token=self.token) + api.cancel_job(job_id=self.job_id, namespace=self.namespace) + + +class UvCommand(BaseHuggingfaceCLICommand): + """Run UV scripts on Hugging Face infrastructure.""" + + @staticmethod + def register_subcommand(parser): + """Register UV run subcommand.""" + uv_parser = parser.add_parser( + "uv", + help="Run UV scripts (Python with inline dependencies) on HF infrastructure", + ) + + subparsers = uv_parser.add_subparsers(dest="uv_command", help="UV commands", required=True) + + # Run command only + run_parser = subparsers.add_parser( + "run", + help="Run a UV script (local file or URL) on HF infrastructure", + ) + run_parser.add_argument("script", help="UV script to run (local file or URL)") + run_parser.add_argument("script_args", nargs="...", help="Arguments for the script", default=[]) + run_parser.add_argument("--image", type=str, help="Use a custom Docker image with `uv` installed.") + run_parser.add_argument( + "--repo", + help="Repository name for the script (creates ephemeral if not specified)", + ) + run_parser.add_argument( + "--flavor", + type=str, + help=f"Flavor for the hardware, as in HF Spaces. Defaults to `cpu-basic`. Possible values: {', '.join(SUGGESTED_FLAVORS)}.", + ) + run_parser.add_argument("-e", "--env", action="append", help="Environment variables") + run_parser.add_argument( + "-s", + "--secrets", + action="append", + help=( + "Set secret environment variables. E.g. --secrets SECRET=value " + "or `--secrets HF_TOKEN` to pass your Hugging Face token." + ), + ) + run_parser.add_argument("--env-file", type=str, help="Read in a file of environment variables.") + run_parser.add_argument( + "--secrets-file", + type=str, + help="Read in a file of secret environment variables.", + ) + run_parser.add_argument("--timeout", type=str, help="Max duration (e.g., 30s, 5m, 1h)") + run_parser.add_argument("-d", "--detach", action="store_true", help="Run in background") + run_parser.add_argument( + "--namespace", + type=str, + help="The namespace where the Job will be created. Defaults to the current user's namespace.", + ) + run_parser.add_argument("--token", type=str, help="HF token") + # UV options + run_parser.add_argument("--with", action="append", help="Run with the given packages installed", dest="with_") + run_parser.add_argument( + "-p", "--python", type=str, help="The Python interpreter to use for the run environment" + ) + run_parser.set_defaults(func=UvCommand) + + def __init__(self, args: Namespace) -> None: + """Initialize the command with parsed arguments.""" + self.script = args.script + self.script_args = args.script_args + self.dependencies = args.with_ + self.python = args.python + self.image = args.image + self.env: dict[str, Optional[str]] = {} + if args.env_file: + self.env.update(load_dotenv(Path(args.env_file).read_text(), environ=os.environ.copy())) + for env_value in args.env or []: + self.env.update(load_dotenv(env_value, environ=os.environ.copy())) + self.secrets: dict[str, Optional[str]] = {} + extended_environ = _get_extended_environ() + if args.secrets_file: + self.secrets.update(load_dotenv(Path(args.secrets_file).read_text(), environ=extended_environ)) + for secret in args.secrets or []: + self.secrets.update(load_dotenv(secret, environ=extended_environ)) + self.flavor: Optional[SpaceHardware] = args.flavor + self.timeout: Optional[str] = args.timeout + self.detach: bool = args.detach + self.namespace: Optional[str] = args.namespace + self.token: Optional[str] = args.token + self._repo = args.repo + + def run(self) -> None: + """Execute UV command.""" + logging.set_verbosity(logging.INFO) + api = HfApi(token=self.token) + job = api.run_uv_job( + script=self.script, + script_args=self.script_args, + dependencies=self.dependencies, + python=self.python, + image=self.image, + env=self.env, + secrets=self.secrets, + flavor=self.flavor, + timeout=self.timeout, + namespace=self.namespace, + _repo=self._repo, + ) + + # Always print the job ID to the user + print(f"Job started with ID: {job.id}") + print(f"View at: {job.url}") + + if self.detach: + return + + # Now let's stream the logs + for log in api.fetch_job_logs(job_id=job.id): + print(log) + + +def _get_extended_environ() -> Dict[str, str]: + extended_environ = os.environ.copy() + if (token := get_token()) is not None: + extended_environ["HF_TOKEN"] = token + return extended_environ + + +class ScheduledJobsCommands(BaseHuggingfaceCLICommand): + @staticmethod + def register_subcommand(parser: _SubParsersAction): + scheduled_jobs_parser = parser.add_parser("scheduled", help="Create and manage scheduled Jobs on the Hub.") + scheduled_jobs_subparsers = scheduled_jobs_parser.add_subparsers( + help="huggingface.co scheduled jobs related commands" + ) + + # Show help if no subcommand is provided + scheduled_jobs_parser.set_defaults(func=lambda args: scheduled_jobs_subparsers.print_help()) + + # Register commands + ScheduledRunCommand.register_subcommand(scheduled_jobs_subparsers) + ScheduledPsCommand.register_subcommand(scheduled_jobs_subparsers) + ScheduledInspectCommand.register_subcommand(scheduled_jobs_subparsers) + ScheduledDeleteCommand.register_subcommand(scheduled_jobs_subparsers) + ScheduledSuspendCommand.register_subcommand(scheduled_jobs_subparsers) + ScheduledResumeCommand.register_subcommand(scheduled_jobs_subparsers) + ScheduledUvCommand.register_subcommand(scheduled_jobs_subparsers) + + +class ScheduledRunCommand(BaseHuggingfaceCLICommand): + @staticmethod + def register_subcommand(parser: _SubParsersAction) -> None: + run_parser = parser.add_parser("run", help="Schedule a Job") + run_parser.add_argument( + "schedule", + type=str, + help="One of annually, yearly, monthly, weekly, daily, hourly, or a CRON schedule expression.", + ) + run_parser.add_argument("image", type=str, help="The Docker image to use.") + run_parser.add_argument( + "--suspend", + action="store_true", + help="Suspend (pause) the scheduled Job", + default=None, + ) + run_parser.add_argument( + "--concurrency", + action="store_true", + help="Allow multiple instances of this Job to run concurrently", + default=None, + ) + run_parser.add_argument("-e", "--env", action="append", help="Set environment variables. E.g. --env ENV=value") + run_parser.add_argument( + "-s", + "--secrets", + action="append", + help=( + "Set secret environment variables. E.g. --secrets SECRET=value " + "or `--secrets HF_TOKEN` to pass your Hugging Face token." + ), + ) + run_parser.add_argument("--env-file", type=str, help="Read in a file of environment variables.") + run_parser.add_argument("--secrets-file", type=str, help="Read in a file of secret environment variables.") + run_parser.add_argument( + "--flavor", + type=str, + help=f"Flavor for the hardware, as in HF Spaces. Defaults to `cpu-basic`. Possible values: {', '.join(SUGGESTED_FLAVORS)}.", + ) + run_parser.add_argument( + "--timeout", + type=str, + help="Max duration: int/float with s (seconds, default), m (minutes), h (hours) or d (days).", + ) + run_parser.add_argument( + "--namespace", + type=str, + help="The namespace where the scheduled Job will be created. Defaults to the current user's namespace.", + ) + run_parser.add_argument( + "--token", + type=str, + help="A User Access Token generated from https://huggingface.co/settings/tokens", + ) + run_parser.add_argument("command", nargs="...", help="The command to run.") + run_parser.set_defaults(func=ScheduledRunCommand) + + def __init__(self, args: Namespace) -> None: + self.schedule: str = args.schedule + self.image: str = args.image + self.command: List[str] = args.command + self.suspend: Optional[bool] = args.suspend + self.concurrency: Optional[bool] = args.concurrency + self.env: dict[str, Optional[str]] = {} + if args.env_file: + self.env.update(load_dotenv(Path(args.env_file).read_text(), environ=os.environ.copy())) + for env_value in args.env or []: + self.env.update(load_dotenv(env_value, environ=os.environ.copy())) + self.secrets: dict[str, Optional[str]] = {} + extended_environ = _get_extended_environ() + if args.secrets_file: + self.secrets.update(load_dotenv(Path(args.secrets_file).read_text(), environ=extended_environ)) + for secret in args.secrets or []: + self.secrets.update(load_dotenv(secret, environ=extended_environ)) + self.flavor: Optional[SpaceHardware] = args.flavor + self.timeout: Optional[str] = args.timeout + self.namespace: Optional[str] = args.namespace + self.token: Optional[str] = args.token + + def run(self) -> None: + api = HfApi(token=self.token) + scheduled_job = api.create_scheduled_job( + image=self.image, + command=self.command, + schedule=self.schedule, + suspend=self.suspend, + concurrency=self.concurrency, + env=self.env, + secrets=self.secrets, + flavor=self.flavor, + timeout=self.timeout, + namespace=self.namespace, + ) + # Always print the scheduled job ID to the user + print(f"Scheduled Job created with ID: {scheduled_job.id}") + + +class ScheduledPsCommand(BaseHuggingfaceCLICommand): + @staticmethod + def register_subcommand(parser: _SubParsersAction) -> None: + run_parser = parser.add_parser("ps", help="List scheduled Jobs") + run_parser.add_argument( + "-a", + "--all", + action="store_true", + help="Show all scheduled Jobs (default hides suspended)", + ) + run_parser.add_argument( + "--namespace", + type=str, + help="The namespace from where it lists the jobs. Defaults to the current user's namespace.", + ) + run_parser.add_argument( + "--token", + type=str, + help="A User Access Token generated from https://huggingface.co/settings/tokens", + ) + # Add Docker-style filtering argument + run_parser.add_argument( + "-f", + "--filter", + action="append", + default=[], + help="Filter output based on conditions provided (format: key=value)", + ) + # Add option to format output + run_parser.add_argument( + "--format", + type=str, + help="Format output using a custom template", + ) + run_parser.set_defaults(func=ScheduledPsCommand) + + def __init__(self, args: Namespace) -> None: + self.all: bool = args.all + self.namespace: Optional[str] = args.namespace + self.token: Optional[str] = args.token + self.format: Optional[str] = args.format + self.filters: Dict[str, str] = {} + + # Parse filter arguments (key=value pairs) + for f in args.filter: + if "=" in f: + key, value = f.split("=", 1) + self.filters[key.lower()] = value + else: + print(f"Warning: Ignoring invalid filter format '{f}'. Use key=value format.") + + def run(self) -> None: + """ + Fetch and display scheduked job information for the current user. + Uses Docker-style filtering with -f/--filter flag and key=value pairs. + """ + try: + api = HfApi(token=self.token) + + # Fetch jobs data + scheduled_jobs = api.list_scheduled_jobs(namespace=self.namespace) + + # Define table headers + table_headers = [ + "ID", + "SCHEDULE", + "IMAGE/SPACE", + "COMMAND", + "LAST RUN", + "NEXT RUN", + "SUSPEND", + ] + + # Process jobs data + rows = [] + + for scheduled_job in scheduled_jobs: + # Extract job data for filtering + suspend = scheduled_job.suspend + + # Skip job if not all jobs should be shown and status doesn't match criteria + if not self.all and suspend: + continue + + # Extract job ID + scheduled_job_id = scheduled_job.id + + # Extract schedule + schedule = scheduled_job.schedule + + # Extract image or space information + image_or_space = scheduled_job.job_spec.docker_image or "N/A" + + # Extract and format command + command = scheduled_job.job_spec.command or [] + command_str = " ".join(command) if command else "N/A" + + # Extract status + last_job_at = ( + scheduled_job.status.last_job.at.strftime("%Y-%m-%d %H:%M:%S") + if scheduled_job.status.last_job + else "N/A" + ) + next_job_run_at = ( + scheduled_job.status.next_job_run_at.strftime("%Y-%m-%d %H:%M:%S") + if scheduled_job.status.next_job_run_at + else "N/A" + ) + + # Create a dict with all job properties for filtering + job_properties = { + "id": scheduled_job_id, + "image": image_or_space, + "suspend": str(suspend), + "command": command_str, + } + + # Check if job matches all filters + if not self._matches_filters(job_properties): + continue + + # Create row + rows.append( + [ + scheduled_job_id, + schedule, + image_or_space, + command_str, + last_job_at, + next_job_run_at, + suspend, + ] + ) + + # Handle empty results + if not rows: + filters_msg = "" + if self.filters: + filters_msg = f" matching filters: {', '.join([f'{k}={v}' for k, v in self.filters.items()])}" + + print(f"No scheduled jobs found{filters_msg}") + return + + # Apply custom format if provided or use default tabular format + self._print_output(rows, table_headers) + + except requests.RequestException as e: + print(f"Error fetching scheduled jobs data: {e}") + except (KeyError, ValueError, TypeError) as e: + print(f"Error processing scheduled jobs data: {e}") + except Exception as e: + print(f"Unexpected error - {type(e).__name__}: {e}") + + def _matches_filters(self, job_properties: Dict[str, str]) -> bool: + """Check if scheduled job matches all specified filters.""" + for key, pattern in self.filters.items(): + # Check if property exists + if key not in job_properties: + return False + + # Support pattern matching with wildcards + if "*" in pattern or "?" in pattern: + # Convert glob pattern to regex + regex_pattern = pattern.replace("*", ".*").replace("?", ".") + if not re.search(f"^{regex_pattern}$", job_properties[key], re.IGNORECASE): + return False + # Simple substring matching + elif pattern.lower() not in job_properties[key].lower(): + return False + + return True + + def _print_output(self, rows, headers): + """Print output according to the chosen format.""" + if self.format: + # Custom template formatting (simplified) + template = self.format + for row in rows: + line = template + for i, field in enumerate( + ["id", "schedule", "image", "command", "last_job_at", "next_job_run_at", "suspend"] + ): + placeholder = f"{{{{.{field}}}}}" + if placeholder in line: + line = line.replace(placeholder, str(row[i])) + print(line) + else: + # Default tabular format + print( + _tabulate( + rows, + headers=headers, + ) + ) + + +class ScheduledInspectCommand(BaseHuggingfaceCLICommand): + @staticmethod + def register_subcommand(parser: _SubParsersAction) -> None: + run_parser = parser.add_parser("inspect", help="Display detailed information on one or more scheduled Jobs") + run_parser.add_argument( + "--namespace", + type=str, + help="The namespace where the scheduled job is. Defaults to the current user's namespace.", + ) + run_parser.add_argument( + "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens" + ) + run_parser.add_argument("scheduled_job_ids", nargs="...", help="The scheduled jobs to inspect") + run_parser.set_defaults(func=ScheduledInspectCommand) + + def __init__(self, args: Namespace) -> None: + self.namespace: Optional[str] = args.namespace + self.token: Optional[str] = args.token + self.scheduled_job_ids: List[str] = args.scheduled_job_ids + + def run(self) -> None: + api = HfApi(token=self.token) + scheduled_jobs = [ + api.inspect_scheduled_job(scheduled_job_id=scheduled_job_id, namespace=self.namespace) + for scheduled_job_id in self.scheduled_job_ids + ] + print(json.dumps([asdict(scheduled_job) for scheduled_job in scheduled_jobs], indent=4, default=str)) + + +class ScheduledDeleteCommand(BaseHuggingfaceCLICommand): + @staticmethod + def register_subcommand(parser: _SubParsersAction) -> None: + run_parser = parser.add_parser("delete", help="Delete a scheduled Job") + run_parser.add_argument("scheduled_job_id", type=str, help="Scheduled Job ID") + run_parser.add_argument( + "--namespace", + type=str, + help="The namespace where the scheduled job is. Defaults to the current user's namespace.", + ) + run_parser.add_argument( + "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens" + ) + run_parser.set_defaults(func=ScheduledDeleteCommand) + + def __init__(self, args: Namespace) -> None: + self.scheduled_job_id: str = args.scheduled_job_id + self.namespace = args.namespace + self.token: Optional[str] = args.token + + def run(self) -> None: + api = HfApi(token=self.token) + api.delete_scheduled_job(scheduled_job_id=self.scheduled_job_id, namespace=self.namespace) + + +class ScheduledSuspendCommand(BaseHuggingfaceCLICommand): + @staticmethod + def register_subcommand(parser: _SubParsersAction) -> None: + run_parser = parser.add_parser("suspend", help="Suspend (pause) a scheduled Job") + run_parser.add_argument("scheduled_job_id", type=str, help="Scheduled Job ID") + run_parser.add_argument( + "--namespace", + type=str, + help="The namespace where the scheduled job is. Defaults to the current user's namespace.", + ) + run_parser.add_argument( + "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens" + ) + run_parser.set_defaults(func=ScheduledSuspendCommand) + + def __init__(self, args: Namespace) -> None: + self.scheduled_job_id: str = args.scheduled_job_id + self.namespace = args.namespace + self.token: Optional[str] = args.token + + def run(self) -> None: + api = HfApi(token=self.token) + api.suspend_scheduled_job(scheduled_job_id=self.scheduled_job_id, namespace=self.namespace) + + +class ScheduledResumeCommand(BaseHuggingfaceCLICommand): + @staticmethod + def register_subcommand(parser: _SubParsersAction) -> None: + run_parser = parser.add_parser("resume", help="Resume (unpause) a scheduled Job") + run_parser.add_argument("scheduled_job_id", type=str, help="Scheduled Job ID") + run_parser.add_argument( + "--namespace", + type=str, + help="The namespace where the scheduled job is. Defaults to the current user's namespace.", + ) + run_parser.add_argument( + "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens" + ) + run_parser.set_defaults(func=ScheduledResumeCommand) + + def __init__(self, args: Namespace) -> None: + self.scheduled_job_id: str = args.scheduled_job_id + self.namespace = args.namespace + self.token: Optional[str] = args.token + + def run(self) -> None: + api = HfApi(token=self.token) + api.resume_scheduled_job(scheduled_job_id=self.scheduled_job_id, namespace=self.namespace) + + +class ScheduledUvCommand(BaseHuggingfaceCLICommand): + """Schedule UV scripts on Hugging Face infrastructure.""" + + @staticmethod + def register_subcommand(parser): + """Register UV run subcommand.""" + uv_parser = parser.add_parser( + "uv", + help="Schedule UV scripts (Python with inline dependencies) on HF infrastructure", + ) + + subparsers = uv_parser.add_subparsers(dest="uv_command", help="UV commands", required=True) + + # Run command only + run_parser = subparsers.add_parser( + "run", + help="Run a UV script (local file or URL) on HF infrastructure", + ) + run_parser.add_argument( + "schedule", + type=str, + help="One of annually, yearly, monthly, weekly, daily, hourly, or a CRON schedule expression.", + ) + run_parser.add_argument("script", help="UV script to run (local file or URL)") + run_parser.add_argument("script_args", nargs="...", help="Arguments for the script", default=[]) + run_parser.add_argument( + "--suspend", + action="store_true", + help="Suspend (pause) the scheduled Job", + default=None, + ) + run_parser.add_argument( + "--concurrency", + action="store_true", + help="Allow multiple instances of this Job to run concurrently", + default=None, + ) + run_parser.add_argument("--image", type=str, help="Use a custom Docker image with `uv` installed.") + run_parser.add_argument( + "--repo", + help="Repository name for the script (creates ephemeral if not specified)", + ) + run_parser.add_argument( + "--flavor", + type=str, + help=f"Flavor for the hardware, as in HF Spaces. Defaults to `cpu-basic`. Possible values: {', '.join(SUGGESTED_FLAVORS)}.", + ) + run_parser.add_argument("-e", "--env", action="append", help="Environment variables") + run_parser.add_argument( + "-s", + "--secrets", + action="append", + help=( + "Set secret environment variables. E.g. --secrets SECRET=value " + "or `--secrets HF_TOKEN` to pass your Hugging Face token." + ), + ) + run_parser.add_argument("--env-file", type=str, help="Read in a file of environment variables.") + run_parser.add_argument( + "--secrets-file", + type=str, + help="Read in a file of secret environment variables.", + ) + run_parser.add_argument("--timeout", type=str, help="Max duration (e.g., 30s, 5m, 1h)") + run_parser.add_argument("-d", "--detach", action="store_true", help="Run in background") + run_parser.add_argument( + "--namespace", + type=str, + help="The namespace where the Job will be created. Defaults to the current user's namespace.", + ) + run_parser.add_argument("--token", type=str, help="HF token") + # UV options + run_parser.add_argument("--with", action="append", help="Run with the given packages installed", dest="with_") + run_parser.add_argument( + "-p", "--python", type=str, help="The Python interpreter to use for the run environment" + ) + run_parser.set_defaults(func=ScheduledUvCommand) + + def __init__(self, args: Namespace) -> None: + """Initialize the command with parsed arguments.""" + self.schedule: str = args.schedule + self.script = args.script + self.script_args = args.script_args + self.suspend: Optional[bool] = args.suspend + self.concurrency: Optional[bool] = args.concurrency + self.dependencies = args.with_ + self.python = args.python + self.image = args.image + self.env: dict[str, Optional[str]] = {} + if args.env_file: + self.env.update(load_dotenv(Path(args.env_file).read_text(), environ=os.environ.copy())) + for env_value in args.env or []: + self.env.update(load_dotenv(env_value, environ=os.environ.copy())) + self.secrets: dict[str, Optional[str]] = {} + extended_environ = _get_extended_environ() + if args.secrets_file: + self.secrets.update(load_dotenv(Path(args.secrets_file).read_text(), environ=extended_environ)) + for secret in args.secrets or []: + self.secrets.update(load_dotenv(secret, environ=extended_environ)) + self.flavor: Optional[SpaceHardware] = args.flavor + self.timeout: Optional[str] = args.timeout + self.detach: bool = args.detach + self.namespace: Optional[str] = args.namespace + self.token: Optional[str] = args.token + self._repo = args.repo + + def run(self) -> None: + """Schedule UV command.""" + logging.set_verbosity(logging.INFO) + api = HfApi(token=self.token) + job = api.create_scheduled_uv_job( + script=self.script, + script_args=self.script_args, + schedule=self.schedule, + suspend=self.suspend, + concurrency=self.concurrency, + dependencies=self.dependencies, + python=self.python, + image=self.image, + env=self.env, + secrets=self.secrets, + flavor=self.flavor, + timeout=self.timeout, + namespace=self.namespace, + _repo=self._repo, + ) + + # Always print the job ID to the user + print(f"Scheduled Job created with ID: {job.id}") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/lfs.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/lfs.py new file mode 100644 index 0000000000000000000000000000000000000000..e4c5b900c816494c260f6c440843a2d83703fab5 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/lfs.py @@ -0,0 +1,198 @@ +""" +Implementation of a custom transfer agent for the transfer type "multipart" for +git-lfs. + +Inspired by: +github.com/cbartz/git-lfs-swift-transfer-agent/blob/master/git_lfs_swift_transfer.py + +Spec is: github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md + + +To launch debugger while developing: + +``` [lfs "customtransfer.multipart"] +path = /path/to/huggingface_hub/.env/bin/python args = -m debugpy --listen 5678 +--wait-for-client +/path/to/huggingface_hub/src/huggingface_hub/commands/huggingface_cli.py +lfs-multipart-upload ```""" + +import json +import os +import subprocess +import sys +from argparse import _SubParsersAction +from typing import Dict, List, Optional + +from huggingface_hub.commands import BaseHuggingfaceCLICommand +from huggingface_hub.lfs import LFS_MULTIPART_UPLOAD_COMMAND + +from ..utils import get_session, hf_raise_for_status, logging +from ..utils._lfs import SliceFileObj + + +logger = logging.get_logger(__name__) + + +class LfsCommands(BaseHuggingfaceCLICommand): + """ + Implementation of a custom transfer agent for the transfer type "multipart" + for git-lfs. This lets users upload large files >5GB 🔥. Spec for LFS custom + transfer agent is: + https://github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md + + This introduces two commands to the CLI: + + 1. $ hf lfs-enable-largefiles + + This should be executed once for each model repo that contains a model file + >5GB. It's documented in the error message you get if you just try to git + push a 5GB file without having enabled it before. + + 2. $ hf lfs-multipart-upload + + This command is called by lfs directly and is not meant to be called by the + user. + """ + + @staticmethod + def register_subcommand(parser: _SubParsersAction): + enable_parser = parser.add_parser("lfs-enable-largefiles", add_help=False) + enable_parser.add_argument("path", type=str, help="Local path to repository you want to configure.") + enable_parser.set_defaults(func=lambda args: LfsEnableCommand(args)) + + # Command will get called by git-lfs, do not call it directly. + upload_parser = parser.add_parser(LFS_MULTIPART_UPLOAD_COMMAND, add_help=False) + upload_parser.set_defaults(func=lambda args: LfsUploadCommand(args)) + + +class LfsEnableCommand: + def __init__(self, args): + self.args = args + + def run(self): + local_path = os.path.abspath(self.args.path) + if not os.path.isdir(local_path): + print("This does not look like a valid git repo.") + exit(1) + subprocess.run( + "git config lfs.customtransfer.multipart.path hf".split(), + check=True, + cwd=local_path, + ) + subprocess.run( + f"git config lfs.customtransfer.multipart.args {LFS_MULTIPART_UPLOAD_COMMAND}".split(), + check=True, + cwd=local_path, + ) + print("Local repo set up for largefiles") + + +def write_msg(msg: Dict): + """Write out the message in Line delimited JSON.""" + msg_str = json.dumps(msg) + "\n" + sys.stdout.write(msg_str) + sys.stdout.flush() + + +def read_msg() -> Optional[Dict]: + """Read Line delimited JSON from stdin.""" + msg = json.loads(sys.stdin.readline().strip()) + + if "terminate" in (msg.get("type"), msg.get("event")): + # terminate message received + return None + + if msg.get("event") not in ("download", "upload"): + logger.critical("Received unexpected message") + sys.exit(1) + + return msg + + +class LfsUploadCommand: + def __init__(self, args) -> None: + self.args = args + + def run(self) -> None: + # Immediately after invoking a custom transfer process, git-lfs + # sends initiation data to the process over stdin. + # This tells the process useful information about the configuration. + init_msg = json.loads(sys.stdin.readline().strip()) + if not (init_msg.get("event") == "init" and init_msg.get("operation") == "upload"): + write_msg({"error": {"code": 32, "message": "Wrong lfs init operation"}}) + sys.exit(1) + + # The transfer process should use the information it needs from the + # initiation structure, and also perform any one-off setup tasks it + # needs to do. It should then respond on stdout with a simple empty + # confirmation structure, as follows: + write_msg({}) + + # After the initiation exchange, git-lfs will send any number of + # transfer requests to the stdin of the transfer process, in a serial sequence. + while True: + msg = read_msg() + if msg is None: + # When all transfers have been processed, git-lfs will send + # a terminate event to the stdin of the transfer process. + # On receiving this message the transfer process should + # clean up and terminate. No response is expected. + sys.exit(0) + + oid = msg["oid"] + filepath = msg["path"] + completion_url = msg["action"]["href"] + header = msg["action"]["header"] + chunk_size = int(header.pop("chunk_size")) + presigned_urls: List[str] = list(header.values()) + + # Send a "started" progress event to allow other workers to start. + # Otherwise they're delayed until first "progress" event is reported, + # i.e. after the first 5GB by default (!) + write_msg( + { + "event": "progress", + "oid": oid, + "bytesSoFar": 1, + "bytesSinceLast": 0, + } + ) + + parts = [] + with open(filepath, "rb") as file: + for i, presigned_url in enumerate(presigned_urls): + with SliceFileObj( + file, + seek_from=i * chunk_size, + read_limit=chunk_size, + ) as data: + r = get_session().put(presigned_url, data=data) + hf_raise_for_status(r) + parts.append( + { + "etag": r.headers.get("etag"), + "partNumber": i + 1, + } + ) + # In order to support progress reporting while data is uploading / downloading, + # the transfer process should post messages to stdout + write_msg( + { + "event": "progress", + "oid": oid, + "bytesSoFar": (i + 1) * chunk_size, + "bytesSinceLast": chunk_size, + } + ) + # Not precise but that's ok. + + r = get_session().post( + completion_url, + json={ + "oid": oid, + "parts": parts, + }, + ) + hf_raise_for_status(r) + + write_msg({"event": "complete", "oid": oid}) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/repo.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/repo.py new file mode 100644 index 0000000000000000000000000000000000000000..ef0e3313580e3753a2617745e15762933229b15f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/repo.py @@ -0,0 +1,249 @@ +# Copyright 2025 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains commands to interact with repositories on the Hugging Face Hub. + +Usage: + # create a new dataset repo on the Hub + hf repo create my-cool-dataset --repo-type=dataset + + # create a private model repo on the Hub + hf repo create my-cool-model --private +""" + +import argparse +from argparse import _SubParsersAction +from typing import Optional + +from requests.exceptions import HTTPError + +from huggingface_hub.commands import BaseHuggingfaceCLICommand +from huggingface_hub.commands._cli_utils import ANSI +from huggingface_hub.constants import REPO_TYPES, SPACES_SDK_TYPES +from huggingface_hub.errors import HfHubHTTPError, RepositoryNotFoundError, RevisionNotFoundError +from huggingface_hub.hf_api import HfApi +from huggingface_hub.utils import logging + + +logger = logging.get_logger(__name__) + + +class RepoCommands(BaseHuggingfaceCLICommand): + @staticmethod + def register_subcommand(parser: _SubParsersAction): + repo_parser = parser.add_parser("repo", help="Manage repos on the Hub.") + repo_subparsers = repo_parser.add_subparsers(help="huggingface.co repos related commands") + + # Show help if no subcommand is provided + repo_parser.set_defaults(func=lambda args: repo_parser.print_help()) + + # CREATE + repo_create_parser = repo_subparsers.add_parser("create", help="Create a new repo on huggingface.co") + repo_create_parser.add_argument( + "repo_id", + type=str, + help="The ID of the repo to create to (e.g. `username/repo-name`). The username is optional and will be set to your username if not provided.", + ) + repo_create_parser.add_argument( + "--repo-type", + type=str, + help='Optional: set to "dataset" or "space" if creating a dataset or space, default is model.', + ) + repo_create_parser.add_argument( + "--space_sdk", + type=str, + help='Optional: Hugging Face Spaces SDK type. Required when --type is set to "space".', + choices=SPACES_SDK_TYPES, + ) + repo_create_parser.add_argument( + "--private", + action="store_true", + help="Whether to create a private repository. Defaults to public unless the organization's default is private.", + ) + repo_create_parser.add_argument( + "--token", + type=str, + help="Hugging Face token. Will default to the locally saved token if not provided.", + ) + repo_create_parser.add_argument( + "--exist-ok", + action="store_true", + help="Do not raise an error if repo already exists.", + ) + repo_create_parser.add_argument( + "--resource-group-id", + type=str, + help="Resource group in which to create the repo. Resource groups is only available for Enterprise Hub organizations.", + ) + repo_create_parser.set_defaults(func=lambda args: RepoCreateCommand(args)) + + # TAG SUBCOMMANDS + repo_tag_parser = repo_subparsers.add_parser("tag", help="Manage tags for a repo on the Hub.") + tag_subparsers = repo_tag_parser.add_subparsers(help="Tag actions", dest="tag_action", required=True) + + # tag create + tag_create_parser = tag_subparsers.add_parser("create", help="Create a tag for a repo.") + tag_create_parser.add_argument( + "repo_id", type=str, help="The ID of the repo to tag (e.g. `username/repo-name`)." + ) + tag_create_parser.add_argument("tag", type=str, help="The name of the tag to create.") + tag_create_parser.add_argument("-m", "--message", type=str, help="The description of the tag to create.") + tag_create_parser.add_argument("--revision", type=str, help="The git revision to tag.") + tag_create_parser.add_argument( + "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens." + ) + tag_create_parser.add_argument( + "--repo-type", + choices=["model", "dataset", "space"], + default="model", + help="Set the type of repository (model, dataset, or space).", + ) + tag_create_parser.set_defaults(func=lambda args: RepoTagCreateCommand(args)) + + # tag list + tag_list_parser = tag_subparsers.add_parser("list", help="List tags for a repo.") + tag_list_parser.add_argument( + "repo_id", type=str, help="The ID of the repo to list tags for (e.g. `username/repo-name`)." + ) + tag_list_parser.add_argument( + "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens." + ) + tag_list_parser.add_argument( + "--repo-type", + choices=["model", "dataset", "space"], + default="model", + help="Set the type of repository (model, dataset, or space).", + ) + tag_list_parser.set_defaults(func=lambda args: RepoTagListCommand(args)) + + # tag delete + tag_delete_parser = tag_subparsers.add_parser("delete", help="Delete a tag from a repo.") + tag_delete_parser.add_argument( + "repo_id", type=str, help="The ID of the repo to delete the tag from (e.g. `username/repo-name`)." + ) + tag_delete_parser.add_argument("tag", type=str, help="The name of the tag to delete.") + tag_delete_parser.add_argument("-y", "--yes", action="store_true", help="Answer Yes to prompts automatically.") + tag_delete_parser.add_argument( + "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens." + ) + tag_delete_parser.add_argument( + "--repo-type", + choices=["model", "dataset", "space"], + default="model", + help="Set the type of repository (model, dataset, or space).", + ) + tag_delete_parser.set_defaults(func=lambda args: RepoTagDeleteCommand(args)) + + +class RepoCreateCommand: + def __init__(self, args: argparse.Namespace): + self.repo_id: str = args.repo_id + self.repo_type: Optional[str] = args.repo_type + self.space_sdk: Optional[str] = args.space_sdk + self.private: bool = args.private + self.token: Optional[str] = args.token + self.exist_ok: bool = args.exist_ok + self.resource_group_id: Optional[str] = args.resource_group_id + self._api = HfApi() + + def run(self): + repo_url = self._api.create_repo( + repo_id=self.repo_id, + repo_type=self.repo_type, + private=self.private, + token=self.token, + exist_ok=self.exist_ok, + resource_group_id=self.resource_group_id, + space_sdk=self.space_sdk, + ) + print(f"Successfully created {ANSI.bold(repo_url.repo_id)} on the Hub.") + print(f"Your repo is now available at {ANSI.bold(repo_url)}") + + +class RepoTagCommand: + def __init__(self, args): + self.args = args + self.api = HfApi(token=getattr(args, "token", None)) + self.repo_id = args.repo_id + self.repo_type = getattr(args, "repo_type", "model") + if self.repo_type not in REPO_TYPES: + print("Invalid repo --repo-type") + exit(1) + + +class RepoTagCreateCommand(RepoTagCommand): + def run(self): + print( + f"You are about to create tag {ANSI.bold(str(self.args.tag))} on {self.repo_type} {ANSI.bold(self.repo_id)}" + ) + try: + self.api.create_tag( + repo_id=self.repo_id, + tag=self.args.tag, + tag_message=getattr(self.args, "message", None), + revision=getattr(self.args, "revision", None), + repo_type=self.repo_type, + ) + except RepositoryNotFoundError: + print(f"{self.repo_type.capitalize()} {ANSI.bold(self.repo_id)} not found.") + exit(1) + except RevisionNotFoundError: + print(f"Revision {ANSI.bold(str(getattr(self.args, 'revision', None)))} not found.") + exit(1) + except HfHubHTTPError as e: + if e.response.status_code == 409: + print(f"Tag {ANSI.bold(str(self.args.tag))} already exists on {ANSI.bold(self.repo_id)}") + exit(1) + raise e + print(f"Tag {ANSI.bold(str(self.args.tag))} created on {ANSI.bold(self.repo_id)}") + + +class RepoTagListCommand(RepoTagCommand): + def run(self): + try: + refs = self.api.list_repo_refs( + repo_id=self.repo_id, + repo_type=self.repo_type, + ) + except RepositoryNotFoundError: + print(f"{self.repo_type.capitalize()} {ANSI.bold(self.repo_id)} not found.") + exit(1) + except HTTPError as e: + print(e) + print(ANSI.red(e.response.text)) + exit(1) + if len(refs.tags) == 0: + print("No tags found") + exit(0) + print(f"Tags for {self.repo_type} {ANSI.bold(self.repo_id)}:") + for tag in refs.tags: + print(tag.name) + + +class RepoTagDeleteCommand(RepoTagCommand): + def run(self): + print(f"You are about to delete tag {ANSI.bold(self.args.tag)} on {self.repo_type} {ANSI.bold(self.repo_id)}") + if not getattr(self.args, "yes", False): + choice = input("Proceed? [Y/n] ").lower() + if choice not in ("", "y", "yes"): + print("Abort") + exit() + try: + self.api.delete_tag(repo_id=self.repo_id, tag=self.args.tag, repo_type=self.repo_type) + except RepositoryNotFoundError: + print(f"{self.repo_type.capitalize()} {ANSI.bold(self.repo_id)} not found.") + exit(1) + except RevisionNotFoundError: + print(f"Tag {ANSI.bold(self.args.tag)} not found on {ANSI.bold(self.repo_id)}") + exit(1) + print(f"Tag {ANSI.bold(self.args.tag)} deleted on {ANSI.bold(self.repo_id)}") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/repo_files.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/repo_files.py new file mode 100644 index 0000000000000000000000000000000000000000..403d3126e234c7561cde5fbf8f1d49d7e3271da8 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/repo_files.py @@ -0,0 +1,128 @@ +# coding=utf-8 +# Copyright 2023-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains command to update or delete files in a repository using the CLI. + +Usage: + # delete all + hf repo-files delete "*" + + # delete single file + hf repo-files delete file.txt + + # delete single folder + hf repo-files delete folder/ + + # delete multiple + hf repo-files delete file.txt folder/ file2.txt + + # delete multiple patterns + hf repo-files delete file.txt "*.json" "folder/*.parquet" + + # delete from different revision / repo-type + hf repo-files delete file.txt --revision=refs/pr/1 --repo-type=dataset +""" + +from argparse import _SubParsersAction +from typing import List, Optional + +from huggingface_hub import logging +from huggingface_hub.commands import BaseHuggingfaceCLICommand +from huggingface_hub.hf_api import HfApi + + +logger = logging.get_logger(__name__) + + +class DeleteFilesSubCommand: + def __init__(self, args) -> None: + self.args = args + self.repo_id: str = args.repo_id + self.repo_type: Optional[str] = args.repo_type + self.revision: Optional[str] = args.revision + self.api: HfApi = HfApi(token=args.token, library_name="huggingface-cli") + self.patterns: List[str] = args.patterns + self.commit_message: Optional[str] = args.commit_message + self.commit_description: Optional[str] = args.commit_description + self.create_pr: bool = args.create_pr + self.token: Optional[str] = args.token + + def run(self) -> None: + logging.set_verbosity_info() + url = self.api.delete_files( + delete_patterns=self.patterns, + repo_id=self.repo_id, + repo_type=self.repo_type, + revision=self.revision, + commit_message=self.commit_message, + commit_description=self.commit_description, + create_pr=self.create_pr, + ) + print(f"Files correctly deleted from repo. Commit: {url}.") + logging.set_verbosity_warning() + + +class RepoFilesCommand(BaseHuggingfaceCLICommand): + @staticmethod + def register_subcommand(parser: _SubParsersAction): + repo_files_parser = parser.add_parser("repo-files", help="Manage files in a repo on the Hub.") + repo_files_subparsers = repo_files_parser.add_subparsers( + help="Action to execute against the files.", + required=True, + ) + delete_subparser = repo_files_subparsers.add_parser( + "delete", + help="Delete files from a repo on the Hub", + ) + delete_subparser.set_defaults(func=lambda args: DeleteFilesSubCommand(args)) + delete_subparser.add_argument( + "repo_id", type=str, help="The ID of the repo to manage (e.g. `username/repo-name`)." + ) + delete_subparser.add_argument( + "patterns", + nargs="+", + type=str, + help="Glob patterns to match files to delete.", + ) + delete_subparser.add_argument( + "--repo-type", + choices=["model", "dataset", "space"], + default="model", + help="Type of the repo to upload to (e.g. `dataset`).", + ) + delete_subparser.add_argument( + "--revision", + type=str, + help=( + "An optional Git revision to push to. It can be a branch name " + "or a PR reference. If revision does not" + " exist and `--create-pr` is not set, a branch will be automatically created." + ), + ) + delete_subparser.add_argument( + "--commit-message", type=str, help="The summary / title / first line of the generated commit." + ) + delete_subparser.add_argument( + "--commit-description", type=str, help="The description of the generated commit." + ) + delete_subparser.add_argument( + "--create-pr", action="store_true", help="Whether to create a new Pull Request for these changes." + ) + delete_subparser.add_argument( + "--token", + type=str, + help="A User Access Token generated from https://huggingface.co/settings/tokens", + ) + + repo_files_parser.set_defaults(func=RepoFilesCommand) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/system.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/system.py new file mode 100644 index 0000000000000000000000000000000000000000..03650175e9b71e329755de5c86e5bbf50569d4b7 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/system.py @@ -0,0 +1,52 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains commands to print information about the environment and version. + +Usage: + hf env + hf version +""" + +from argparse import _SubParsersAction + +from huggingface_hub import __version__ + +from ..utils import dump_environment_info +from . import BaseHuggingfaceCLICommand + + +class EnvironmentCommand(BaseHuggingfaceCLICommand): + def __init__(self, args): + self.args = args + + @staticmethod + def register_subcommand(parser: _SubParsersAction): + env_parser = parser.add_parser("env", help="Print information about the environment.") + env_parser.set_defaults(func=EnvironmentCommand) + + def run(self) -> None: + dump_environment_info() + + +class VersionCommand(BaseHuggingfaceCLICommand): + def __init__(self, args): + self.args = args + + @staticmethod + def register_subcommand(parser: _SubParsersAction): + version_parser = parser.add_parser("version", help="Print information about the hf version.") + version_parser.set_defaults(func=VersionCommand) + + def run(self) -> None: + print(f"huggingface_hub version: {__version__}") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/upload.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/upload.py new file mode 100644 index 0000000000000000000000000000000000000000..0306bf9f5715fdc180dc4fa9819852388fca8b99 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/upload.py @@ -0,0 +1,316 @@ +# coding=utf-8 +# Copyright 2023-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains command to upload a repo or file with the CLI. + +Usage: + # Upload file (implicit) + hf upload my-cool-model ./my-cool-model.safetensors + + # Upload file (explicit) + hf upload my-cool-model ./my-cool-model.safetensors model.safetensors + + # Upload directory (implicit). If `my-cool-model/` is a directory it will be uploaded, otherwise an exception is raised. + hf upload my-cool-model + + # Upload directory (explicit) + hf upload my-cool-model ./models/my-cool-model . + + # Upload filtered directory (example: tensorboard logs except for the last run) + hf upload my-cool-model ./model/training /logs --include "*.tfevents.*" --exclude "*20230905*" + + # Upload with wildcard + hf upload my-cool-model "./model/training/*.safetensors" + + # Upload private dataset + hf upload Wauplin/my-cool-dataset ./data . --repo-type=dataset --private + + # Upload with token + hf upload Wauplin/my-cool-model --token=hf_**** + + # Sync local Space with Hub (upload new files, delete removed files) + hf upload Wauplin/space-example --repo-type=space --exclude="/logs/*" --delete="*" --commit-message="Sync local Space with Hub" + + # Schedule commits every 30 minutes + hf upload Wauplin/my-cool-model --every=30 +""" + +import os +import time +import warnings +from argparse import Namespace, _SubParsersAction +from typing import List, Optional + +from huggingface_hub import logging +from huggingface_hub._commit_scheduler import CommitScheduler +from huggingface_hub.commands import BaseHuggingfaceCLICommand +from huggingface_hub.constants import HF_HUB_ENABLE_HF_TRANSFER +from huggingface_hub.errors import RevisionNotFoundError +from huggingface_hub.hf_api import HfApi +from huggingface_hub.utils import disable_progress_bars, enable_progress_bars +from huggingface_hub.utils._runtime import is_xet_available + + +logger = logging.get_logger(__name__) + + +class UploadCommand(BaseHuggingfaceCLICommand): + @staticmethod + def register_subcommand(parser: _SubParsersAction): + upload_parser = parser.add_parser( + "upload", help="Upload a file or a folder to the Hub. Recommended for single-commit uploads." + ) + upload_parser.add_argument( + "repo_id", type=str, help="The ID of the repo to upload to (e.g. `username/repo-name`)." + ) + upload_parser.add_argument( + "local_path", + nargs="?", + help="Local path to the file or folder to upload. Wildcard patterns are supported. Defaults to current directory.", + ) + upload_parser.add_argument( + "path_in_repo", + nargs="?", + help="Path of the file or folder in the repo. Defaults to the relative path of the file or folder.", + ) + upload_parser.add_argument( + "--repo-type", + choices=["model", "dataset", "space"], + default="model", + help="Type of the repo to upload to (e.g. `dataset`).", + ) + upload_parser.add_argument( + "--revision", + type=str, + help=( + "An optional Git revision to push to. It can be a branch name or a PR reference. If revision does not" + " exist and `--create-pr` is not set, a branch will be automatically created." + ), + ) + upload_parser.add_argument( + "--private", + action="store_true", + help=( + "Whether to create a private repo if repo doesn't exist on the Hub. Ignored if the repo already" + " exists." + ), + ) + upload_parser.add_argument("--include", nargs="*", type=str, help="Glob patterns to match files to upload.") + upload_parser.add_argument( + "--exclude", nargs="*", type=str, help="Glob patterns to exclude from files to upload." + ) + upload_parser.add_argument( + "--delete", + nargs="*", + type=str, + help="Glob patterns for file to be deleted from the repo while committing.", + ) + upload_parser.add_argument( + "--commit-message", type=str, help="The summary / title / first line of the generated commit." + ) + upload_parser.add_argument("--commit-description", type=str, help="The description of the generated commit.") + upload_parser.add_argument( + "--create-pr", action="store_true", help="Whether to upload content as a new Pull Request." + ) + upload_parser.add_argument( + "--every", + type=float, + help="If set, a background job is scheduled to create commits every `every` minutes.", + ) + upload_parser.add_argument( + "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens" + ) + upload_parser.add_argument( + "--quiet", + action="store_true", + help="If True, progress bars are disabled and only the path to the uploaded files is printed.", + ) + upload_parser.set_defaults(func=UploadCommand) + + def __init__(self, args: Namespace) -> None: + self.repo_id: str = args.repo_id + self.repo_type: Optional[str] = args.repo_type + self.revision: Optional[str] = args.revision + self.private: bool = args.private + + self.include: Optional[List[str]] = args.include + self.exclude: Optional[List[str]] = args.exclude + self.delete: Optional[List[str]] = args.delete + + self.commit_message: Optional[str] = args.commit_message + self.commit_description: Optional[str] = args.commit_description + self.create_pr: bool = args.create_pr + self.api: HfApi = HfApi(token=args.token, library_name="huggingface-cli") + self.quiet: bool = args.quiet # disable warnings and progress bars + + # Check `--every` is valid + if args.every is not None and args.every <= 0: + raise ValueError(f"`every` must be a positive value (got '{args.every}')") + self.every: Optional[float] = args.every + + # Resolve `local_path` and `path_in_repo` + repo_name: str = args.repo_id.split("/")[-1] # e.g. "Wauplin/my-cool-model" => "my-cool-model" + self.local_path: str + self.path_in_repo: str + + if args.local_path is not None and any(c in args.local_path for c in ["*", "?", "["]): + if args.include is not None: + raise ValueError("Cannot set `--include` when passing a `local_path` containing a wildcard.") + if args.path_in_repo is not None and args.path_in_repo != ".": + raise ValueError("Cannot set `path_in_repo` when passing a `local_path` containing a wildcard.") + self.local_path = "." + self.include = args.local_path + self.path_in_repo = "." + elif args.local_path is None and os.path.isfile(repo_name): + # Implicit case 1: user provided only a repo_id which happen to be a local file as well => upload it with same name + self.local_path = repo_name + self.path_in_repo = repo_name + elif args.local_path is None and os.path.isdir(repo_name): + # Implicit case 2: user provided only a repo_id which happen to be a local folder as well => upload it at root + self.local_path = repo_name + self.path_in_repo = "." + elif args.local_path is None: + # Implicit case 3: user provided only a repo_id that does not match a local file or folder + # => the user must explicitly provide a local_path => raise exception + raise ValueError(f"'{repo_name}' is not a local file or folder. Please set `local_path` explicitly.") + elif args.path_in_repo is None and os.path.isfile(args.local_path): + # Explicit local path to file, no path in repo => upload it at root with same name + self.local_path = args.local_path + self.path_in_repo = os.path.basename(args.local_path) + elif args.path_in_repo is None: + # Explicit local path to folder, no path in repo => upload at root + self.local_path = args.local_path + self.path_in_repo = "." + else: + # Finally, if both paths are explicit + self.local_path = args.local_path + self.path_in_repo = args.path_in_repo + + def run(self) -> None: + if self.quiet: + disable_progress_bars() + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + print(self._upload()) + enable_progress_bars() + else: + logging.set_verbosity_info() + print(self._upload()) + logging.set_verbosity_warning() + + def _upload(self) -> str: + if os.path.isfile(self.local_path): + if self.include is not None and len(self.include) > 0: + warnings.warn("Ignoring `--include` since a single file is uploaded.") + if self.exclude is not None and len(self.exclude) > 0: + warnings.warn("Ignoring `--exclude` since a single file is uploaded.") + if self.delete is not None and len(self.delete) > 0: + warnings.warn("Ignoring `--delete` since a single file is uploaded.") + + if not is_xet_available() and not HF_HUB_ENABLE_HF_TRANSFER: + logger.info( + "Consider using `hf_transfer` for faster uploads. This solution comes with some limitations. See" + " https://huggingface.co/docs/huggingface_hub/hf_transfer for more details." + ) + + # Schedule commits if `every` is set + if self.every is not None: + if os.path.isfile(self.local_path): + # If file => watch entire folder + use allow_patterns + folder_path = os.path.dirname(self.local_path) + path_in_repo = ( + self.path_in_repo[: -len(self.local_path)] # remove filename from path_in_repo + if self.path_in_repo.endswith(self.local_path) + else self.path_in_repo + ) + allow_patterns = [self.local_path] + ignore_patterns = [] + else: + folder_path = self.local_path + path_in_repo = self.path_in_repo + allow_patterns = self.include or [] + ignore_patterns = self.exclude or [] + if self.delete is not None and len(self.delete) > 0: + warnings.warn("Ignoring `--delete` when uploading with scheduled commits.") + + scheduler = CommitScheduler( + folder_path=folder_path, + repo_id=self.repo_id, + repo_type=self.repo_type, + revision=self.revision, + allow_patterns=allow_patterns, + ignore_patterns=ignore_patterns, + path_in_repo=path_in_repo, + private=self.private, + every=self.every, + hf_api=self.api, + ) + print(f"Scheduling commits every {self.every} minutes to {scheduler.repo_id}.") + try: # Block main thread until KeyboardInterrupt + while True: + time.sleep(100) + except KeyboardInterrupt: + scheduler.stop() + return "Stopped scheduled commits." + + # Otherwise, create repo and proceed with the upload + if not os.path.isfile(self.local_path) and not os.path.isdir(self.local_path): + raise FileNotFoundError(f"No such file or directory: '{self.local_path}'.") + repo_id = self.api.create_repo( + repo_id=self.repo_id, + repo_type=self.repo_type, + exist_ok=True, + private=self.private, + space_sdk="gradio" if self.repo_type == "space" else None, + # ^ We don't want it to fail when uploading to a Space => let's set Gradio by default. + # ^ I'd rather not add CLI args to set it explicitly as we already have `hf repo create` for that. + ).repo_id + + # Check if branch already exists and if not, create it + if self.revision is not None and not self.create_pr: + try: + self.api.repo_info(repo_id=repo_id, repo_type=self.repo_type, revision=self.revision) + except RevisionNotFoundError: + logger.info(f"Branch '{self.revision}' not found. Creating it...") + self.api.create_branch(repo_id=repo_id, repo_type=self.repo_type, branch=self.revision, exist_ok=True) + # ^ `exist_ok=True` to avoid race concurrency issues + + # File-based upload + if os.path.isfile(self.local_path): + return self.api.upload_file( + path_or_fileobj=self.local_path, + path_in_repo=self.path_in_repo, + repo_id=repo_id, + repo_type=self.repo_type, + revision=self.revision, + commit_message=self.commit_message, + commit_description=self.commit_description, + create_pr=self.create_pr, + ) + + # Folder-based upload + else: + return self.api.upload_folder( + folder_path=self.local_path, + path_in_repo=self.path_in_repo, + repo_id=repo_id, + repo_type=self.repo_type, + revision=self.revision, + commit_message=self.commit_message, + commit_description=self.commit_description, + create_pr=self.create_pr, + allow_patterns=self.include, + ignore_patterns=self.exclude, + delete_patterns=self.delete, + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/upload_large_folder.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/upload_large_folder.py new file mode 100644 index 0000000000000000000000000000000000000000..675c9ffe3dcd70242a9acd7837c6c2f00d8836df --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/cli/upload_large_folder.py @@ -0,0 +1,132 @@ +# coding=utf-8 +# Copyright 2023-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains command to upload a large folder with the CLI.""" + +import os +from argparse import Namespace, _SubParsersAction +from typing import List, Optional + +from huggingface_hub import logging +from huggingface_hub.commands import BaseHuggingfaceCLICommand +from huggingface_hub.hf_api import HfApi +from huggingface_hub.utils import disable_progress_bars + +from ._cli_utils import ANSI + + +logger = logging.get_logger(__name__) + + +class UploadLargeFolderCommand(BaseHuggingfaceCLICommand): + @staticmethod + def register_subcommand(parser: _SubParsersAction): + subparser = parser.add_parser( + "upload-large-folder", + help="Upload a large folder to the Hub. Recommended for resumable uploads.", + ) + subparser.add_argument( + "repo_id", type=str, help="The ID of the repo to upload to (e.g. `username/repo-name`)." + ) + subparser.add_argument("local_path", type=str, help="Local path to the file or folder to upload.") + subparser.add_argument( + "--repo-type", + choices=["model", "dataset", "space"], + help="Type of the repo to upload to (e.g. `dataset`).", + ) + subparser.add_argument( + "--revision", + type=str, + help=("An optional Git revision to push to. It can be a branch name or a PR reference."), + ) + subparser.add_argument( + "--private", + action="store_true", + help=( + "Whether to create a private repo if repo doesn't exist on the Hub. Ignored if the repo already exists." + ), + ) + subparser.add_argument("--include", nargs="*", type=str, help="Glob patterns to match files to upload.") + subparser.add_argument("--exclude", nargs="*", type=str, help="Glob patterns to exclude from files to upload.") + subparser.add_argument( + "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens" + ) + subparser.add_argument( + "--num-workers", type=int, help="Number of workers to use to hash, upload and commit files." + ) + subparser.add_argument("--no-report", action="store_true", help="Whether to disable regular status report.") + subparser.add_argument("--no-bars", action="store_true", help="Whether to disable progress bars.") + subparser.set_defaults(func=UploadLargeFolderCommand) + + def __init__(self, args: Namespace) -> None: + self.repo_id: str = args.repo_id + self.local_path: str = args.local_path + self.repo_type: str = args.repo_type + self.revision: Optional[str] = args.revision + self.private: bool = args.private + + self.include: Optional[List[str]] = args.include + self.exclude: Optional[List[str]] = args.exclude + + self.api: HfApi = HfApi(token=args.token, library_name="huggingface-cli") + + self.num_workers: Optional[int] = args.num_workers + self.no_report: bool = args.no_report + self.no_bars: bool = args.no_bars + + if not os.path.isdir(self.local_path): + raise ValueError("Large upload is only supported for folders.") + + def run(self) -> None: + logging.set_verbosity_info() + + print( + ANSI.yellow( + "You are about to upload a large folder to the Hub using `hf upload-large-folder`. " + "This is a new feature so feedback is very welcome!\n" + "\n" + "A few things to keep in mind:\n" + " - Repository limits still apply: https://huggingface.co/docs/hub/repositories-recommendations\n" + " - Do not start several processes in parallel.\n" + " - You can interrupt and resume the process at any time. " + "The script will pick up where it left off except for partially uploaded files that would have to be entirely reuploaded.\n" + " - Do not upload the same folder to several repositories. If you need to do so, you must delete the `./.cache/huggingface/` folder first.\n" + "\n" + f"Some temporary metadata will be stored under `{self.local_path}/.cache/huggingface`.\n" + " - You must not modify those files manually.\n" + " - You must not delete the `./.cache/huggingface/` folder while a process is running.\n" + " - You can delete the `./.cache/huggingface/` folder to reinitialize the upload state when process is not running. Files will have to be hashed and preuploaded again, except for already committed files.\n" + "\n" + "If the process output is too verbose, you can disable the progress bars with `--no-bars`. " + "You can also entirely disable the status report with `--no-report`.\n" + "\n" + "For more details, run `hf upload-large-folder --help` or check the documentation at " + "https://huggingface.co/docs/huggingface_hub/guides/upload#upload-a-large-folder." + ) + ) + + if self.no_bars: + disable_progress_bars() + + self.api.upload_large_folder( + repo_id=self.repo_id, + folder_path=self.local_path, + repo_type=self.repo_type, + revision=self.revision, + private=self.private, + allow_patterns=self.include, + ignore_patterns=self.exclude, + num_workers=self.num_workers, + print_report=not self.no_report, + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..49d088214505b9604964ab142e7f8a5b38ccd5ef --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/__init__.py @@ -0,0 +1,27 @@ +# Copyright 2020 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import ABC, abstractmethod +from argparse import _SubParsersAction + + +class BaseHuggingfaceCLICommand(ABC): + @staticmethod + @abstractmethod + def register_subcommand(parser: _SubParsersAction): + raise NotImplementedError() + + @abstractmethod + def run(self): + raise NotImplementedError() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/_cli_utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/_cli_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..bf4a1c0373b4d4bb71a3f4e8ea39da5a01cc79a7 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/_cli_utils.py @@ -0,0 +1,74 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains a utility for good-looking prints.""" + +import os +from typing import List, Union + + +class ANSI: + """ + Helper for en.wikipedia.org/wiki/ANSI_escape_code + """ + + _bold = "\u001b[1m" + _gray = "\u001b[90m" + _red = "\u001b[31m" + _reset = "\u001b[0m" + _yellow = "\u001b[33m" + + @classmethod + def bold(cls, s: str) -> str: + return cls._format(s, cls._bold) + + @classmethod + def gray(cls, s: str) -> str: + return cls._format(s, cls._gray) + + @classmethod + def red(cls, s: str) -> str: + return cls._format(s, cls._bold + cls._red) + + @classmethod + def yellow(cls, s: str) -> str: + return cls._format(s, cls._yellow) + + @classmethod + def _format(cls, s: str, code: str) -> str: + if os.environ.get("NO_COLOR"): + # See https://no-color.org/ + return s + return f"{code}{s}{cls._reset}" + + +def tabulate(rows: List[List[Union[str, int]]], headers: List[str]) -> str: + """ + Inspired by: + + - stackoverflow.com/a/8356620/593036 + - stackoverflow.com/questions/9535954/printing-lists-as-tabular-data + """ + col_widths = [max(len(str(x)) for x in col) for col in zip(*rows, headers)] + row_format = ("{{:{}}} " * len(headers)).format(*col_widths) + lines = [] + lines.append(row_format.format(*headers)) + lines.append(row_format.format(*["-" * w for w in col_widths])) + for row in rows: + lines.append(row_format.format(*row)) + return "\n".join(lines) + + +def show_deprecation_warning(old_command: str, new_command: str): + """Show a yellow warning about deprecated CLI command.""" + print(ANSI.yellow(f"⚠️ Warning: '{old_command}' is deprecated. Use '{new_command}' instead.")) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/delete_cache.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/delete_cache.py new file mode 100644 index 0000000000000000000000000000000000000000..78ea1179678371807b3686b8acf17b9f0997035f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/delete_cache.py @@ -0,0 +1,476 @@ +# coding=utf-8 +# Copyright 2022-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains command to delete some revisions from the HF cache directory. + +Usage: + huggingface-cli delete-cache + huggingface-cli delete-cache --disable-tui + huggingface-cli delete-cache --dir ~/.cache/huggingface/hub + huggingface-cli delete-cache --sort=size + +NOTE: + This command is based on `InquirerPy` to build the multiselect menu in the terminal. + This dependency has to be installed with `pip install "huggingface_hub[cli]"`. Since + we want to avoid as much as possible cross-platform issues, I chose a library that + is built on top of `python-prompt-toolkit` which seems to be a reference in terminal + GUI (actively maintained on both Unix and Windows, 7.9k stars). + + For the moment, the TUI feature is in beta. + + See: + - https://github.com/kazhala/InquirerPy + - https://inquirerpy.readthedocs.io/en/latest/ + - https://github.com/prompt-toolkit/python-prompt-toolkit + + Other solutions could have been: + - `simple_term_menu`: would be good as well for our use case but some issues suggest + that Windows is less supported. + See: https://github.com/IngoMeyer441/simple-term-menu + - `PyInquirer`: very similar to `InquirerPy` but older and not maintained anymore. + In particular, no support of Python3.10. + See: https://github.com/CITGuru/PyInquirer + - `pick` (or `pickpack`): easy to use and flexible but built on top of Python's + standard library `curses` that is specific to Unix (not implemented on Windows). + See https://github.com/wong2/pick and https://github.com/anafvana/pickpack. + - `inquirer`: lot of traction (700 stars) but explicitly states "experimental + support of Windows". Not built on top of `python-prompt-toolkit`. + See https://github.com/magmax/python-inquirer + +TODO: add support for `huggingface-cli delete-cache aaaaaa bbbbbb cccccc (...)` ? +TODO: add "--keep-last" arg to delete revisions that are not on `main` ref +TODO: add "--filter" arg to filter repositories by name ? +TODO: add "--limit" arg to limit to X repos ? +TODO: add "-y" arg for immediate deletion ? +See discussions in https://github.com/huggingface/huggingface_hub/issues/1025. +""" + +import os +from argparse import Namespace, _SubParsersAction +from functools import wraps +from tempfile import mkstemp +from typing import Any, Callable, Iterable, List, Literal, Optional, Union + +from ..utils import CachedRepoInfo, CachedRevisionInfo, HFCacheInfo, scan_cache_dir +from . import BaseHuggingfaceCLICommand +from ._cli_utils import ANSI, show_deprecation_warning + + +try: + from InquirerPy import inquirer + from InquirerPy.base.control import Choice + from InquirerPy.separator import Separator + + _inquirer_py_available = True +except ImportError: + _inquirer_py_available = False + +SortingOption_T = Literal["alphabetical", "lastUpdated", "lastUsed", "size"] + + +def require_inquirer_py(fn: Callable) -> Callable: + """Decorator to flag methods that require `InquirerPy`.""" + + # TODO: refactor this + imports in a unified pattern across codebase + @wraps(fn) + def _inner(*args, **kwargs): + if not _inquirer_py_available: + raise ImportError( + "The `delete-cache` command requires extra dependencies to work with" + ' the TUI.\nPlease run `pip install "huggingface_hub[cli]"` to install' + " them.\nOtherwise, disable TUI using the `--disable-tui` flag." + ) + + return fn(*args, **kwargs) + + return _inner + + +# Possibility for the user to cancel deletion +_CANCEL_DELETION_STR = "CANCEL_DELETION" + + +class DeleteCacheCommand(BaseHuggingfaceCLICommand): + @staticmethod + def register_subcommand(parser: _SubParsersAction): + delete_cache_parser = parser.add_parser("delete-cache", help="Delete revisions from the cache directory.") + + delete_cache_parser.add_argument( + "--dir", + type=str, + default=None, + help="cache directory (optional). Default to the default HuggingFace cache.", + ) + + delete_cache_parser.add_argument( + "--disable-tui", + action="store_true", + help=( + "Disable Terminal User Interface (TUI) mode. Useful if your" + " platform/terminal doesn't support the multiselect menu." + ), + ) + + delete_cache_parser.add_argument( + "--sort", + nargs="?", + choices=["alphabetical", "lastUpdated", "lastUsed", "size"], + help=( + "Sort repositories by the specified criteria. Options: " + "'alphabetical' (A-Z), " + "'lastUpdated' (newest first), " + "'lastUsed' (most recent first), " + "'size' (largest first)." + ), + ) + + delete_cache_parser.set_defaults(func=DeleteCacheCommand) + + def __init__(self, args: Namespace) -> None: + self.cache_dir: Optional[str] = args.dir + self.disable_tui: bool = args.disable_tui + self.sort_by: Optional[SortingOption_T] = args.sort + + def run(self): + """Run `delete-cache` command with or without TUI.""" + show_deprecation_warning("huggingface-cli delete-cache", "hf cache delete") + + # Scan cache directory + hf_cache_info = scan_cache_dir(self.cache_dir) + + # Manual review from the user + if self.disable_tui: + selected_hashes = _manual_review_no_tui(hf_cache_info, preselected=[], sort_by=self.sort_by) + else: + selected_hashes = _manual_review_tui(hf_cache_info, preselected=[], sort_by=self.sort_by) + + # If deletion is not cancelled + if len(selected_hashes) > 0 and _CANCEL_DELETION_STR not in selected_hashes: + confirm_message = _get_expectations_str(hf_cache_info, selected_hashes) + " Confirm deletion ?" + + # Confirm deletion + if self.disable_tui: + confirmed = _ask_for_confirmation_no_tui(confirm_message) + else: + confirmed = _ask_for_confirmation_tui(confirm_message) + + # Deletion is confirmed + if confirmed: + strategy = hf_cache_info.delete_revisions(*selected_hashes) + print("Start deletion.") + strategy.execute() + print( + f"Done. Deleted {len(strategy.repos)} repo(s) and" + f" {len(strategy.snapshots)} revision(s) for a total of" + f" {strategy.expected_freed_size_str}." + ) + return + + # Deletion is cancelled + print("Deletion is cancelled. Do nothing.") + + +def _get_repo_sorting_key(repo: CachedRepoInfo, sort_by: Optional[SortingOption_T] = None): + if sort_by == "alphabetical": + return (repo.repo_type, repo.repo_id.lower()) # by type then name + elif sort_by == "lastUpdated": + return -max(rev.last_modified for rev in repo.revisions) # newest first + elif sort_by == "lastUsed": + return -repo.last_accessed # most recently used first + elif sort_by == "size": + return -repo.size_on_disk # largest first + else: + return (repo.repo_type, repo.repo_id) # default stable order + + +@require_inquirer_py +def _manual_review_tui( + hf_cache_info: HFCacheInfo, + preselected: List[str], + sort_by: Optional[SortingOption_T] = None, +) -> List[str]: + """Ask the user for a manual review of the revisions to delete. + + Displays a multi-select menu in the terminal (TUI). + """ + # Define multiselect list + choices = _get_tui_choices_from_scan( + repos=hf_cache_info.repos, + preselected=preselected, + sort_by=sort_by, + ) + checkbox = inquirer.checkbox( + message="Select revisions to delete:", + choices=choices, # List of revisions with some pre-selection + cycle=False, # No loop between top and bottom + height=100, # Large list if possible + # We use the instruction to display to the user the expected effect of the + # deletion. + instruction=_get_expectations_str( + hf_cache_info, + selected_hashes=[c.value for c in choices if isinstance(c, Choice) and c.enabled], + ), + # We use the long instruction to should keybindings instructions to the user + long_instruction="Press to select, to validate and to quit without modification.", + # Message that is displayed once the user validates its selection. + transformer=lambda result: f"{len(result)} revision(s) selected.", + ) + + # Add a callback to update the information line when a revision is + # selected/unselected + def _update_expectations(_) -> None: + # Hacky way to dynamically set an instruction message to the checkbox when + # a revision hash is selected/unselected. + checkbox._instruction = _get_expectations_str( + hf_cache_info, + selected_hashes=[choice["value"] for choice in checkbox.content_control.choices if choice["enabled"]], + ) + + checkbox.kb_func_lookup["toggle"].append({"func": _update_expectations}) + + # Finally display the form to the user. + try: + return checkbox.execute() + except KeyboardInterrupt: + return [] # Quit without deletion + + +@require_inquirer_py +def _ask_for_confirmation_tui(message: str, default: bool = True) -> bool: + """Ask for confirmation using Inquirer.""" + return inquirer.confirm(message, default=default).execute() + + +def _get_tui_choices_from_scan( + repos: Iterable[CachedRepoInfo], + preselected: List[str], + sort_by: Optional[SortingOption_T] = None, +) -> List: + """Build a list of choices from the scanned repos. + + Args: + repos (*Iterable[`CachedRepoInfo`]*): + List of scanned repos on which we want to delete revisions. + preselected (*List[`str`]*): + List of revision hashes that will be preselected. + sort_by (*Optional[SortingOption_T]*): + Sorting direction. Choices: "alphabetical", "lastUpdated", "lastUsed", "size". + + Return: + The list of choices to pass to `inquirer.checkbox`. + """ + choices: List[Union[Choice, Separator]] = [] + + # First choice is to cancel the deletion + choices.append( + Choice( + _CANCEL_DELETION_STR, + name="None of the following (if selected, nothing will be deleted).", + enabled=False, + ) + ) + + # Sort repos based on specified criteria + sorted_repos = sorted(repos, key=lambda repo: _get_repo_sorting_key(repo, sort_by)) + + for repo in sorted_repos: + # Repo as separator + choices.append( + Separator( + f"\n{repo.repo_type.capitalize()} {repo.repo_id} ({repo.size_on_disk_str}," + f" used {repo.last_accessed_str})" + ) + ) + for revision in sorted(repo.revisions, key=_revision_sorting_order): + # Revision as choice + choices.append( + Choice( + revision.commit_hash, + name=( + f"{revision.commit_hash[:8]}:" + f" {', '.join(sorted(revision.refs)) or '(detached)'} #" + f" modified {revision.last_modified_str}" + ), + enabled=revision.commit_hash in preselected, + ) + ) + + # Return choices + return choices + + +def _manual_review_no_tui( + hf_cache_info: HFCacheInfo, + preselected: List[str], + sort_by: Optional[SortingOption_T] = None, +) -> List[str]: + """Ask the user for a manual review of the revisions to delete. + + Used when TUI is disabled. Manual review happens in a separate tmp file that the + user can manually edit. + """ + # 1. Generate temporary file with delete commands. + fd, tmp_path = mkstemp(suffix=".txt") # suffix to make it easier to find by editors + os.close(fd) + + lines = [] + + sorted_repos = sorted(hf_cache_info.repos, key=lambda repo: _get_repo_sorting_key(repo, sort_by)) + + for repo in sorted_repos: + lines.append( + f"\n# {repo.repo_type.capitalize()} {repo.repo_id} ({repo.size_on_disk_str}," + f" used {repo.last_accessed_str})" + ) + for revision in sorted(repo.revisions, key=_revision_sorting_order): + lines.append( + # Deselect by prepending a '#' + f"{'' if revision.commit_hash in preselected else '#'} " + f" {revision.commit_hash} # Refs:" + # Print `refs` as comment on same line + f" {', '.join(sorted(revision.refs)) or '(detached)'} # modified" + # Print `last_modified` as comment on same line + f" {revision.last_modified_str}" + ) + + with open(tmp_path, "w") as f: + f.write(_MANUAL_REVIEW_NO_TUI_INSTRUCTIONS) + f.write("\n".join(lines)) + + # 2. Prompt instructions to user. + instructions = f""" + TUI is disabled. In order to select which revisions you want to delete, please edit + the following file using the text editor of your choice. Instructions for manual + editing are located at the beginning of the file. Edit the file, save it and confirm + to continue. + File to edit: {ANSI.bold(tmp_path)} + """ + print("\n".join(line.strip() for line in instructions.strip().split("\n"))) + + # 3. Wait for user confirmation. + while True: + selected_hashes = _read_manual_review_tmp_file(tmp_path) + if _ask_for_confirmation_no_tui( + _get_expectations_str(hf_cache_info, selected_hashes) + " Continue ?", + default=False, + ): + break + + # 4. Return selected_hashes sorted to maintain stable order + os.remove(tmp_path) + return sorted(selected_hashes) # Sort to maintain stable order + + +def _ask_for_confirmation_no_tui(message: str, default: bool = True) -> bool: + """Ask for confirmation using pure-python.""" + YES = ("y", "yes", "1") + NO = ("n", "no", "0") + DEFAULT = "" + ALL = YES + NO + (DEFAULT,) + full_message = message + (" (Y/n) " if default else " (y/N) ") + while True: + answer = input(full_message).lower() + if answer == DEFAULT: + return default + if answer in YES: + return True + if answer in NO: + return False + print(f"Invalid input. Must be one of {ALL}") + + +def _get_expectations_str(hf_cache_info: HFCacheInfo, selected_hashes: List[str]) -> str: + """Format a string to display to the user how much space would be saved. + + Example: + ``` + >>> _get_expectations_str(hf_cache_info, selected_hashes) + '7 revisions selected counting for 4.3G.' + ``` + """ + if _CANCEL_DELETION_STR in selected_hashes: + return "Nothing will be deleted." + strategy = hf_cache_info.delete_revisions(*selected_hashes) + return f"{len(selected_hashes)} revisions selected counting for {strategy.expected_freed_size_str}." + + +def _read_manual_review_tmp_file(tmp_path: str) -> List[str]: + """Read the manually reviewed instruction file and return a list of revision hash. + + Example: + ```txt + # This is the tmp file content + ### + + # Commented out line + 123456789 # revision hash + + # Something else + # a_newer_hash # 2 days ago + an_older_hash # 3 days ago + ``` + + ```py + >>> _read_manual_review_tmp_file(tmp_path) + ['123456789', 'an_older_hash'] + ``` + """ + with open(tmp_path) as f: + content = f.read() + + # Split lines + lines = [line.strip() for line in content.split("\n")] + + # Filter commented lines + selected_lines = [line for line in lines if not line.startswith("#")] + + # Select only before comment + selected_hashes = [line.split("#")[0].strip() for line in selected_lines] + + # Return revision hashes + return [hash for hash in selected_hashes if len(hash) > 0] + + +_MANUAL_REVIEW_NO_TUI_INSTRUCTIONS = f""" +# INSTRUCTIONS +# ------------ +# This is a temporary file created by running `huggingface-cli delete-cache` with the +# `--disable-tui` option. It contains a set of revisions that can be deleted from your +# local cache directory. +# +# Please manually review the revisions you want to delete: +# - Revision hashes can be commented out with '#'. +# - Only non-commented revisions in this file will be deleted. +# - Revision hashes that are removed from this file are ignored as well. +# - If `{_CANCEL_DELETION_STR}` line is uncommented, the all cache deletion is cancelled and +# no changes will be applied. +# +# Once you've manually reviewed this file, please confirm deletion in the terminal. This +# file will be automatically removed once done. +# ------------ + +# KILL SWITCH +# ------------ +# Un-comment following line to completely cancel the deletion process +# {_CANCEL_DELETION_STR} +# ------------ + +# REVISIONS +# ------------ +""".strip() + + +def _revision_sorting_order(revision: CachedRevisionInfo) -> Any: + # Sort by last modified (oldest first) + return revision.last_modified diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/download.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/download.py new file mode 100644 index 0000000000000000000000000000000000000000..0dd2c1070ead01f9ad6855de3929928d268279c2 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/download.py @@ -0,0 +1,204 @@ +# coding=utf-8 +# Copyright 2023-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains command to download files from the Hub with the CLI. + +Usage: + huggingface-cli download --help + + # Download file + huggingface-cli download gpt2 config.json + + # Download entire repo + huggingface-cli download fffiloni/zeroscope --repo-type=space --revision=refs/pr/78 + + # Download repo with filters + huggingface-cli download gpt2 --include="*.safetensors" + + # Download with token + huggingface-cli download Wauplin/private-model --token=hf_*** + + # Download quietly (no progress bar, no warnings, only the returned path) + huggingface-cli download gpt2 config.json --quiet + + # Download to local dir + huggingface-cli download gpt2 --local-dir=./models/gpt2 +""" + +import warnings +from argparse import Namespace, _SubParsersAction +from typing import List, Optional + +from huggingface_hub import logging +from huggingface_hub._snapshot_download import snapshot_download +from huggingface_hub.commands import BaseHuggingfaceCLICommand +from huggingface_hub.file_download import hf_hub_download +from huggingface_hub.utils import disable_progress_bars, enable_progress_bars + +from ._cli_utils import show_deprecation_warning + + +logger = logging.get_logger(__name__) + + +class DownloadCommand(BaseHuggingfaceCLICommand): + @staticmethod + def register_subcommand(parser: _SubParsersAction): + download_parser = parser.add_parser("download", help="Download files from the Hub") + download_parser.add_argument( + "repo_id", type=str, help="ID of the repo to download from (e.g. `username/repo-name`)." + ) + download_parser.add_argument( + "filenames", type=str, nargs="*", help="Files to download (e.g. `config.json`, `data/metadata.jsonl`)." + ) + download_parser.add_argument( + "--repo-type", + choices=["model", "dataset", "space"], + default="model", + help="Type of repo to download from (defaults to 'model').", + ) + download_parser.add_argument( + "--revision", + type=str, + help="An optional Git revision id which can be a branch name, a tag, or a commit hash.", + ) + download_parser.add_argument( + "--include", nargs="*", type=str, help="Glob patterns to match files to download." + ) + download_parser.add_argument( + "--exclude", nargs="*", type=str, help="Glob patterns to exclude from files to download." + ) + download_parser.add_argument( + "--cache-dir", type=str, help="Path to the directory where to save the downloaded files." + ) + download_parser.add_argument( + "--local-dir", + type=str, + help=( + "If set, the downloaded file will be placed under this directory. Check out" + " https://huggingface.co/docs/huggingface_hub/guides/download#download-files-to-local-folder for more" + " details." + ), + ) + download_parser.add_argument( + "--local-dir-use-symlinks", + choices=["auto", "True", "False"], + help=("Deprecated and ignored. Downloading to a local directory does not use symlinks anymore."), + ) + download_parser.add_argument( + "--force-download", + action="store_true", + help="If True, the files will be downloaded even if they are already cached.", + ) + download_parser.add_argument( + "--resume-download", + action="store_true", + help="Deprecated and ignored. Downloading a file to local dir always attempts to resume previously interrupted downloads (unless hf-transfer is enabled).", + ) + download_parser.add_argument( + "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens" + ) + download_parser.add_argument( + "--quiet", + action="store_true", + help="If True, progress bars are disabled and only the path to the download files is printed.", + ) + download_parser.add_argument( + "--max-workers", + type=int, + default=8, + help="Maximum number of workers to use for downloading files. Default is 8.", + ) + download_parser.set_defaults(func=DownloadCommand) + + def __init__(self, args: Namespace) -> None: + self.token = args.token + self.repo_id: str = args.repo_id + self.filenames: List[str] = args.filenames + self.repo_type: str = args.repo_type + self.revision: Optional[str] = args.revision + self.include: Optional[List[str]] = args.include + self.exclude: Optional[List[str]] = args.exclude + self.cache_dir: Optional[str] = args.cache_dir + self.local_dir: Optional[str] = args.local_dir + self.force_download: bool = args.force_download + self.resume_download: Optional[bool] = args.resume_download or None + self.quiet: bool = args.quiet + self.max_workers: int = args.max_workers + + if args.local_dir_use_symlinks is not None: + warnings.warn( + "Ignoring --local-dir-use-symlinks. Downloading to a local directory does not use symlinks anymore.", + FutureWarning, + ) + + def run(self) -> None: + show_deprecation_warning("huggingface-cli download", "hf download") + + if self.quiet: + disable_progress_bars() + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + print(self._download()) # Print path to downloaded files + enable_progress_bars() + else: + logging.set_verbosity_info() + print(self._download()) # Print path to downloaded files + logging.set_verbosity_warning() + + def _download(self) -> str: + # Warn user if patterns are ignored + if len(self.filenames) > 0: + if self.include is not None and len(self.include) > 0: + warnings.warn("Ignoring `--include` since filenames have being explicitly set.") + if self.exclude is not None and len(self.exclude) > 0: + warnings.warn("Ignoring `--exclude` since filenames have being explicitly set.") + + # Single file to download: use `hf_hub_download` + if len(self.filenames) == 1: + return hf_hub_download( + repo_id=self.repo_id, + repo_type=self.repo_type, + revision=self.revision, + filename=self.filenames[0], + cache_dir=self.cache_dir, + resume_download=self.resume_download, + force_download=self.force_download, + token=self.token, + local_dir=self.local_dir, + library_name="huggingface-cli", + ) + + # Otherwise: use `snapshot_download` to ensure all files comes from same revision + elif len(self.filenames) == 0: + allow_patterns = self.include + ignore_patterns = self.exclude + else: + allow_patterns = self.filenames + ignore_patterns = None + + return snapshot_download( + repo_id=self.repo_id, + repo_type=self.repo_type, + revision=self.revision, + allow_patterns=allow_patterns, + ignore_patterns=ignore_patterns, + resume_download=self.resume_download, + force_download=self.force_download, + cache_dir=self.cache_dir, + token=self.token, + local_dir=self.local_dir, + library_name="huggingface-cli", + max_workers=self.max_workers, + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/env.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/env.py new file mode 100644 index 0000000000000000000000000000000000000000..ad674738b2f137ec0b79c11ef35057a351de6d86 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/env.py @@ -0,0 +1,39 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains command to print information about the environment. + +Usage: + huggingface-cli env +""" + +from argparse import _SubParsersAction + +from ..utils import dump_environment_info +from . import BaseHuggingfaceCLICommand +from ._cli_utils import show_deprecation_warning + + +class EnvironmentCommand(BaseHuggingfaceCLICommand): + def __init__(self, args): + self.args = args + + @staticmethod + def register_subcommand(parser: _SubParsersAction): + env_parser = parser.add_parser("env", help="Print information about the environment.") + env_parser.set_defaults(func=EnvironmentCommand) + + def run(self) -> None: + show_deprecation_warning("huggingface-cli env", "hf env") + + dump_environment_info() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/huggingface_cli.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/huggingface_cli.py new file mode 100644 index 0000000000000000000000000000000000000000..697c85d1e386d9c954be0f8112cb12e1bc84e7fe --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/huggingface_cli.py @@ -0,0 +1,65 @@ +# Copyright 2020 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from argparse import ArgumentParser + +from huggingface_hub.commands._cli_utils import show_deprecation_warning +from huggingface_hub.commands.delete_cache import DeleteCacheCommand +from huggingface_hub.commands.download import DownloadCommand +from huggingface_hub.commands.env import EnvironmentCommand +from huggingface_hub.commands.lfs import LfsCommands +from huggingface_hub.commands.repo import RepoCommands +from huggingface_hub.commands.repo_files import RepoFilesCommand +from huggingface_hub.commands.scan_cache import ScanCacheCommand +from huggingface_hub.commands.tag import TagCommands +from huggingface_hub.commands.upload import UploadCommand +from huggingface_hub.commands.upload_large_folder import UploadLargeFolderCommand +from huggingface_hub.commands.user import UserCommands +from huggingface_hub.commands.version import VersionCommand + + +def main(): + parser = ArgumentParser("huggingface-cli", usage="huggingface-cli []") + commands_parser = parser.add_subparsers(help="huggingface-cli command helpers") + + # Register commands + DownloadCommand.register_subcommand(commands_parser) + UploadCommand.register_subcommand(commands_parser) + RepoFilesCommand.register_subcommand(commands_parser) + EnvironmentCommand.register_subcommand(commands_parser) + UserCommands.register_subcommand(commands_parser) + RepoCommands.register_subcommand(commands_parser) + LfsCommands.register_subcommand(commands_parser) + ScanCacheCommand.register_subcommand(commands_parser) + DeleteCacheCommand.register_subcommand(commands_parser) + TagCommands.register_subcommand(commands_parser) + VersionCommand.register_subcommand(commands_parser) + + # Experimental + UploadLargeFolderCommand.register_subcommand(commands_parser) + + # Let's go + args = parser.parse_args() + if not hasattr(args, "func"): + show_deprecation_warning("huggingface-cli", "hf") + parser.print_help() + exit(1) + + # Run + service = args.func(args) + service.run() + + +if __name__ == "__main__": + main() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/lfs.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/lfs.py new file mode 100644 index 0000000000000000000000000000000000000000..e510e345e6a4bf6da03f71b35cbfa2a4f0eb7325 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/lfs.py @@ -0,0 +1,200 @@ +""" +Implementation of a custom transfer agent for the transfer type "multipart" for +git-lfs. + +Inspired by: +github.com/cbartz/git-lfs-swift-transfer-agent/blob/master/git_lfs_swift_transfer.py + +Spec is: github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md + + +To launch debugger while developing: + +``` [lfs "customtransfer.multipart"] +path = /path/to/huggingface_hub/.env/bin/python args = -m debugpy --listen 5678 +--wait-for-client +/path/to/huggingface_hub/src/huggingface_hub/commands/huggingface_cli.py +lfs-multipart-upload ```""" + +import json +import os +import subprocess +import sys +from argparse import _SubParsersAction +from typing import Dict, List, Optional + +from huggingface_hub.commands import BaseHuggingfaceCLICommand +from huggingface_hub.lfs import LFS_MULTIPART_UPLOAD_COMMAND + +from ..utils import get_session, hf_raise_for_status, logging +from ..utils._lfs import SliceFileObj + + +logger = logging.get_logger(__name__) + + +class LfsCommands(BaseHuggingfaceCLICommand): + """ + Implementation of a custom transfer agent for the transfer type "multipart" + for git-lfs. This lets users upload large files >5GB 🔥. Spec for LFS custom + transfer agent is: + https://github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md + + This introduces two commands to the CLI: + + 1. $ huggingface-cli lfs-enable-largefiles + + This should be executed once for each model repo that contains a model file + >5GB. It's documented in the error message you get if you just try to git + push a 5GB file without having enabled it before. + + 2. $ huggingface-cli lfs-multipart-upload + + This command is called by lfs directly and is not meant to be called by the + user. + """ + + @staticmethod + def register_subcommand(parser: _SubParsersAction): + enable_parser = parser.add_parser( + "lfs-enable-largefiles", help="Configure your repository to enable upload of files > 5GB." + ) + enable_parser.add_argument("path", type=str, help="Local path to repository you want to configure.") + enable_parser.set_defaults(func=lambda args: LfsEnableCommand(args)) + + # Command will get called by git-lfs, do not call it directly. + upload_parser = parser.add_parser(LFS_MULTIPART_UPLOAD_COMMAND, add_help=False) + upload_parser.set_defaults(func=lambda args: LfsUploadCommand(args)) + + +class LfsEnableCommand: + def __init__(self, args): + self.args = args + + def run(self): + local_path = os.path.abspath(self.args.path) + if not os.path.isdir(local_path): + print("This does not look like a valid git repo.") + exit(1) + subprocess.run( + "git config lfs.customtransfer.multipart.path huggingface-cli".split(), + check=True, + cwd=local_path, + ) + subprocess.run( + f"git config lfs.customtransfer.multipart.args {LFS_MULTIPART_UPLOAD_COMMAND}".split(), + check=True, + cwd=local_path, + ) + print("Local repo set up for largefiles") + + +def write_msg(msg: Dict): + """Write out the message in Line delimited JSON.""" + msg_str = json.dumps(msg) + "\n" + sys.stdout.write(msg_str) + sys.stdout.flush() + + +def read_msg() -> Optional[Dict]: + """Read Line delimited JSON from stdin.""" + msg = json.loads(sys.stdin.readline().strip()) + + if "terminate" in (msg.get("type"), msg.get("event")): + # terminate message received + return None + + if msg.get("event") not in ("download", "upload"): + logger.critical("Received unexpected message") + sys.exit(1) + + return msg + + +class LfsUploadCommand: + def __init__(self, args) -> None: + self.args = args + + def run(self) -> None: + # Immediately after invoking a custom transfer process, git-lfs + # sends initiation data to the process over stdin. + # This tells the process useful information about the configuration. + init_msg = json.loads(sys.stdin.readline().strip()) + if not (init_msg.get("event") == "init" and init_msg.get("operation") == "upload"): + write_msg({"error": {"code": 32, "message": "Wrong lfs init operation"}}) + sys.exit(1) + + # The transfer process should use the information it needs from the + # initiation structure, and also perform any one-off setup tasks it + # needs to do. It should then respond on stdout with a simple empty + # confirmation structure, as follows: + write_msg({}) + + # After the initiation exchange, git-lfs will send any number of + # transfer requests to the stdin of the transfer process, in a serial sequence. + while True: + msg = read_msg() + if msg is None: + # When all transfers have been processed, git-lfs will send + # a terminate event to the stdin of the transfer process. + # On receiving this message the transfer process should + # clean up and terminate. No response is expected. + sys.exit(0) + + oid = msg["oid"] + filepath = msg["path"] + completion_url = msg["action"]["href"] + header = msg["action"]["header"] + chunk_size = int(header.pop("chunk_size")) + presigned_urls: List[str] = list(header.values()) + + # Send a "started" progress event to allow other workers to start. + # Otherwise they're delayed until first "progress" event is reported, + # i.e. after the first 5GB by default (!) + write_msg( + { + "event": "progress", + "oid": oid, + "bytesSoFar": 1, + "bytesSinceLast": 0, + } + ) + + parts = [] + with open(filepath, "rb") as file: + for i, presigned_url in enumerate(presigned_urls): + with SliceFileObj( + file, + seek_from=i * chunk_size, + read_limit=chunk_size, + ) as data: + r = get_session().put(presigned_url, data=data) + hf_raise_for_status(r) + parts.append( + { + "etag": r.headers.get("etag"), + "partNumber": i + 1, + } + ) + # In order to support progress reporting while data is uploading / downloading, + # the transfer process should post messages to stdout + write_msg( + { + "event": "progress", + "oid": oid, + "bytesSoFar": (i + 1) * chunk_size, + "bytesSinceLast": chunk_size, + } + ) + # Not precise but that's ok. + + r = get_session().post( + completion_url, + json={ + "oid": oid, + "parts": parts, + }, + ) + hf_raise_for_status(r) + + write_msg({"event": "complete", "oid": oid}) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/repo.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/repo.py new file mode 100644 index 0000000000000000000000000000000000000000..fe75349d67bdc0314afe737daa7224b2a090f810 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/repo.py @@ -0,0 +1,151 @@ +# Copyright 2025 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains commands to interact with repositories on the Hugging Face Hub. + +Usage: + # create a new dataset repo on the Hub + huggingface-cli repo create my-cool-dataset --repo-type=dataset + + # create a private model repo on the Hub + huggingface-cli repo create my-cool-model --private +""" + +import argparse +from argparse import _SubParsersAction +from typing import Optional + +from huggingface_hub.commands import BaseHuggingfaceCLICommand +from huggingface_hub.commands._cli_utils import ANSI +from huggingface_hub.constants import SPACES_SDK_TYPES +from huggingface_hub.hf_api import HfApi +from huggingface_hub.utils import logging + +from ._cli_utils import show_deprecation_warning + + +logger = logging.get_logger(__name__) + + +class RepoCommands(BaseHuggingfaceCLICommand): + @staticmethod + def register_subcommand(parser: _SubParsersAction): + repo_parser = parser.add_parser("repo", help="{create} Commands to interact with your huggingface.co repos.") + repo_subparsers = repo_parser.add_subparsers(help="huggingface.co repos related commands") + repo_create_parser = repo_subparsers.add_parser("create", help="Create a new repo on huggingface.co") + repo_create_parser.add_argument( + "repo_id", + type=str, + help="The ID of the repo to create to (e.g. `username/repo-name`). The username is optional and will be set to your username if not provided.", + ) + repo_create_parser.add_argument( + "--repo-type", + type=str, + help='Optional: set to "dataset" or "space" if creating a dataset or space, default is model.', + ) + repo_create_parser.add_argument( + "--space_sdk", + type=str, + help='Optional: Hugging Face Spaces SDK type. Required when --type is set to "space".', + choices=SPACES_SDK_TYPES, + ) + repo_create_parser.add_argument( + "--private", + action="store_true", + help="Whether to create a private repository. Defaults to public unless the organization's default is private.", + ) + repo_create_parser.add_argument( + "--token", + type=str, + help="Hugging Face token. Will default to the locally saved token if not provided.", + ) + repo_create_parser.add_argument( + "--exist-ok", + action="store_true", + help="Do not raise an error if repo already exists.", + ) + repo_create_parser.add_argument( + "--resource-group-id", + type=str, + help="Resource group in which to create the repo. Resource groups is only available for Enterprise Hub organizations.", + ) + repo_create_parser.add_argument( + "--type", + type=str, + help="[Deprecated]: use --repo-type instead.", + ) + repo_create_parser.add_argument( + "-y", + "--yes", + action="store_true", + help="[Deprecated] no effect.", + ) + repo_create_parser.add_argument( + "--organization", type=str, help="[Deprecated] Pass the organization namespace directly in the repo_id." + ) + repo_create_parser.set_defaults(func=lambda args: RepoCreateCommand(args)) + + +class RepoCreateCommand: + def __init__(self, args: argparse.Namespace): + self.repo_id: str = args.repo_id + self.repo_type: Optional[str] = args.repo_type or args.type + self.space_sdk: Optional[str] = args.space_sdk + self.organization: Optional[str] = args.organization + self.yes: bool = args.yes + self.private: bool = args.private + self.token: Optional[str] = args.token + self.exist_ok: bool = args.exist_ok + self.resource_group_id: Optional[str] = args.resource_group_id + + if args.type is not None: + print( + ANSI.yellow( + "The --type argument is deprecated and will be removed in a future version. Use --repo-type instead." + ) + ) + if self.organization is not None: + print( + ANSI.yellow( + "The --organization argument is deprecated and will be removed in a future version. Pass the organization namespace directly in the repo_id." + ) + ) + if self.yes: + print( + ANSI.yellow( + "The --yes argument is deprecated and will be removed in a future version. It does not have any effect." + ) + ) + + self._api = HfApi() + + def run(self): + show_deprecation_warning("huggingface-cli repo", "hf repo") + + if self.organization is not None: + if "/" in self.repo_id: + print(ANSI.red("You cannot pass both --organization and a repo_id with a namespace.")) + exit(1) + self.repo_id = f"{self.organization}/{self.repo_id}" + + repo_url = self._api.create_repo( + repo_id=self.repo_id, + repo_type=self.repo_type, + private=self.private, + token=self.token, + exist_ok=self.exist_ok, + resource_group_id=self.resource_group_id, + space_sdk=self.space_sdk, + ) + print(f"Successfully created {ANSI.bold(repo_url.repo_id)} on the Hub.") + print(f"Your repo is now available at {ANSI.bold(repo_url)}") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/repo_files.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/repo_files.py new file mode 100644 index 0000000000000000000000000000000000000000..da9685315ea67dc9d1e9921ecb2656244cae8783 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/repo_files.py @@ -0,0 +1,132 @@ +# coding=utf-8 +# Copyright 2023-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains command to update or delete files in a repository using the CLI. + +Usage: + # delete all + huggingface-cli repo-files delete "*" + + # delete single file + huggingface-cli repo-files delete file.txt + + # delete single folder + huggingface-cli repo-files delete folder/ + + # delete multiple + huggingface-cli repo-files delete file.txt folder/ file2.txt + + # delete multiple patterns + huggingface-cli repo-files delete file.txt "*.json" "folder/*.parquet" + + # delete from different revision / repo-type + huggingface-cli repo-files delete file.txt --revision=refs/pr/1 --repo-type=dataset +""" + +from argparse import _SubParsersAction +from typing import List, Optional + +from huggingface_hub import logging +from huggingface_hub.commands import BaseHuggingfaceCLICommand +from huggingface_hub.hf_api import HfApi + +from ._cli_utils import show_deprecation_warning + + +logger = logging.get_logger(__name__) + + +class DeleteFilesSubCommand: + def __init__(self, args) -> None: + self.args = args + self.repo_id: str = args.repo_id + self.repo_type: Optional[str] = args.repo_type + self.revision: Optional[str] = args.revision + self.api: HfApi = HfApi(token=args.token, library_name="huggingface-cli") + self.patterns: List[str] = args.patterns + self.commit_message: Optional[str] = args.commit_message + self.commit_description: Optional[str] = args.commit_description + self.create_pr: bool = args.create_pr + self.token: Optional[str] = args.token + + def run(self) -> None: + show_deprecation_warning("huggingface-cli repo-files", "hf repo-files") + + logging.set_verbosity_info() + url = self.api.delete_files( + delete_patterns=self.patterns, + repo_id=self.repo_id, + repo_type=self.repo_type, + revision=self.revision, + commit_message=self.commit_message, + commit_description=self.commit_description, + create_pr=self.create_pr, + ) + print(f"Files correctly deleted from repo. Commit: {url}.") + logging.set_verbosity_warning() + + +class RepoFilesCommand(BaseHuggingfaceCLICommand): + @staticmethod + def register_subcommand(parser: _SubParsersAction): + repo_files_parser = parser.add_parser("repo-files", help="Manage files in a repo on the Hub") + repo_files_parser.add_argument( + "repo_id", type=str, help="The ID of the repo to manage (e.g. `username/repo-name`)." + ) + repo_files_subparsers = repo_files_parser.add_subparsers( + help="Action to execute against the files.", + required=True, + ) + delete_subparser = repo_files_subparsers.add_parser( + "delete", + help="Delete files from a repo on the Hub", + ) + delete_subparser.set_defaults(func=lambda args: DeleteFilesSubCommand(args)) + delete_subparser.add_argument( + "patterns", + nargs="+", + type=str, + help="Glob patterns to match files to delete.", + ) + delete_subparser.add_argument( + "--repo-type", + choices=["model", "dataset", "space"], + default="model", + help="Type of the repo to upload to (e.g. `dataset`).", + ) + delete_subparser.add_argument( + "--revision", + type=str, + help=( + "An optional Git revision to push to. It can be a branch name " + "or a PR reference. If revision does not" + " exist and `--create-pr` is not set, a branch will be automatically created." + ), + ) + delete_subparser.add_argument( + "--commit-message", type=str, help="The summary / title / first line of the generated commit." + ) + delete_subparser.add_argument( + "--commit-description", type=str, help="The description of the generated commit." + ) + delete_subparser.add_argument( + "--create-pr", action="store_true", help="Whether to create a new Pull Request for these changes." + ) + repo_files_parser.add_argument( + "--token", + type=str, + help="A User Access Token generated from https://huggingface.co/settings/tokens", + ) + + repo_files_parser.set_defaults(func=RepoFilesCommand) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/scan_cache.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/scan_cache.py new file mode 100644 index 0000000000000000000000000000000000000000..711a5d09cc2b64b9c7f22a298e26a198b4dc48f1 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/scan_cache.py @@ -0,0 +1,183 @@ +# coding=utf-8 +# Copyright 2022-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains command to scan the HF cache directory. + +Usage: + huggingface-cli scan-cache + huggingface-cli scan-cache -v + huggingface-cli scan-cache -vvv + huggingface-cli scan-cache --dir ~/.cache/huggingface/hub +""" + +import time +from argparse import Namespace, _SubParsersAction +from typing import Optional + +from ..utils import CacheNotFound, HFCacheInfo, scan_cache_dir +from . import BaseHuggingfaceCLICommand +from ._cli_utils import ANSI, show_deprecation_warning, tabulate + + +class ScanCacheCommand(BaseHuggingfaceCLICommand): + @staticmethod + def register_subcommand(parser: _SubParsersAction): + scan_cache_parser = parser.add_parser("scan-cache", help="Scan cache directory.") + + scan_cache_parser.add_argument( + "--dir", + type=str, + default=None, + help="cache directory to scan (optional). Default to the default HuggingFace cache.", + ) + scan_cache_parser.add_argument( + "-v", + "--verbose", + action="count", + default=0, + help="show a more verbose output", + ) + scan_cache_parser.set_defaults(func=ScanCacheCommand) + + def __init__(self, args: Namespace) -> None: + self.verbosity: int = args.verbose + self.cache_dir: Optional[str] = args.dir + + def run(self): + show_deprecation_warning("huggingface-cli scan-cache", "hf cache scan") + + try: + t0 = time.time() + hf_cache_info = scan_cache_dir(self.cache_dir) + t1 = time.time() + except CacheNotFound as exc: + cache_dir = exc.cache_dir + print(f"Cache directory not found: {cache_dir}") + return + + self._print_hf_cache_info_as_table(hf_cache_info) + + print( + f"\nDone in {round(t1 - t0, 1)}s. Scanned {len(hf_cache_info.repos)} repo(s)" + f" for a total of {ANSI.red(hf_cache_info.size_on_disk_str)}." + ) + if len(hf_cache_info.warnings) > 0: + message = f"Got {len(hf_cache_info.warnings)} warning(s) while scanning." + if self.verbosity >= 3: + print(ANSI.gray(message)) + for warning in hf_cache_info.warnings: + print(ANSI.gray(str(warning))) + else: + print(ANSI.gray(message + " Use -vvv to print details.")) + + def _print_hf_cache_info_as_table(self, hf_cache_info: HFCacheInfo) -> None: + print(get_table(hf_cache_info, verbosity=self.verbosity)) + + +def get_table(hf_cache_info: HFCacheInfo, *, verbosity: int = 0) -> str: + """Generate a table from the [`HFCacheInfo`] object. + + Pass `verbosity=0` to get a table with a single row per repo, with columns + "repo_id", "repo_type", "size_on_disk", "nb_files", "last_accessed", "last_modified", "refs", "local_path". + + Pass `verbosity=1` to get a table with a row per repo and revision (thus multiple rows can appear for a single repo), with columns + "repo_id", "repo_type", "revision", "size_on_disk", "nb_files", "last_modified", "refs", "local_path". + + Example: + ```py + >>> from huggingface_hub.utils import scan_cache_dir + >>> from huggingface_hub.commands.scan_cache import get_table + + >>> hf_cache_info = scan_cache_dir() + HFCacheInfo(...) + + >>> print(get_table(hf_cache_info, verbosity=0)) + REPO ID REPO TYPE SIZE ON DISK NB FILES LAST_ACCESSED LAST_MODIFIED REFS LOCAL PATH + --------------------------------------------------- --------- ------------ -------- ------------- ------------- ---- -------------------------------------------------------------------------------------------------- + roberta-base model 2.7M 5 1 day ago 1 week ago main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--roberta-base + suno/bark model 8.8K 1 1 week ago 1 week ago main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--suno--bark + t5-base model 893.8M 4 4 days ago 7 months ago main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--t5-base + t5-large model 3.0G 4 5 weeks ago 5 months ago main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--t5-large + + >>> print(get_table(hf_cache_info, verbosity=1)) + REPO ID REPO TYPE REVISION SIZE ON DISK NB FILES LAST_MODIFIED REFS LOCAL PATH + --------------------------------------------------- --------- ---------------------------------------- ------------ -------- ------------- ---- ----------------------------------------------------------------------------------------------------------------------------------------------------- + roberta-base model e2da8e2f811d1448a5b465c236feacd80ffbac7b 2.7M 5 1 week ago main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--roberta-base\\snapshots\\e2da8e2f811d1448a5b465c236feacd80ffbac7b + suno/bark model 70a8a7d34168586dc5d028fa9666aceade177992 8.8K 1 1 week ago main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--suno--bark\\snapshots\\70a8a7d34168586dc5d028fa9666aceade177992 + t5-base model a9723ea7f1b39c1eae772870f3b547bf6ef7e6c1 893.8M 4 7 months ago main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--t5-base\\snapshots\\a9723ea7f1b39c1eae772870f3b547bf6ef7e6c1 + t5-large model 150ebc2c4b72291e770f58e6057481c8d2ed331a 3.0G 4 5 months ago main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--t5-large\\snapshots\\150ebc2c4b72291e770f58e6057481c8d2ed331a ``` + ``` + + Args: + hf_cache_info ([`HFCacheInfo`]): + The HFCacheInfo object to print. + verbosity (`int`, *optional*): + The verbosity level. Defaults to 0. + + Returns: + `str`: The table as a string. + """ + if verbosity == 0: + return tabulate( + rows=[ + [ + repo.repo_id, + repo.repo_type, + "{:>12}".format(repo.size_on_disk_str), + repo.nb_files, + repo.last_accessed_str, + repo.last_modified_str, + ", ".join(sorted(repo.refs)), + str(repo.repo_path), + ] + for repo in sorted(hf_cache_info.repos, key=lambda repo: repo.repo_path) + ], + headers=[ + "REPO ID", + "REPO TYPE", + "SIZE ON DISK", + "NB FILES", + "LAST_ACCESSED", + "LAST_MODIFIED", + "REFS", + "LOCAL PATH", + ], + ) + else: + return tabulate( + rows=[ + [ + repo.repo_id, + repo.repo_type, + revision.commit_hash, + "{:>12}".format(revision.size_on_disk_str), + revision.nb_files, + revision.last_modified_str, + ", ".join(sorted(revision.refs)), + str(revision.snapshot_path), + ] + for repo in sorted(hf_cache_info.repos, key=lambda repo: repo.repo_path) + for revision in sorted(repo.revisions, key=lambda revision: revision.commit_hash) + ], + headers=[ + "REPO ID", + "REPO TYPE", + "REVISION", + "SIZE ON DISK", + "NB FILES", + "LAST_MODIFIED", + "REFS", + "LOCAL PATH", + ], + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/tag.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/tag.py new file mode 100644 index 0000000000000000000000000000000000000000..405d407f8135d940cf078f905a6e66acd4b1dacc --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/tag.py @@ -0,0 +1,161 @@ +# coding=utf-8 +# Copyright 2024-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Contains commands to perform tag management with the CLI. + +Usage Examples: + - Create a tag: + $ huggingface-cli tag user/my-model 1.0 --message "First release" + $ huggingface-cli tag user/my-model 1.0 -m "First release" --revision develop + $ huggingface-cli tag user/my-dataset 1.0 -m "First release" --repo-type dataset + $ huggingface-cli tag user/my-space 1.0 + - List all tags: + $ huggingface-cli tag -l user/my-model + $ huggingface-cli tag --list user/my-dataset --repo-type dataset + - Delete a tag: + $ huggingface-cli tag -d user/my-model 1.0 + $ huggingface-cli tag --delete user/my-dataset 1.0 --repo-type dataset + $ huggingface-cli tag -d user/my-space 1.0 -y +""" + +from argparse import Namespace, _SubParsersAction + +from requests.exceptions import HTTPError + +from huggingface_hub.commands import BaseHuggingfaceCLICommand +from huggingface_hub.constants import ( + REPO_TYPES, +) +from huggingface_hub.hf_api import HfApi + +from ..errors import HfHubHTTPError, RepositoryNotFoundError, RevisionNotFoundError +from ._cli_utils import ANSI, show_deprecation_warning + + +class TagCommands(BaseHuggingfaceCLICommand): + @staticmethod + def register_subcommand(parser: _SubParsersAction): + tag_parser = parser.add_parser("tag", help="(create, list, delete) tags for a repo in the hub") + + tag_parser.add_argument("repo_id", type=str, help="The ID of the repo to tag (e.g. `username/repo-name`).") + tag_parser.add_argument("tag", nargs="?", type=str, help="The name of the tag for creation or deletion.") + tag_parser.add_argument("-m", "--message", type=str, help="The description of the tag to create.") + tag_parser.add_argument("--revision", type=str, help="The git revision to tag.") + tag_parser.add_argument( + "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens." + ) + tag_parser.add_argument( + "--repo-type", + choices=["model", "dataset", "space"], + default="model", + help="Set the type of repository (model, dataset, or space).", + ) + tag_parser.add_argument("-y", "--yes", action="store_true", help="Answer Yes to prompts automatically.") + + tag_parser.add_argument("-l", "--list", action="store_true", help="List tags for a repository.") + tag_parser.add_argument("-d", "--delete", action="store_true", help="Delete a tag for a repository.") + + tag_parser.set_defaults(func=lambda args: handle_commands(args)) + + +def handle_commands(args: Namespace): + show_deprecation_warning("huggingface-cli tag", "hf repo tag") + + if args.list: + return TagListCommand(args) + elif args.delete: + return TagDeleteCommand(args) + else: + return TagCreateCommand(args) + + +class TagCommand: + def __init__(self, args: Namespace): + self.args = args + self.api = HfApi(token=self.args.token) + self.repo_id = self.args.repo_id + self.repo_type = self.args.repo_type + if self.repo_type not in REPO_TYPES: + print("Invalid repo --repo-type") + exit(1) + + +class TagCreateCommand(TagCommand): + def run(self): + print(f"You are about to create tag {ANSI.bold(self.args.tag)} on {self.repo_type} {ANSI.bold(self.repo_id)}") + + try: + self.api.create_tag( + repo_id=self.repo_id, + tag=self.args.tag, + tag_message=self.args.message, + revision=self.args.revision, + repo_type=self.repo_type, + ) + except RepositoryNotFoundError: + print(f"{self.repo_type.capitalize()} {ANSI.bold(self.repo_id)} not found.") + exit(1) + except RevisionNotFoundError: + print(f"Revision {ANSI.bold(self.args.revision)} not found.") + exit(1) + except HfHubHTTPError as e: + if e.response.status_code == 409: + print(f"Tag {ANSI.bold(self.args.tag)} already exists on {ANSI.bold(self.repo_id)}") + exit(1) + raise e + + print(f"Tag {ANSI.bold(self.args.tag)} created on {ANSI.bold(self.repo_id)}") + + +class TagListCommand(TagCommand): + def run(self): + try: + refs = self.api.list_repo_refs( + repo_id=self.repo_id, + repo_type=self.repo_type, + ) + except RepositoryNotFoundError: + print(f"{self.repo_type.capitalize()} {ANSI.bold(self.repo_id)} not found.") + exit(1) + except HTTPError as e: + print(e) + print(ANSI.red(e.response.text)) + exit(1) + if len(refs.tags) == 0: + print("No tags found") + exit(0) + print(f"Tags for {self.repo_type} {ANSI.bold(self.repo_id)}:") + for tag in refs.tags: + print(tag.name) + + +class TagDeleteCommand(TagCommand): + def run(self): + print(f"You are about to delete tag {ANSI.bold(self.args.tag)} on {self.repo_type} {ANSI.bold(self.repo_id)}") + + if not self.args.yes: + choice = input("Proceed? [Y/n] ").lower() + if choice not in ("", "y", "yes"): + print("Abort") + exit() + try: + self.api.delete_tag(repo_id=self.repo_id, tag=self.args.tag, repo_type=self.repo_type) + except RepositoryNotFoundError: + print(f"{self.repo_type.capitalize()} {ANSI.bold(self.repo_id)} not found.") + exit(1) + except RevisionNotFoundError: + print(f"Tag {ANSI.bold(self.args.tag)} not found on {ANSI.bold(self.repo_id)}") + exit(1) + print(f"Tag {ANSI.bold(self.args.tag)} deleted on {ANSI.bold(self.repo_id)}") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/upload.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/upload.py new file mode 100644 index 0000000000000000000000000000000000000000..c778555cda56eb17c905f0728fef6712acc75cb8 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/upload.py @@ -0,0 +1,318 @@ +# coding=utf-8 +# Copyright 2023-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains command to upload a repo or file with the CLI. + +Usage: + # Upload file (implicit) + huggingface-cli upload my-cool-model ./my-cool-model.safetensors + + # Upload file (explicit) + huggingface-cli upload my-cool-model ./my-cool-model.safetensors model.safetensors + + # Upload directory (implicit). If `my-cool-model/` is a directory it will be uploaded, otherwise an exception is raised. + huggingface-cli upload my-cool-model + + # Upload directory (explicit) + huggingface-cli upload my-cool-model ./models/my-cool-model . + + # Upload filtered directory (example: tensorboard logs except for the last run) + huggingface-cli upload my-cool-model ./model/training /logs --include "*.tfevents.*" --exclude "*20230905*" + + # Upload with wildcard + huggingface-cli upload my-cool-model "./model/training/*.safetensors" + + # Upload private dataset + huggingface-cli upload Wauplin/my-cool-dataset ./data . --repo-type=dataset --private + + # Upload with token + huggingface-cli upload Wauplin/my-cool-model --token=hf_**** + + # Sync local Space with Hub (upload new files, delete removed files) + huggingface-cli upload Wauplin/space-example --repo-type=space --exclude="/logs/*" --delete="*" --commit-message="Sync local Space with Hub" + + # Schedule commits every 30 minutes + huggingface-cli upload Wauplin/my-cool-model --every=30 +""" + +import os +import time +import warnings +from argparse import Namespace, _SubParsersAction +from typing import List, Optional + +from huggingface_hub import logging +from huggingface_hub._commit_scheduler import CommitScheduler +from huggingface_hub.commands import BaseHuggingfaceCLICommand +from huggingface_hub.constants import HF_HUB_ENABLE_HF_TRANSFER +from huggingface_hub.errors import RevisionNotFoundError +from huggingface_hub.hf_api import HfApi +from huggingface_hub.utils import disable_progress_bars, enable_progress_bars +from huggingface_hub.utils._runtime import is_xet_available + +from ._cli_utils import show_deprecation_warning + + +logger = logging.get_logger(__name__) + + +class UploadCommand(BaseHuggingfaceCLICommand): + @staticmethod + def register_subcommand(parser: _SubParsersAction): + upload_parser = parser.add_parser("upload", help="Upload a file or a folder to a repo on the Hub") + upload_parser.add_argument( + "repo_id", type=str, help="The ID of the repo to upload to (e.g. `username/repo-name`)." + ) + upload_parser.add_argument( + "local_path", + nargs="?", + help="Local path to the file or folder to upload. Wildcard patterns are supported. Defaults to current directory.", + ) + upload_parser.add_argument( + "path_in_repo", + nargs="?", + help="Path of the file or folder in the repo. Defaults to the relative path of the file or folder.", + ) + upload_parser.add_argument( + "--repo-type", + choices=["model", "dataset", "space"], + default="model", + help="Type of the repo to upload to (e.g. `dataset`).", + ) + upload_parser.add_argument( + "--revision", + type=str, + help=( + "An optional Git revision to push to. It can be a branch name or a PR reference. If revision does not" + " exist and `--create-pr` is not set, a branch will be automatically created." + ), + ) + upload_parser.add_argument( + "--private", + action="store_true", + help=( + "Whether to create a private repo if repo doesn't exist on the Hub. Ignored if the repo already" + " exists." + ), + ) + upload_parser.add_argument("--include", nargs="*", type=str, help="Glob patterns to match files to upload.") + upload_parser.add_argument( + "--exclude", nargs="*", type=str, help="Glob patterns to exclude from files to upload." + ) + upload_parser.add_argument( + "--delete", + nargs="*", + type=str, + help="Glob patterns for file to be deleted from the repo while committing.", + ) + upload_parser.add_argument( + "--commit-message", type=str, help="The summary / title / first line of the generated commit." + ) + upload_parser.add_argument("--commit-description", type=str, help="The description of the generated commit.") + upload_parser.add_argument( + "--create-pr", action="store_true", help="Whether to upload content as a new Pull Request." + ) + upload_parser.add_argument( + "--every", + type=float, + help="If set, a background job is scheduled to create commits every `every` minutes.", + ) + upload_parser.add_argument( + "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens" + ) + upload_parser.add_argument( + "--quiet", + action="store_true", + help="If True, progress bars are disabled and only the path to the uploaded files is printed.", + ) + upload_parser.set_defaults(func=UploadCommand) + + def __init__(self, args: Namespace) -> None: + self.repo_id: str = args.repo_id + self.repo_type: Optional[str] = args.repo_type + self.revision: Optional[str] = args.revision + self.private: bool = args.private + + self.include: Optional[List[str]] = args.include + self.exclude: Optional[List[str]] = args.exclude + self.delete: Optional[List[str]] = args.delete + + self.commit_message: Optional[str] = args.commit_message + self.commit_description: Optional[str] = args.commit_description + self.create_pr: bool = args.create_pr + self.api: HfApi = HfApi(token=args.token, library_name="huggingface-cli") + self.quiet: bool = args.quiet # disable warnings and progress bars + + # Check `--every` is valid + if args.every is not None and args.every <= 0: + raise ValueError(f"`every` must be a positive value (got '{args.every}')") + self.every: Optional[float] = args.every + + # Resolve `local_path` and `path_in_repo` + repo_name: str = args.repo_id.split("/")[-1] # e.g. "Wauplin/my-cool-model" => "my-cool-model" + self.local_path: str + self.path_in_repo: str + + if args.local_path is not None and any(c in args.local_path for c in ["*", "?", "["]): + if args.include is not None: + raise ValueError("Cannot set `--include` when passing a `local_path` containing a wildcard.") + if args.path_in_repo is not None and args.path_in_repo != ".": + raise ValueError("Cannot set `path_in_repo` when passing a `local_path` containing a wildcard.") + self.local_path = "." + self.include = args.local_path + self.path_in_repo = "." + elif args.local_path is None and os.path.isfile(repo_name): + # Implicit case 1: user provided only a repo_id which happen to be a local file as well => upload it with same name + self.local_path = repo_name + self.path_in_repo = repo_name + elif args.local_path is None and os.path.isdir(repo_name): + # Implicit case 2: user provided only a repo_id which happen to be a local folder as well => upload it at root + self.local_path = repo_name + self.path_in_repo = "." + elif args.local_path is None: + # Implicit case 3: user provided only a repo_id that does not match a local file or folder + # => the user must explicitly provide a local_path => raise exception + raise ValueError(f"'{repo_name}' is not a local file or folder. Please set `local_path` explicitly.") + elif args.path_in_repo is None and os.path.isfile(args.local_path): + # Explicit local path to file, no path in repo => upload it at root with same name + self.local_path = args.local_path + self.path_in_repo = os.path.basename(args.local_path) + elif args.path_in_repo is None: + # Explicit local path to folder, no path in repo => upload at root + self.local_path = args.local_path + self.path_in_repo = "." + else: + # Finally, if both paths are explicit + self.local_path = args.local_path + self.path_in_repo = args.path_in_repo + + def run(self) -> None: + show_deprecation_warning("huggingface-cli upload", "hf upload") + + if self.quiet: + disable_progress_bars() + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + print(self._upload()) + enable_progress_bars() + else: + logging.set_verbosity_info() + print(self._upload()) + logging.set_verbosity_warning() + + def _upload(self) -> str: + if os.path.isfile(self.local_path): + if self.include is not None and len(self.include) > 0: + warnings.warn("Ignoring `--include` since a single file is uploaded.") + if self.exclude is not None and len(self.exclude) > 0: + warnings.warn("Ignoring `--exclude` since a single file is uploaded.") + if self.delete is not None and len(self.delete) > 0: + warnings.warn("Ignoring `--delete` since a single file is uploaded.") + + if not is_xet_available() and not HF_HUB_ENABLE_HF_TRANSFER: + logger.info( + "Consider using `hf_transfer` for faster uploads. This solution comes with some limitations. See" + " https://huggingface.co/docs/huggingface_hub/hf_transfer for more details." + ) + + # Schedule commits if `every` is set + if self.every is not None: + if os.path.isfile(self.local_path): + # If file => watch entire folder + use allow_patterns + folder_path = os.path.dirname(self.local_path) + path_in_repo = ( + self.path_in_repo[: -len(self.local_path)] # remove filename from path_in_repo + if self.path_in_repo.endswith(self.local_path) + else self.path_in_repo + ) + allow_patterns = [self.local_path] + ignore_patterns = [] + else: + folder_path = self.local_path + path_in_repo = self.path_in_repo + allow_patterns = self.include or [] + ignore_patterns = self.exclude or [] + if self.delete is not None and len(self.delete) > 0: + warnings.warn("Ignoring `--delete` when uploading with scheduled commits.") + + scheduler = CommitScheduler( + folder_path=folder_path, + repo_id=self.repo_id, + repo_type=self.repo_type, + revision=self.revision, + allow_patterns=allow_patterns, + ignore_patterns=ignore_patterns, + path_in_repo=path_in_repo, + private=self.private, + every=self.every, + hf_api=self.api, + ) + print(f"Scheduling commits every {self.every} minutes to {scheduler.repo_id}.") + try: # Block main thread until KeyboardInterrupt + while True: + time.sleep(100) + except KeyboardInterrupt: + scheduler.stop() + return "Stopped scheduled commits." + + # Otherwise, create repo and proceed with the upload + if not os.path.isfile(self.local_path) and not os.path.isdir(self.local_path): + raise FileNotFoundError(f"No such file or directory: '{self.local_path}'.") + repo_id = self.api.create_repo( + repo_id=self.repo_id, + repo_type=self.repo_type, + exist_ok=True, + private=self.private, + space_sdk="gradio" if self.repo_type == "space" else None, + # ^ We don't want it to fail when uploading to a Space => let's set Gradio by default. + # ^ I'd rather not add CLI args to set it explicitly as we already have `huggingface-cli repo create` for that. + ).repo_id + + # Check if branch already exists and if not, create it + if self.revision is not None and not self.create_pr: + try: + self.api.repo_info(repo_id=repo_id, repo_type=self.repo_type, revision=self.revision) + except RevisionNotFoundError: + logger.info(f"Branch '{self.revision}' not found. Creating it...") + self.api.create_branch(repo_id=repo_id, repo_type=self.repo_type, branch=self.revision, exist_ok=True) + # ^ `exist_ok=True` to avoid race concurrency issues + + # File-based upload + if os.path.isfile(self.local_path): + return self.api.upload_file( + path_or_fileobj=self.local_path, + path_in_repo=self.path_in_repo, + repo_id=repo_id, + repo_type=self.repo_type, + revision=self.revision, + commit_message=self.commit_message, + commit_description=self.commit_description, + create_pr=self.create_pr, + ) + + # Folder-based upload + else: + return self.api.upload_folder( + folder_path=self.local_path, + path_in_repo=self.path_in_repo, + repo_id=repo_id, + repo_type=self.repo_type, + revision=self.revision, + commit_message=self.commit_message, + commit_description=self.commit_description, + create_pr=self.create_pr, + allow_patterns=self.include, + ignore_patterns=self.exclude, + delete_patterns=self.delete, + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/upload_large_folder.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/upload_large_folder.py new file mode 100644 index 0000000000000000000000000000000000000000..3105ba3f57f5644aa18e627aa5d1d18e61515ae7 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/upload_large_folder.py @@ -0,0 +1,131 @@ +# coding=utf-8 +# Copyright 2023-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains command to upload a large folder with the CLI.""" + +import os +from argparse import Namespace, _SubParsersAction +from typing import List, Optional + +from huggingface_hub import logging +from huggingface_hub.commands import BaseHuggingfaceCLICommand +from huggingface_hub.hf_api import HfApi +from huggingface_hub.utils import disable_progress_bars + +from ._cli_utils import ANSI, show_deprecation_warning + + +logger = logging.get_logger(__name__) + + +class UploadLargeFolderCommand(BaseHuggingfaceCLICommand): + @staticmethod + def register_subcommand(parser: _SubParsersAction): + subparser = parser.add_parser("upload-large-folder", help="Upload a large folder to a repo on the Hub") + subparser.add_argument( + "repo_id", type=str, help="The ID of the repo to upload to (e.g. `username/repo-name`)." + ) + subparser.add_argument("local_path", type=str, help="Local path to the file or folder to upload.") + subparser.add_argument( + "--repo-type", + choices=["model", "dataset", "space"], + help="Type of the repo to upload to (e.g. `dataset`).", + ) + subparser.add_argument( + "--revision", + type=str, + help=("An optional Git revision to push to. It can be a branch name or a PR reference."), + ) + subparser.add_argument( + "--private", + action="store_true", + help=( + "Whether to create a private repo if repo doesn't exist on the Hub. Ignored if the repo already exists." + ), + ) + subparser.add_argument("--include", nargs="*", type=str, help="Glob patterns to match files to upload.") + subparser.add_argument("--exclude", nargs="*", type=str, help="Glob patterns to exclude from files to upload.") + subparser.add_argument( + "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens" + ) + subparser.add_argument( + "--num-workers", type=int, help="Number of workers to use to hash, upload and commit files." + ) + subparser.add_argument("--no-report", action="store_true", help="Whether to disable regular status report.") + subparser.add_argument("--no-bars", action="store_true", help="Whether to disable progress bars.") + subparser.set_defaults(func=UploadLargeFolderCommand) + + def __init__(self, args: Namespace) -> None: + self.repo_id: str = args.repo_id + self.local_path: str = args.local_path + self.repo_type: str = args.repo_type + self.revision: Optional[str] = args.revision + self.private: bool = args.private + + self.include: Optional[List[str]] = args.include + self.exclude: Optional[List[str]] = args.exclude + + self.api: HfApi = HfApi(token=args.token, library_name="huggingface-cli") + + self.num_workers: Optional[int] = args.num_workers + self.no_report: bool = args.no_report + self.no_bars: bool = args.no_bars + + if not os.path.isdir(self.local_path): + raise ValueError("Large upload is only supported for folders.") + + def run(self) -> None: + show_deprecation_warning("huggingface-cli upload-large-folder", "hf upload-large-folder") + + logging.set_verbosity_info() + + print( + ANSI.yellow( + "You are about to upload a large folder to the Hub using `huggingface-cli upload-large-folder`. " + "This is a new feature so feedback is very welcome!\n" + "\n" + "A few things to keep in mind:\n" + " - Repository limits still apply: https://huggingface.co/docs/hub/repositories-recommendations\n" + " - Do not start several processes in parallel.\n" + " - You can interrupt and resume the process at any time. " + "The script will pick up where it left off except for partially uploaded files that would have to be entirely reuploaded.\n" + " - Do not upload the same folder to several repositories. If you need to do so, you must delete the `./.cache/huggingface/` folder first.\n" + "\n" + f"Some temporary metadata will be stored under `{self.local_path}/.cache/huggingface`.\n" + " - You must not modify those files manually.\n" + " - You must not delete the `./.cache/huggingface/` folder while a process is running.\n" + " - You can delete the `./.cache/huggingface/` folder to reinitialize the upload state when process is not running. Files will have to be hashed and preuploaded again, except for already committed files.\n" + "\n" + "If the process output is too verbose, you can disable the progress bars with `--no-bars`. " + "You can also entirely disable the status report with `--no-report`.\n" + "\n" + "For more details, run `huggingface-cli upload-large-folder --help` or check the documentation at " + "https://huggingface.co/docs/huggingface_hub/guides/upload#upload-a-large-folder." + ) + ) + + if self.no_bars: + disable_progress_bars() + + self.api.upload_large_folder( + repo_id=self.repo_id, + folder_path=self.local_path, + repo_type=self.repo_type, + revision=self.revision, + private=self.private, + allow_patterns=self.include, + ignore_patterns=self.exclude, + num_workers=self.num_workers, + print_report=not self.no_report, + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/user.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/user.py new file mode 100644 index 0000000000000000000000000000000000000000..3f4da0f45d0dae5bc4458f844f776db9c3971208 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/user.py @@ -0,0 +1,208 @@ +# Copyright 2020 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains commands to authenticate to the Hugging Face Hub and interact with your repositories. + +Usage: + # login and save token locally. + huggingface-cli login --token=hf_*** --add-to-git-credential + + # switch between tokens + huggingface-cli auth switch + + # list all tokens + huggingface-cli auth list + + # logout from a specific token, if no token-name is provided, all tokens will be deleted from your machine. + huggingface-cli logout --token-name=your_token_name + + # find out which huggingface.co account you are logged in as + huggingface-cli whoami +""" + +from argparse import _SubParsersAction +from typing import List, Optional + +from requests.exceptions import HTTPError + +from huggingface_hub.commands import BaseHuggingfaceCLICommand +from huggingface_hub.constants import ENDPOINT +from huggingface_hub.hf_api import HfApi + +from .._login import auth_list, auth_switch, login, logout +from ..utils import get_stored_tokens, get_token, logging +from ._cli_utils import ANSI, show_deprecation_warning + + +logger = logging.get_logger(__name__) + +try: + from InquirerPy import inquirer + from InquirerPy.base.control import Choice + + _inquirer_py_available = True +except ImportError: + _inquirer_py_available = False + + +class UserCommands(BaseHuggingfaceCLICommand): + @staticmethod + def register_subcommand(parser: _SubParsersAction): + login_parser = parser.add_parser("login", help="Log in using a token from huggingface.co/settings/tokens") + login_parser.add_argument( + "--token", + type=str, + help="Token generated from https://huggingface.co/settings/tokens", + ) + login_parser.add_argument( + "--add-to-git-credential", + action="store_true", + help="Optional: Save token to git credential helper.", + ) + login_parser.set_defaults(func=lambda args: LoginCommand(args)) + whoami_parser = parser.add_parser("whoami", help="Find out which huggingface.co account you are logged in as.") + whoami_parser.set_defaults(func=lambda args: WhoamiCommand(args)) + + logout_parser = parser.add_parser("logout", help="Log out") + logout_parser.add_argument( + "--token-name", + type=str, + help="Optional: Name of the access token to log out from.", + ) + logout_parser.set_defaults(func=lambda args: LogoutCommand(args)) + + auth_parser = parser.add_parser("auth", help="Other authentication related commands") + auth_subparsers = auth_parser.add_subparsers(help="Authentication subcommands") + auth_switch_parser = auth_subparsers.add_parser("switch", help="Switch between access tokens") + auth_switch_parser.add_argument( + "--token-name", + type=str, + help="Optional: Name of the access token to switch to.", + ) + auth_switch_parser.add_argument( + "--add-to-git-credential", + action="store_true", + help="Optional: Save token to git credential helper.", + ) + auth_switch_parser.set_defaults(func=lambda args: AuthSwitchCommand(args)) + auth_list_parser = auth_subparsers.add_parser("list", help="List all stored access tokens") + auth_list_parser.set_defaults(func=lambda args: AuthListCommand(args)) + + +class BaseUserCommand: + def __init__(self, args): + self.args = args + self._api = HfApi() + + +class LoginCommand(BaseUserCommand): + def run(self): + show_deprecation_warning("huggingface-cli login", "hf auth login") + + logging.set_verbosity_info() + login( + token=self.args.token, + add_to_git_credential=self.args.add_to_git_credential, + ) + + +class LogoutCommand(BaseUserCommand): + def run(self): + show_deprecation_warning("huggingface-cli logout", "hf auth logout") + + logging.set_verbosity_info() + logout(token_name=self.args.token_name) + + +class AuthSwitchCommand(BaseUserCommand): + def run(self): + show_deprecation_warning("huggingface-cli auth switch", "hf auth switch") + + logging.set_verbosity_info() + token_name = self.args.token_name + if token_name is None: + token_name = self._select_token_name() + + if token_name is None: + print("No token name provided. Aborting.") + exit() + auth_switch(token_name, add_to_git_credential=self.args.add_to_git_credential) + + def _select_token_name(self) -> Optional[str]: + token_names = list(get_stored_tokens().keys()) + + if not token_names: + logger.error("No stored tokens found. Please login first.") + return None + + if _inquirer_py_available: + return self._select_token_name_tui(token_names) + # if inquirer is not available, use a simpler terminal UI + print("Available stored tokens:") + for i, token_name in enumerate(token_names, 1): + print(f"{i}. {token_name}") + while True: + try: + choice = input("Enter the number of the token to switch to (or 'q' to quit): ") + if choice.lower() == "q": + return None + index = int(choice) - 1 + if 0 <= index < len(token_names): + return token_names[index] + else: + print("Invalid selection. Please try again.") + except ValueError: + print("Invalid input. Please enter a number or 'q' to quit.") + + def _select_token_name_tui(self, token_names: List[str]) -> Optional[str]: + choices = [Choice(token_name, name=token_name) for token_name in token_names] + try: + return inquirer.select( + message="Select a token to switch to:", + choices=choices, + default=None, + ).execute() + except KeyboardInterrupt: + logger.info("Token selection cancelled.") + return None + + +class AuthListCommand(BaseUserCommand): + def run(self): + show_deprecation_warning("huggingface-cli auth list", "hf auth list") + + logging.set_verbosity_info() + auth_list() + + +class WhoamiCommand(BaseUserCommand): + def run(self): + show_deprecation_warning("huggingface-cli whoami", "hf auth whoami") + + token = get_token() + if token is None: + print("Not logged in") + exit() + try: + info = self._api.whoami(token) + print(info["name"]) + orgs = [org["name"] for org in info["orgs"]] + if orgs: + print(ANSI.bold("orgs: "), ",".join(orgs)) + + if ENDPOINT != "https://huggingface.co": + print(f"Authenticated through private endpoint: {ENDPOINT}") + except HTTPError as e: + print(e) + print(ANSI.red(e.response.text)) + exit(1) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/version.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/version.py new file mode 100644 index 0000000000000000000000000000000000000000..10d341bcdb93e0616fcf80370ac8dde63b15ce9c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/commands/version.py @@ -0,0 +1,40 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains command to print information about the version. + +Usage: + huggingface-cli version +""" + +from argparse import _SubParsersAction + +from huggingface_hub import __version__ + +from . import BaseHuggingfaceCLICommand +from ._cli_utils import show_deprecation_warning + + +class VersionCommand(BaseHuggingfaceCLICommand): + def __init__(self, args): + self.args = args + + @staticmethod + def register_subcommand(parser: _SubParsersAction): + version_parser = parser.add_parser("version", help="Print information about the huggingface-cli version.") + version_parser.set_defaults(func=VersionCommand) + + def run(self) -> None: + show_deprecation_warning("huggingface-cli version", "hf version") + + print(f"huggingface_hub version: {__version__}") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0d2ba4a2a83768d8408d2ab8a469469e12664bed Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bfffc0ae3bce71532382ee87d03c40dc376cfae7 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/__init__.py @@ -0,0 +1,192 @@ +# This file is auto-generated by `utils/generate_inference_types.py`. +# Do not modify it manually. +# +# ruff: noqa: F401 + +from .audio_classification import ( + AudioClassificationInput, + AudioClassificationOutputElement, + AudioClassificationOutputTransform, + AudioClassificationParameters, +) +from .audio_to_audio import AudioToAudioInput, AudioToAudioOutputElement +from .automatic_speech_recognition import ( + AutomaticSpeechRecognitionEarlyStoppingEnum, + AutomaticSpeechRecognitionGenerationParameters, + AutomaticSpeechRecognitionInput, + AutomaticSpeechRecognitionOutput, + AutomaticSpeechRecognitionOutputChunk, + AutomaticSpeechRecognitionParameters, +) +from .base import BaseInferenceType +from .chat_completion import ( + ChatCompletionInput, + ChatCompletionInputFunctionDefinition, + ChatCompletionInputFunctionName, + ChatCompletionInputGrammarType, + ChatCompletionInputJSONSchema, + ChatCompletionInputMessage, + ChatCompletionInputMessageChunk, + ChatCompletionInputMessageChunkType, + ChatCompletionInputResponseFormatJSONObject, + ChatCompletionInputResponseFormatJSONSchema, + ChatCompletionInputResponseFormatText, + ChatCompletionInputStreamOptions, + ChatCompletionInputTool, + ChatCompletionInputToolCall, + ChatCompletionInputToolChoiceClass, + ChatCompletionInputToolChoiceEnum, + ChatCompletionInputURL, + ChatCompletionOutput, + ChatCompletionOutputComplete, + ChatCompletionOutputFunctionDefinition, + ChatCompletionOutputLogprob, + ChatCompletionOutputLogprobs, + ChatCompletionOutputMessage, + ChatCompletionOutputToolCall, + ChatCompletionOutputTopLogprob, + ChatCompletionOutputUsage, + ChatCompletionStreamOutput, + ChatCompletionStreamOutputChoice, + ChatCompletionStreamOutputDelta, + ChatCompletionStreamOutputDeltaToolCall, + ChatCompletionStreamOutputFunction, + ChatCompletionStreamOutputLogprob, + ChatCompletionStreamOutputLogprobs, + ChatCompletionStreamOutputTopLogprob, + ChatCompletionStreamOutputUsage, +) +from .depth_estimation import DepthEstimationInput, DepthEstimationOutput +from .document_question_answering import ( + DocumentQuestionAnsweringInput, + DocumentQuestionAnsweringInputData, + DocumentQuestionAnsweringOutputElement, + DocumentQuestionAnsweringParameters, +) +from .feature_extraction import FeatureExtractionInput, FeatureExtractionInputTruncationDirection +from .fill_mask import FillMaskInput, FillMaskOutputElement, FillMaskParameters +from .image_classification import ( + ImageClassificationInput, + ImageClassificationOutputElement, + ImageClassificationOutputTransform, + ImageClassificationParameters, +) +from .image_segmentation import ( + ImageSegmentationInput, + ImageSegmentationOutputElement, + ImageSegmentationParameters, + ImageSegmentationSubtask, +) +from .image_to_image import ImageToImageInput, ImageToImageOutput, ImageToImageParameters, ImageToImageTargetSize +from .image_to_text import ( + ImageToTextEarlyStoppingEnum, + ImageToTextGenerationParameters, + ImageToTextInput, + ImageToTextOutput, + ImageToTextParameters, +) +from .image_to_video import ImageToVideoInput, ImageToVideoOutput, ImageToVideoParameters, ImageToVideoTargetSize +from .object_detection import ( + ObjectDetectionBoundingBox, + ObjectDetectionInput, + ObjectDetectionOutputElement, + ObjectDetectionParameters, +) +from .question_answering import ( + QuestionAnsweringInput, + QuestionAnsweringInputData, + QuestionAnsweringOutputElement, + QuestionAnsweringParameters, +) +from .sentence_similarity import SentenceSimilarityInput, SentenceSimilarityInputData +from .summarization import ( + SummarizationInput, + SummarizationOutput, + SummarizationParameters, + SummarizationTruncationStrategy, +) +from .table_question_answering import ( + Padding, + TableQuestionAnsweringInput, + TableQuestionAnsweringInputData, + TableQuestionAnsweringOutputElement, + TableQuestionAnsweringParameters, +) +from .text2text_generation import ( + Text2TextGenerationInput, + Text2TextGenerationOutput, + Text2TextGenerationParameters, + Text2TextGenerationTruncationStrategy, +) +from .text_classification import ( + TextClassificationInput, + TextClassificationOutputElement, + TextClassificationOutputTransform, + TextClassificationParameters, +) +from .text_generation import ( + TextGenerationInput, + TextGenerationInputGenerateParameters, + TextGenerationInputGrammarType, + TextGenerationOutput, + TextGenerationOutputBestOfSequence, + TextGenerationOutputDetails, + TextGenerationOutputFinishReason, + TextGenerationOutputPrefillToken, + TextGenerationOutputToken, + TextGenerationStreamOutput, + TextGenerationStreamOutputStreamDetails, + TextGenerationStreamOutputToken, + TypeEnum, +) +from .text_to_audio import ( + TextToAudioEarlyStoppingEnum, + TextToAudioGenerationParameters, + TextToAudioInput, + TextToAudioOutput, + TextToAudioParameters, +) +from .text_to_image import TextToImageInput, TextToImageOutput, TextToImageParameters +from .text_to_speech import ( + TextToSpeechEarlyStoppingEnum, + TextToSpeechGenerationParameters, + TextToSpeechInput, + TextToSpeechOutput, + TextToSpeechParameters, +) +from .text_to_video import TextToVideoInput, TextToVideoOutput, TextToVideoParameters +from .token_classification import ( + TokenClassificationAggregationStrategy, + TokenClassificationInput, + TokenClassificationOutputElement, + TokenClassificationParameters, +) +from .translation import TranslationInput, TranslationOutput, TranslationParameters, TranslationTruncationStrategy +from .video_classification import ( + VideoClassificationInput, + VideoClassificationOutputElement, + VideoClassificationOutputTransform, + VideoClassificationParameters, +) +from .visual_question_answering import ( + VisualQuestionAnsweringInput, + VisualQuestionAnsweringInputData, + VisualQuestionAnsweringOutputElement, + VisualQuestionAnsweringParameters, +) +from .zero_shot_classification import ( + ZeroShotClassificationInput, + ZeroShotClassificationOutputElement, + ZeroShotClassificationParameters, +) +from .zero_shot_image_classification import ( + ZeroShotImageClassificationInput, + ZeroShotImageClassificationOutputElement, + ZeroShotImageClassificationParameters, +) +from .zero_shot_object_detection import ( + ZeroShotObjectDetectionBoundingBox, + ZeroShotObjectDetectionInput, + ZeroShotObjectDetectionOutputElement, + ZeroShotObjectDetectionParameters, +) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/audio_classification.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/audio_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..053055787bce933e1fbd393cfbc00d81c43c8c2d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/audio_classification.py @@ -0,0 +1,43 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Literal, Optional + +from .base import BaseInferenceType, dataclass_with_extra + + +AudioClassificationOutputTransform = Literal["sigmoid", "softmax", "none"] + + +@dataclass_with_extra +class AudioClassificationParameters(BaseInferenceType): + """Additional inference parameters for Audio Classification""" + + function_to_apply: Optional["AudioClassificationOutputTransform"] = None + """The function to apply to the model outputs in order to retrieve the scores.""" + top_k: Optional[int] = None + """When specified, limits the output to the top K most probable classes.""" + + +@dataclass_with_extra +class AudioClassificationInput(BaseInferenceType): + """Inputs for Audio Classification inference""" + + inputs: str + """The input audio data as a base64-encoded string. If no `parameters` are provided, you can + also provide the audio data as a raw bytes payload. + """ + parameters: Optional[AudioClassificationParameters] = None + """Additional inference parameters for Audio Classification""" + + +@dataclass_with_extra +class AudioClassificationOutputElement(BaseInferenceType): + """Outputs for Audio Classification inference""" + + label: str + """The predicted class label.""" + score: float + """The corresponding probability.""" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/audio_to_audio.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/audio_to_audio.py new file mode 100644 index 0000000000000000000000000000000000000000..43f376b5345fab6b854b028d1c17416c020d7bc1 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/audio_to_audio.py @@ -0,0 +1,30 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any + +from .base import BaseInferenceType, dataclass_with_extra + + +@dataclass_with_extra +class AudioToAudioInput(BaseInferenceType): + """Inputs for Audio to Audio inference""" + + inputs: Any + """The input audio data""" + + +@dataclass_with_extra +class AudioToAudioOutputElement(BaseInferenceType): + """Outputs of inference for the Audio To Audio task + A generated audio file with its label. + """ + + blob: Any + """The generated audio file.""" + content_type: str + """The content type of audio file.""" + label: str + """The label of the audio file.""" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/automatic_speech_recognition.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/automatic_speech_recognition.py new file mode 100644 index 0000000000000000000000000000000000000000..f6bfd28256c82309b160f337aba5a54e2dd11872 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/automatic_speech_recognition.py @@ -0,0 +1,113 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import List, Literal, Optional, Union + +from .base import BaseInferenceType, dataclass_with_extra + + +AutomaticSpeechRecognitionEarlyStoppingEnum = Literal["never"] + + +@dataclass_with_extra +class AutomaticSpeechRecognitionGenerationParameters(BaseInferenceType): + """Parametrization of the text generation process""" + + do_sample: Optional[bool] = None + """Whether to use sampling instead of greedy decoding when generating new tokens.""" + early_stopping: Optional[Union[bool, "AutomaticSpeechRecognitionEarlyStoppingEnum"]] = None + """Controls the stopping condition for beam-based methods.""" + epsilon_cutoff: Optional[float] = None + """If set to float strictly between 0 and 1, only tokens with a conditional probability + greater than epsilon_cutoff will be sampled. In the paper, suggested values range from + 3e-4 to 9e-4, depending on the size of the model. See [Truncation Sampling as Language + Model Desmoothing](https://hf.co/papers/2210.15191) for more details. + """ + eta_cutoff: Optional[float] = None + """Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to + float strictly between 0 and 1, a token is only considered if it is greater than either + eta_cutoff or sqrt(eta_cutoff) * exp(-entropy(softmax(next_token_logits))). The latter + term is intuitively the expected next token probability, scaled by sqrt(eta_cutoff). In + the paper, suggested values range from 3e-4 to 2e-3, depending on the size of the model. + See [Truncation Sampling as Language Model Desmoothing](https://hf.co/papers/2210.15191) + for more details. + """ + max_length: Optional[int] = None + """The maximum length (in tokens) of the generated text, including the input.""" + max_new_tokens: Optional[int] = None + """The maximum number of tokens to generate. Takes precedence over max_length.""" + min_length: Optional[int] = None + """The minimum length (in tokens) of the generated text, including the input.""" + min_new_tokens: Optional[int] = None + """The minimum number of tokens to generate. Takes precedence over min_length.""" + num_beam_groups: Optional[int] = None + """Number of groups to divide num_beams into in order to ensure diversity among different + groups of beams. See [this paper](https://hf.co/papers/1610.02424) for more details. + """ + num_beams: Optional[int] = None + """Number of beams to use for beam search.""" + penalty_alpha: Optional[float] = None + """The value balances the model confidence and the degeneration penalty in contrastive + search decoding. + """ + temperature: Optional[float] = None + """The value used to modulate the next token probabilities.""" + top_k: Optional[int] = None + """The number of highest probability vocabulary tokens to keep for top-k-filtering.""" + top_p: Optional[float] = None + """If set to float < 1, only the smallest set of most probable tokens with probabilities + that add up to top_p or higher are kept for generation. + """ + typical_p: Optional[float] = None + """Local typicality measures how similar the conditional probability of predicting a target + token next is to the expected conditional probability of predicting a random token next, + given the partial text already generated. If set to float < 1, the smallest set of the + most locally typical tokens with probabilities that add up to typical_p or higher are + kept for generation. See [this paper](https://hf.co/papers/2202.00666) for more details. + """ + use_cache: Optional[bool] = None + """Whether the model should use the past last key/values attentions to speed up decoding""" + + +@dataclass_with_extra +class AutomaticSpeechRecognitionParameters(BaseInferenceType): + """Additional inference parameters for Automatic Speech Recognition""" + + generation_parameters: Optional[AutomaticSpeechRecognitionGenerationParameters] = None + """Parametrization of the text generation process""" + return_timestamps: Optional[bool] = None + """Whether to output corresponding timestamps with the generated text""" + + +@dataclass_with_extra +class AutomaticSpeechRecognitionInput(BaseInferenceType): + """Inputs for Automatic Speech Recognition inference""" + + inputs: str + """The input audio data as a base64-encoded string. If no `parameters` are provided, you can + also provide the audio data as a raw bytes payload. + """ + parameters: Optional[AutomaticSpeechRecognitionParameters] = None + """Additional inference parameters for Automatic Speech Recognition""" + + +@dataclass_with_extra +class AutomaticSpeechRecognitionOutputChunk(BaseInferenceType): + text: str + """A chunk of text identified by the model""" + timestamp: List[float] + """The start and end timestamps corresponding with the text""" + + +@dataclass_with_extra +class AutomaticSpeechRecognitionOutput(BaseInferenceType): + """Outputs of inference for the Automatic Speech Recognition task""" + + text: str + """The recognized text.""" + chunks: Optional[List[AutomaticSpeechRecognitionOutputChunk]] = None + """When returnTimestamps is enabled, chunks contains a list of audio chunks identified by + the model. + """ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/base.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/base.py new file mode 100644 index 0000000000000000000000000000000000000000..1f0c4687ceccbfb738da3f38c583c2516d065a01 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/base.py @@ -0,0 +1,161 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains a base class for all inference types.""" + +import inspect +import json +from dataclasses import asdict, dataclass +from typing import Any, Dict, List, Type, TypeVar, Union, get_args + + +T = TypeVar("T", bound="BaseInferenceType") + + +def _repr_with_extra(self): + fields = list(self.__dataclass_fields__.keys()) + other_fields = list(k for k in self.__dict__ if k not in fields) + return f"{self.__class__.__name__}({', '.join(f'{k}={self.__dict__[k]!r}' for k in fields + other_fields)})" + + +def dataclass_with_extra(cls: Type[T]) -> Type[T]: + """Decorator to add a custom __repr__ method to a dataclass, showing all fields, including extra ones. + + This decorator only works with dataclasses that inherit from `BaseInferenceType`. + """ + cls = dataclass(cls) + cls.__repr__ = _repr_with_extra # type: ignore[method-assign] + return cls + + +@dataclass +class BaseInferenceType(dict): + """Base class for all inference types. + + Object is a dataclass and a dict for backward compatibility but plan is to remove the dict part in the future. + + Handle parsing from dict, list and json strings in a permissive way to ensure future-compatibility (e.g. all fields + are made optional, and non-expected fields are added as dict attributes). + """ + + @classmethod + def parse_obj_as_list(cls: Type[T], data: Union[bytes, str, List, Dict]) -> List[T]: + """Alias to parse server response and return a single instance. + + See `parse_obj` for more details. + """ + output = cls.parse_obj(data) + if not isinstance(output, list): + raise ValueError(f"Invalid input data for {cls}. Expected a list, but got {type(output)}.") + return output + + @classmethod + def parse_obj_as_instance(cls: Type[T], data: Union[bytes, str, List, Dict]) -> T: + """Alias to parse server response and return a single instance. + + See `parse_obj` for more details. + """ + output = cls.parse_obj(data) + if isinstance(output, list): + raise ValueError(f"Invalid input data for {cls}. Expected a single instance, but got a list.") + return output + + @classmethod + def parse_obj(cls: Type[T], data: Union[bytes, str, List, Dict]) -> Union[List[T], T]: + """Parse server response as a dataclass or list of dataclasses. + + To enable future-compatibility, we want to handle cases where the server return more fields than expected. + In such cases, we don't want to raise an error but still create the dataclass object. Remaining fields are + added as dict attributes. + """ + # Parse server response (from bytes) + if isinstance(data, bytes): + data = data.decode() + if isinstance(data, str): + data = json.loads(data) + + # If a list, parse each item individually + if isinstance(data, List): + return [cls.parse_obj(d) for d in data] # type: ignore [misc] + + # At this point, we expect a dict + if not isinstance(data, dict): + raise ValueError(f"Invalid data type: {type(data)}") + + init_values = {} + other_values = {} + for key, value in data.items(): + key = normalize_key(key) + if key in cls.__dataclass_fields__ and cls.__dataclass_fields__[key].init: + if isinstance(value, dict) or isinstance(value, list): + field_type = cls.__dataclass_fields__[key].type + + # if `field_type` is a `BaseInferenceType`, parse it + if inspect.isclass(field_type) and issubclass(field_type, BaseInferenceType): + value = field_type.parse_obj(value) + + # otherwise, recursively parse nested dataclasses (if possible) + # `get_args` returns handle Union and Optional for us + else: + expected_types = get_args(field_type) + for expected_type in expected_types: + if getattr(expected_type, "_name", None) == "List": + expected_type = get_args(expected_type)[ + 0 + ] # assume same type for all items in the list + if inspect.isclass(expected_type) and issubclass(expected_type, BaseInferenceType): + value = expected_type.parse_obj(value) + break + init_values[key] = value + else: + other_values[key] = value + + # Make all missing fields default to None + # => ensure that dataclass initialization will never fail even if the server does not return all fields. + for key in cls.__dataclass_fields__: + if key not in init_values: + init_values[key] = None + + # Initialize dataclass with expected values + item = cls(**init_values) + + # Add remaining fields as dict attributes + item.update(other_values) + + # Add remaining fields as extra dataclass fields. + # They won't be part of the dataclass fields but will be accessible as attributes. + # Use @dataclass_with_extra to show them in __repr__. + item.__dict__.update(other_values) + return item + + def __post_init__(self): + self.update(asdict(self)) + + def __setitem__(self, __key: Any, __value: Any) -> None: + # Hacky way to keep dataclass values in sync when dict is updated + super().__setitem__(__key, __value) + if __key in self.__dataclass_fields__ and getattr(self, __key, None) != __value: + self.__setattr__(__key, __value) + return + + def __setattr__(self, __name: str, __value: Any) -> None: + # Hacky way to keep dict values is sync when dataclass is updated + super().__setattr__(__name, __value) + if self.get(__name) != __value: + self[__name] = __value + return + + +def normalize_key(key: str) -> str: + # e.g "content-type" -> "content_type", "Accept" -> "accept" + return key.replace("-", "_").replace(" ", "_").lower() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/chat_completion.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/chat_completion.py new file mode 100644 index 0000000000000000000000000000000000000000..ba708a7009bf14cd182a999ccf95f07ee2a002b8 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/chat_completion.py @@ -0,0 +1,347 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any, Dict, List, Literal, Optional, Union + +from .base import BaseInferenceType, dataclass_with_extra + + +@dataclass_with_extra +class ChatCompletionInputURL(BaseInferenceType): + url: str + + +ChatCompletionInputMessageChunkType = Literal["text", "image_url"] + + +@dataclass_with_extra +class ChatCompletionInputMessageChunk(BaseInferenceType): + type: "ChatCompletionInputMessageChunkType" + image_url: Optional[ChatCompletionInputURL] = None + text: Optional[str] = None + + +@dataclass_with_extra +class ChatCompletionInputFunctionDefinition(BaseInferenceType): + name: str + parameters: Any + description: Optional[str] = None + + +@dataclass_with_extra +class ChatCompletionInputToolCall(BaseInferenceType): + function: ChatCompletionInputFunctionDefinition + id: str + type: str + + +@dataclass_with_extra +class ChatCompletionInputMessage(BaseInferenceType): + role: str + content: Optional[Union[List[ChatCompletionInputMessageChunk], str]] = None + name: Optional[str] = None + tool_calls: Optional[List[ChatCompletionInputToolCall]] = None + + +@dataclass_with_extra +class ChatCompletionInputJSONSchema(BaseInferenceType): + name: str + """ + The name of the response format. + """ + description: Optional[str] = None + """ + A description of what the response format is for, used by the model to determine + how to respond in the format. + """ + schema: Optional[Dict[str, object]] = None + """ + The schema for the response format, described as a JSON Schema object. Learn how + to build JSON schemas [here](https://json-schema.org/). + """ + strict: Optional[bool] = None + """ + Whether to enable strict schema adherence when generating the output. If set to + true, the model will always follow the exact schema defined in the `schema` + field. + """ + + +@dataclass_with_extra +class ChatCompletionInputResponseFormatText(BaseInferenceType): + type: Literal["text"] + + +@dataclass_with_extra +class ChatCompletionInputResponseFormatJSONSchema(BaseInferenceType): + type: Literal["json_schema"] + json_schema: ChatCompletionInputJSONSchema + + +@dataclass_with_extra +class ChatCompletionInputResponseFormatJSONObject(BaseInferenceType): + type: Literal["json_object"] + + +ChatCompletionInputGrammarType = Union[ + ChatCompletionInputResponseFormatText, + ChatCompletionInputResponseFormatJSONSchema, + ChatCompletionInputResponseFormatJSONObject, +] + + +@dataclass_with_extra +class ChatCompletionInputStreamOptions(BaseInferenceType): + include_usage: Optional[bool] = None + """If set, an additional chunk will be streamed before the data: [DONE] message. The usage + field on this chunk shows the token usage statistics for the entire request, and the + choices field will always be an empty array. All other chunks will also include a usage + field, but with a null value. + """ + + +@dataclass_with_extra +class ChatCompletionInputFunctionName(BaseInferenceType): + name: str + + +@dataclass_with_extra +class ChatCompletionInputToolChoiceClass(BaseInferenceType): + function: ChatCompletionInputFunctionName + + +ChatCompletionInputToolChoiceEnum = Literal["auto", "none", "required"] + + +@dataclass_with_extra +class ChatCompletionInputTool(BaseInferenceType): + function: ChatCompletionInputFunctionDefinition + type: str + + +@dataclass_with_extra +class ChatCompletionInput(BaseInferenceType): + """Chat Completion Input. + Auto-generated from TGI specs. + For more details, check out + https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts. + """ + + messages: List[ChatCompletionInputMessage] + """A list of messages comprising the conversation so far.""" + frequency_penalty: Optional[float] = None + """Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing + frequency in the text so far, + decreasing the model's likelihood to repeat the same line verbatim. + """ + logit_bias: Optional[List[float]] = None + """UNUSED + Modify the likelihood of specified tokens appearing in the completion. Accepts a JSON + object that maps tokens + (specified by their token ID in the tokenizer) to an associated bias value from -100 to + 100. Mathematically, + the bias is added to the logits generated by the model prior to sampling. The exact + effect will vary per model, + but values between -1 and 1 should decrease or increase likelihood of selection; values + like -100 or 100 should + result in a ban or exclusive selection of the relevant token. + """ + logprobs: Optional[bool] = None + """Whether to return log probabilities of the output tokens or not. If true, returns the log + probabilities of each + output token returned in the content of message. + """ + max_tokens: Optional[int] = None + """The maximum number of tokens that can be generated in the chat completion.""" + model: Optional[str] = None + """[UNUSED] ID of the model to use. See the model endpoint compatibility table for details + on which models work with the Chat API. + """ + n: Optional[int] = None + """UNUSED + How many chat completion choices to generate for each input message. Note that you will + be charged based on the + number of generated tokens across all of the choices. Keep n as 1 to minimize costs. + """ + presence_penalty: Optional[float] = None + """Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they + appear in the text so far, + increasing the model's likelihood to talk about new topics + """ + response_format: Optional[ChatCompletionInputGrammarType] = None + seed: Optional[int] = None + stop: Optional[List[str]] = None + """Up to 4 sequences where the API will stop generating further tokens.""" + stream: Optional[bool] = None + stream_options: Optional[ChatCompletionInputStreamOptions] = None + temperature: Optional[float] = None + """What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the + output more random, while + lower values like 0.2 will make it more focused and deterministic. + We generally recommend altering this or `top_p` but not both. + """ + tool_choice: Optional[Union[ChatCompletionInputToolChoiceClass, "ChatCompletionInputToolChoiceEnum"]] = None + tool_prompt: Optional[str] = None + """A prompt to be appended before the tools""" + tools: Optional[List[ChatCompletionInputTool]] = None + """A list of tools the model may call. Currently, only functions are supported as a tool. + Use this to provide a list of + functions the model may generate JSON inputs for. + """ + top_logprobs: Optional[int] = None + """An integer between 0 and 5 specifying the number of most likely tokens to return at each + token position, each with + an associated log probability. logprobs must be set to true if this parameter is used. + """ + top_p: Optional[float] = None + """An alternative to sampling with temperature, called nucleus sampling, where the model + considers the results of the + tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% + probability mass are considered. + """ + + +@dataclass_with_extra +class ChatCompletionOutputTopLogprob(BaseInferenceType): + logprob: float + token: str + + +@dataclass_with_extra +class ChatCompletionOutputLogprob(BaseInferenceType): + logprob: float + token: str + top_logprobs: List[ChatCompletionOutputTopLogprob] + + +@dataclass_with_extra +class ChatCompletionOutputLogprobs(BaseInferenceType): + content: List[ChatCompletionOutputLogprob] + + +@dataclass_with_extra +class ChatCompletionOutputFunctionDefinition(BaseInferenceType): + arguments: str + name: str + description: Optional[str] = None + + +@dataclass_with_extra +class ChatCompletionOutputToolCall(BaseInferenceType): + function: ChatCompletionOutputFunctionDefinition + id: str + type: str + + +@dataclass_with_extra +class ChatCompletionOutputMessage(BaseInferenceType): + role: str + content: Optional[str] = None + reasoning: Optional[str] = None + tool_call_id: Optional[str] = None + tool_calls: Optional[List[ChatCompletionOutputToolCall]] = None + + +@dataclass_with_extra +class ChatCompletionOutputComplete(BaseInferenceType): + finish_reason: str + index: int + message: ChatCompletionOutputMessage + logprobs: Optional[ChatCompletionOutputLogprobs] = None + + +@dataclass_with_extra +class ChatCompletionOutputUsage(BaseInferenceType): + completion_tokens: int + prompt_tokens: int + total_tokens: int + + +@dataclass_with_extra +class ChatCompletionOutput(BaseInferenceType): + """Chat Completion Output. + Auto-generated from TGI specs. + For more details, check out + https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts. + """ + + choices: List[ChatCompletionOutputComplete] + created: int + id: str + model: str + system_fingerprint: str + usage: ChatCompletionOutputUsage + + +@dataclass_with_extra +class ChatCompletionStreamOutputFunction(BaseInferenceType): + arguments: str + name: Optional[str] = None + + +@dataclass_with_extra +class ChatCompletionStreamOutputDeltaToolCall(BaseInferenceType): + function: ChatCompletionStreamOutputFunction + id: str + index: int + type: str + + +@dataclass_with_extra +class ChatCompletionStreamOutputDelta(BaseInferenceType): + role: str + content: Optional[str] = None + reasoning: Optional[str] = None + tool_call_id: Optional[str] = None + tool_calls: Optional[List[ChatCompletionStreamOutputDeltaToolCall]] = None + + +@dataclass_with_extra +class ChatCompletionStreamOutputTopLogprob(BaseInferenceType): + logprob: float + token: str + + +@dataclass_with_extra +class ChatCompletionStreamOutputLogprob(BaseInferenceType): + logprob: float + token: str + top_logprobs: List[ChatCompletionStreamOutputTopLogprob] + + +@dataclass_with_extra +class ChatCompletionStreamOutputLogprobs(BaseInferenceType): + content: List[ChatCompletionStreamOutputLogprob] + + +@dataclass_with_extra +class ChatCompletionStreamOutputChoice(BaseInferenceType): + delta: ChatCompletionStreamOutputDelta + index: int + finish_reason: Optional[str] = None + logprobs: Optional[ChatCompletionStreamOutputLogprobs] = None + + +@dataclass_with_extra +class ChatCompletionStreamOutputUsage(BaseInferenceType): + completion_tokens: int + prompt_tokens: int + total_tokens: int + + +@dataclass_with_extra +class ChatCompletionStreamOutput(BaseInferenceType): + """Chat Completion Stream Output. + Auto-generated from TGI specs. + For more details, check out + https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts. + """ + + choices: List[ChatCompletionStreamOutputChoice] + created: int + id: str + model: str + system_fingerprint: str + usage: Optional[ChatCompletionStreamOutputUsage] = None diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/depth_estimation.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/depth_estimation.py new file mode 100644 index 0000000000000000000000000000000000000000..1e09bdffa194f97444e484de6e930f67ac030207 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/depth_estimation.py @@ -0,0 +1,28 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any, Dict, Optional + +from .base import BaseInferenceType, dataclass_with_extra + + +@dataclass_with_extra +class DepthEstimationInput(BaseInferenceType): + """Inputs for Depth Estimation inference""" + + inputs: Any + """The input image data""" + parameters: Optional[Dict[str, Any]] = None + """Additional inference parameters for Depth Estimation""" + + +@dataclass_with_extra +class DepthEstimationOutput(BaseInferenceType): + """Outputs of inference for the Depth Estimation task""" + + depth: Any + """The predicted depth as an image""" + predicted_depth: Any + """The predicted depth as a tensor""" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/document_question_answering.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/document_question_answering.py new file mode 100644 index 0000000000000000000000000000000000000000..2457d2c8c237f055f660e0e8291d846bb036949d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/document_question_answering.py @@ -0,0 +1,80 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any, List, Optional, Union + +from .base import BaseInferenceType, dataclass_with_extra + + +@dataclass_with_extra +class DocumentQuestionAnsweringInputData(BaseInferenceType): + """One (document, question) pair to answer""" + + image: Any + """The image on which the question is asked""" + question: str + """A question to ask of the document""" + + +@dataclass_with_extra +class DocumentQuestionAnsweringParameters(BaseInferenceType): + """Additional inference parameters for Document Question Answering""" + + doc_stride: Optional[int] = None + """If the words in the document are too long to fit with the question for the model, it will + be split in several chunks with some overlap. This argument controls the size of that + overlap. + """ + handle_impossible_answer: Optional[bool] = None + """Whether to accept impossible as an answer""" + lang: Optional[str] = None + """Language to use while running OCR. Defaults to english.""" + max_answer_len: Optional[int] = None + """The maximum length of predicted answers (e.g., only answers with a shorter length are + considered). + """ + max_question_len: Optional[int] = None + """The maximum length of the question after tokenization. It will be truncated if needed.""" + max_seq_len: Optional[int] = None + """The maximum length of the total sentence (context + question) in tokens of each chunk + passed to the model. The context will be split in several chunks (using doc_stride as + overlap) if needed. + """ + top_k: Optional[int] = None + """The number of answers to return (will be chosen by order of likelihood). Can return less + than top_k answers if there are not enough options available within the context. + """ + word_boxes: Optional[List[Union[List[float], str]]] = None + """A list of words and bounding boxes (normalized 0->1000). If provided, the inference will + skip the OCR step and use the provided bounding boxes instead. + """ + + +@dataclass_with_extra +class DocumentQuestionAnsweringInput(BaseInferenceType): + """Inputs for Document Question Answering inference""" + + inputs: DocumentQuestionAnsweringInputData + """One (document, question) pair to answer""" + parameters: Optional[DocumentQuestionAnsweringParameters] = None + """Additional inference parameters for Document Question Answering""" + + +@dataclass_with_extra +class DocumentQuestionAnsweringOutputElement(BaseInferenceType): + """Outputs of inference for the Document Question Answering task""" + + answer: str + """The answer to the question.""" + end: int + """The end word index of the answer (in the OCR’d version of the input or provided word + boxes). + """ + score: float + """The probability associated to the answer.""" + start: int + """The start word index of the answer (in the OCR’d version of the input or provided word + boxes). + """ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/feature_extraction.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/feature_extraction.py new file mode 100644 index 0000000000000000000000000000000000000000..e965ddbac2af0a5bf73e662a7c18c847611d18a1 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/feature_extraction.py @@ -0,0 +1,36 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import List, Literal, Optional, Union + +from .base import BaseInferenceType, dataclass_with_extra + + +FeatureExtractionInputTruncationDirection = Literal["Left", "Right"] + + +@dataclass_with_extra +class FeatureExtractionInput(BaseInferenceType): + """Feature Extraction Input. + Auto-generated from TEI specs. + For more details, check out + https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tei-import.ts. + """ + + inputs: Union[List[str], str] + """The text or list of texts to embed.""" + normalize: Optional[bool] = None + prompt_name: Optional[str] = None + """The name of the prompt that should be used by for encoding. If not set, no prompt + will be applied. + Must be a key in the `sentence-transformers` configuration `prompts` dictionary. + For example if ``prompt_name`` is "query" and the ``prompts`` is {"query": "query: ", + ...}, + then the sentence "What is the capital of France?" will be encoded as + "query: What is the capital of France?" because the prompt text will be prepended before + any text to encode. + """ + truncate: Optional[bool] = None + truncation_direction: Optional["FeatureExtractionInputTruncationDirection"] = None diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/fill_mask.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/fill_mask.py new file mode 100644 index 0000000000000000000000000000000000000000..dfcdc56bc507e50280d38e0f63b024ada6a7ea94 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/fill_mask.py @@ -0,0 +1,47 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any, List, Optional + +from .base import BaseInferenceType, dataclass_with_extra + + +@dataclass_with_extra +class FillMaskParameters(BaseInferenceType): + """Additional inference parameters for Fill Mask""" + + targets: Optional[List[str]] = None + """When passed, the model will limit the scores to the passed targets instead of looking up + in the whole vocabulary. If the provided targets are not in the model vocab, they will be + tokenized and the first resulting token will be used (with a warning, and that might be + slower). + """ + top_k: Optional[int] = None + """When passed, overrides the number of predictions to return.""" + + +@dataclass_with_extra +class FillMaskInput(BaseInferenceType): + """Inputs for Fill Mask inference""" + + inputs: str + """The text with masked tokens""" + parameters: Optional[FillMaskParameters] = None + """Additional inference parameters for Fill Mask""" + + +@dataclass_with_extra +class FillMaskOutputElement(BaseInferenceType): + """Outputs of inference for the Fill Mask task""" + + score: float + """The corresponding probability""" + sequence: str + """The corresponding input with the mask token prediction.""" + token: int + """The predicted token id (to replace the masked one).""" + token_str: Any + fill_mask_output_token_str: Optional[str] = None + """The predicted token (to replace the masked one).""" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/image_classification.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/image_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..0fdda6c83ff4c7aee5dc7794f0530e89d6b43047 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/image_classification.py @@ -0,0 +1,43 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Literal, Optional + +from .base import BaseInferenceType, dataclass_with_extra + + +ImageClassificationOutputTransform = Literal["sigmoid", "softmax", "none"] + + +@dataclass_with_extra +class ImageClassificationParameters(BaseInferenceType): + """Additional inference parameters for Image Classification""" + + function_to_apply: Optional["ImageClassificationOutputTransform"] = None + """The function to apply to the model outputs in order to retrieve the scores.""" + top_k: Optional[int] = None + """When specified, limits the output to the top K most probable classes.""" + + +@dataclass_with_extra +class ImageClassificationInput(BaseInferenceType): + """Inputs for Image Classification inference""" + + inputs: str + """The input image data as a base64-encoded string. If no `parameters` are provided, you can + also provide the image data as a raw bytes payload. + """ + parameters: Optional[ImageClassificationParameters] = None + """Additional inference parameters for Image Classification""" + + +@dataclass_with_extra +class ImageClassificationOutputElement(BaseInferenceType): + """Outputs of inference for the Image Classification task""" + + label: str + """The predicted class label.""" + score: float + """The corresponding probability.""" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/image_segmentation.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/image_segmentation.py new file mode 100644 index 0000000000000000000000000000000000000000..3dbf61db83ec2ae6ceafd901c4425567cd2e5b03 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/image_segmentation.py @@ -0,0 +1,51 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Literal, Optional + +from .base import BaseInferenceType, dataclass_with_extra + + +ImageSegmentationSubtask = Literal["instance", "panoptic", "semantic"] + + +@dataclass_with_extra +class ImageSegmentationParameters(BaseInferenceType): + """Additional inference parameters for Image Segmentation""" + + mask_threshold: Optional[float] = None + """Threshold to use when turning the predicted masks into binary values.""" + overlap_mask_area_threshold: Optional[float] = None + """Mask overlap threshold to eliminate small, disconnected segments.""" + subtask: Optional["ImageSegmentationSubtask"] = None + """Segmentation task to be performed, depending on model capabilities.""" + threshold: Optional[float] = None + """Probability threshold to filter out predicted masks.""" + + +@dataclass_with_extra +class ImageSegmentationInput(BaseInferenceType): + """Inputs for Image Segmentation inference""" + + inputs: str + """The input image data as a base64-encoded string. If no `parameters` are provided, you can + also provide the image data as a raw bytes payload. + """ + parameters: Optional[ImageSegmentationParameters] = None + """Additional inference parameters for Image Segmentation""" + + +@dataclass_with_extra +class ImageSegmentationOutputElement(BaseInferenceType): + """Outputs of inference for the Image Segmentation task + A predicted mask / segment + """ + + label: str + """The label of the predicted segment.""" + mask: str + """The corresponding mask as a black-and-white image (base64-encoded).""" + score: Optional[float] = None + """The score or confidence degree the model has.""" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/image_to_image.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/image_to_image.py new file mode 100644 index 0000000000000000000000000000000000000000..b14c79fedf228bb66fa88327c6d2601e77b8d6c6 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/image_to_image.py @@ -0,0 +1,60 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any, Optional + +from .base import BaseInferenceType, dataclass_with_extra + + +@dataclass_with_extra +class ImageToImageTargetSize(BaseInferenceType): + """The size in pixels of the output image. This parameter is only supported by some + providers and for specific models. It will be ignored when unsupported. + """ + + height: int + width: int + + +@dataclass_with_extra +class ImageToImageParameters(BaseInferenceType): + """Additional inference parameters for Image To Image""" + + guidance_scale: Optional[float] = None + """For diffusion models. A higher guidance scale value encourages the model to generate + images closely linked to the text prompt at the expense of lower image quality. + """ + negative_prompt: Optional[str] = None + """One prompt to guide what NOT to include in image generation.""" + num_inference_steps: Optional[int] = None + """For diffusion models. The number of denoising steps. More denoising steps usually lead to + a higher quality image at the expense of slower inference. + """ + prompt: Optional[str] = None + """The text prompt to guide the image generation.""" + target_size: Optional[ImageToImageTargetSize] = None + """The size in pixels of the output image. This parameter is only supported by some + providers and for specific models. It will be ignored when unsupported. + """ + + +@dataclass_with_extra +class ImageToImageInput(BaseInferenceType): + """Inputs for Image To Image inference""" + + inputs: str + """The input image data as a base64-encoded string. If no `parameters` are provided, you can + also provide the image data as a raw bytes payload. + """ + parameters: Optional[ImageToImageParameters] = None + """Additional inference parameters for Image To Image""" + + +@dataclass_with_extra +class ImageToImageOutput(BaseInferenceType): + """Outputs of inference for the Image To Image task""" + + image: Any + """The output image returned as raw bytes in the payload.""" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/image_to_text.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/image_to_text.py new file mode 100644 index 0000000000000000000000000000000000000000..b65e0e0068e80dbcab5a4706fb5d49be2538c4ca --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/image_to_text.py @@ -0,0 +1,100 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any, Literal, Optional, Union + +from .base import BaseInferenceType, dataclass_with_extra + + +ImageToTextEarlyStoppingEnum = Literal["never"] + + +@dataclass_with_extra +class ImageToTextGenerationParameters(BaseInferenceType): + """Parametrization of the text generation process""" + + do_sample: Optional[bool] = None + """Whether to use sampling instead of greedy decoding when generating new tokens.""" + early_stopping: Optional[Union[bool, "ImageToTextEarlyStoppingEnum"]] = None + """Controls the stopping condition for beam-based methods.""" + epsilon_cutoff: Optional[float] = None + """If set to float strictly between 0 and 1, only tokens with a conditional probability + greater than epsilon_cutoff will be sampled. In the paper, suggested values range from + 3e-4 to 9e-4, depending on the size of the model. See [Truncation Sampling as Language + Model Desmoothing](https://hf.co/papers/2210.15191) for more details. + """ + eta_cutoff: Optional[float] = None + """Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to + float strictly between 0 and 1, a token is only considered if it is greater than either + eta_cutoff or sqrt(eta_cutoff) * exp(-entropy(softmax(next_token_logits))). The latter + term is intuitively the expected next token probability, scaled by sqrt(eta_cutoff). In + the paper, suggested values range from 3e-4 to 2e-3, depending on the size of the model. + See [Truncation Sampling as Language Model Desmoothing](https://hf.co/papers/2210.15191) + for more details. + """ + max_length: Optional[int] = None + """The maximum length (in tokens) of the generated text, including the input.""" + max_new_tokens: Optional[int] = None + """The maximum number of tokens to generate. Takes precedence over max_length.""" + min_length: Optional[int] = None + """The minimum length (in tokens) of the generated text, including the input.""" + min_new_tokens: Optional[int] = None + """The minimum number of tokens to generate. Takes precedence over min_length.""" + num_beam_groups: Optional[int] = None + """Number of groups to divide num_beams into in order to ensure diversity among different + groups of beams. See [this paper](https://hf.co/papers/1610.02424) for more details. + """ + num_beams: Optional[int] = None + """Number of beams to use for beam search.""" + penalty_alpha: Optional[float] = None + """The value balances the model confidence and the degeneration penalty in contrastive + search decoding. + """ + temperature: Optional[float] = None + """The value used to modulate the next token probabilities.""" + top_k: Optional[int] = None + """The number of highest probability vocabulary tokens to keep for top-k-filtering.""" + top_p: Optional[float] = None + """If set to float < 1, only the smallest set of most probable tokens with probabilities + that add up to top_p or higher are kept for generation. + """ + typical_p: Optional[float] = None + """Local typicality measures how similar the conditional probability of predicting a target + token next is to the expected conditional probability of predicting a random token next, + given the partial text already generated. If set to float < 1, the smallest set of the + most locally typical tokens with probabilities that add up to typical_p or higher are + kept for generation. See [this paper](https://hf.co/papers/2202.00666) for more details. + """ + use_cache: Optional[bool] = None + """Whether the model should use the past last key/values attentions to speed up decoding""" + + +@dataclass_with_extra +class ImageToTextParameters(BaseInferenceType): + """Additional inference parameters for Image To Text""" + + generation_parameters: Optional[ImageToTextGenerationParameters] = None + """Parametrization of the text generation process""" + max_new_tokens: Optional[int] = None + """The amount of maximum tokens to generate.""" + + +@dataclass_with_extra +class ImageToTextInput(BaseInferenceType): + """Inputs for Image To Text inference""" + + inputs: Any + """The input image data""" + parameters: Optional[ImageToTextParameters] = None + """Additional inference parameters for Image To Text""" + + +@dataclass_with_extra +class ImageToTextOutput(BaseInferenceType): + """Outputs of inference for the Image To Text task""" + + generated_text: Any + image_to_text_output_generated_text: Optional[str] = None + """The generated text.""" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/image_to_video.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/image_to_video.py new file mode 100644 index 0000000000000000000000000000000000000000..92192a2a05b7a825c6dd55e96702fece0f3b3316 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/image_to_video.py @@ -0,0 +1,60 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any, Optional + +from .base import BaseInferenceType, dataclass_with_extra + + +@dataclass_with_extra +class ImageToVideoTargetSize(BaseInferenceType): + """The size in pixel of the output video frames.""" + + height: int + width: int + + +@dataclass_with_extra +class ImageToVideoParameters(BaseInferenceType): + """Additional inference parameters for Image To Video""" + + guidance_scale: Optional[float] = None + """For diffusion models. A higher guidance scale value encourages the model to generate + videos closely linked to the text prompt at the expense of lower image quality. + """ + negative_prompt: Optional[str] = None + """One prompt to guide what NOT to include in video generation.""" + num_frames: Optional[float] = None + """The num_frames parameter determines how many video frames are generated.""" + num_inference_steps: Optional[int] = None + """The number of denoising steps. More denoising steps usually lead to a higher quality + video at the expense of slower inference. + """ + prompt: Optional[str] = None + """The text prompt to guide the video generation.""" + seed: Optional[int] = None + """Seed for the random number generator.""" + target_size: Optional[ImageToVideoTargetSize] = None + """The size in pixel of the output video frames.""" + + +@dataclass_with_extra +class ImageToVideoInput(BaseInferenceType): + """Inputs for Image To Video inference""" + + inputs: str + """The input image data as a base64-encoded string. If no `parameters` are provided, you can + also provide the image data as a raw bytes payload. + """ + parameters: Optional[ImageToVideoParameters] = None + """Additional inference parameters for Image To Video""" + + +@dataclass_with_extra +class ImageToVideoOutput(BaseInferenceType): + """Outputs of inference for the Image To Video task""" + + video: Any + """The generated video returned as raw bytes in the payload.""" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/object_detection.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/object_detection.py new file mode 100644 index 0000000000000000000000000000000000000000..75f3ebcfe1199462d0df60879b5ba6e517f7001e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/object_detection.py @@ -0,0 +1,58 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Optional + +from .base import BaseInferenceType, dataclass_with_extra + + +@dataclass_with_extra +class ObjectDetectionParameters(BaseInferenceType): + """Additional inference parameters for Object Detection""" + + threshold: Optional[float] = None + """The probability necessary to make a prediction.""" + + +@dataclass_with_extra +class ObjectDetectionInput(BaseInferenceType): + """Inputs for Object Detection inference""" + + inputs: str + """The input image data as a base64-encoded string. If no `parameters` are provided, you can + also provide the image data as a raw bytes payload. + """ + parameters: Optional[ObjectDetectionParameters] = None + """Additional inference parameters for Object Detection""" + + +@dataclass_with_extra +class ObjectDetectionBoundingBox(BaseInferenceType): + """The predicted bounding box. Coordinates are relative to the top left corner of the input + image. + """ + + xmax: int + """The x-coordinate of the bottom-right corner of the bounding box.""" + xmin: int + """The x-coordinate of the top-left corner of the bounding box.""" + ymax: int + """The y-coordinate of the bottom-right corner of the bounding box.""" + ymin: int + """The y-coordinate of the top-left corner of the bounding box.""" + + +@dataclass_with_extra +class ObjectDetectionOutputElement(BaseInferenceType): + """Outputs of inference for the Object Detection task""" + + box: ObjectDetectionBoundingBox + """The predicted bounding box. Coordinates are relative to the top left corner of the input + image. + """ + label: str + """The predicted label for the bounding box.""" + score: float + """The associated score / probability.""" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/question_answering.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/question_answering.py new file mode 100644 index 0000000000000000000000000000000000000000..014ab41893c560a2c266bc04a1d60bc933be31c7 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/question_answering.py @@ -0,0 +1,74 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Optional + +from .base import BaseInferenceType, dataclass_with_extra + + +@dataclass_with_extra +class QuestionAnsweringInputData(BaseInferenceType): + """One (context, question) pair to answer""" + + context: str + """The context to be used for answering the question""" + question: str + """The question to be answered""" + + +@dataclass_with_extra +class QuestionAnsweringParameters(BaseInferenceType): + """Additional inference parameters for Question Answering""" + + align_to_words: Optional[bool] = None + """Attempts to align the answer to real words. Improves quality on space separated + languages. Might hurt on non-space-separated languages (like Japanese or Chinese) + """ + doc_stride: Optional[int] = None + """If the context is too long to fit with the question for the model, it will be split in + several chunks with some overlap. This argument controls the size of that overlap. + """ + handle_impossible_answer: Optional[bool] = None + """Whether to accept impossible as an answer.""" + max_answer_len: Optional[int] = None + """The maximum length of predicted answers (e.g., only answers with a shorter length are + considered). + """ + max_question_len: Optional[int] = None + """The maximum length of the question after tokenization. It will be truncated if needed.""" + max_seq_len: Optional[int] = None + """The maximum length of the total sentence (context + question) in tokens of each chunk + passed to the model. The context will be split in several chunks (using docStride as + overlap) if needed. + """ + top_k: Optional[int] = None + """The number of answers to return (will be chosen by order of likelihood). Note that we + return less than topk answers if there are not enough options available within the + context. + """ + + +@dataclass_with_extra +class QuestionAnsweringInput(BaseInferenceType): + """Inputs for Question Answering inference""" + + inputs: QuestionAnsweringInputData + """One (context, question) pair to answer""" + parameters: Optional[QuestionAnsweringParameters] = None + """Additional inference parameters for Question Answering""" + + +@dataclass_with_extra +class QuestionAnsweringOutputElement(BaseInferenceType): + """Outputs of inference for the Question Answering task""" + + answer: str + """The answer to the question.""" + end: int + """The character position in the input where the answer ends.""" + score: float + """The probability associated to the answer.""" + start: int + """The character position in the input where the answer begins.""" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/sentence_similarity.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/sentence_similarity.py new file mode 100644 index 0000000000000000000000000000000000000000..66e8bb4d9322d4847556b7a17dc17bd208a37d0c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/sentence_similarity.py @@ -0,0 +1,27 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any, Dict, List, Optional + +from .base import BaseInferenceType, dataclass_with_extra + + +@dataclass_with_extra +class SentenceSimilarityInputData(BaseInferenceType): + sentences: List[str] + """A list of strings which will be compared against the source_sentence.""" + source_sentence: str + """The string that you wish to compare the other strings with. This can be a phrase, + sentence, or longer passage, depending on the model being used. + """ + + +@dataclass_with_extra +class SentenceSimilarityInput(BaseInferenceType): + """Inputs for Sentence similarity inference""" + + inputs: SentenceSimilarityInputData + parameters: Optional[Dict[str, Any]] = None + """Additional inference parameters for Sentence Similarity""" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/summarization.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/summarization.py new file mode 100644 index 0000000000000000000000000000000000000000..33eae6fcba0e8724babf145f93be005868429c33 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/summarization.py @@ -0,0 +1,41 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any, Dict, Literal, Optional + +from .base import BaseInferenceType, dataclass_with_extra + + +SummarizationTruncationStrategy = Literal["do_not_truncate", "longest_first", "only_first", "only_second"] + + +@dataclass_with_extra +class SummarizationParameters(BaseInferenceType): + """Additional inference parameters for summarization.""" + + clean_up_tokenization_spaces: Optional[bool] = None + """Whether to clean up the potential extra spaces in the text output.""" + generate_parameters: Optional[Dict[str, Any]] = None + """Additional parametrization of the text generation algorithm.""" + truncation: Optional["SummarizationTruncationStrategy"] = None + """The truncation strategy to use.""" + + +@dataclass_with_extra +class SummarizationInput(BaseInferenceType): + """Inputs for Summarization inference""" + + inputs: str + """The input text to summarize.""" + parameters: Optional[SummarizationParameters] = None + """Additional inference parameters for summarization.""" + + +@dataclass_with_extra +class SummarizationOutput(BaseInferenceType): + """Outputs of inference for the Summarization task""" + + summary_text: str + """The summarized text.""" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/table_question_answering.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/table_question_answering.py new file mode 100644 index 0000000000000000000000000000000000000000..10e208eeeb50a689d2826a160432a2b005ec006c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/table_question_answering.py @@ -0,0 +1,62 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Dict, List, Literal, Optional + +from .base import BaseInferenceType, dataclass_with_extra + + +@dataclass_with_extra +class TableQuestionAnsweringInputData(BaseInferenceType): + """One (table, question) pair to answer""" + + question: str + """The question to be answered about the table""" + table: Dict[str, List[str]] + """The table to serve as context for the questions""" + + +Padding = Literal["do_not_pad", "longest", "max_length"] + + +@dataclass_with_extra +class TableQuestionAnsweringParameters(BaseInferenceType): + """Additional inference parameters for Table Question Answering""" + + padding: Optional["Padding"] = None + """Activates and controls padding.""" + sequential: Optional[bool] = None + """Whether to do inference sequentially or as a batch. Batching is faster, but models like + SQA require the inference to be done sequentially to extract relations within sequences, + given their conversational nature. + """ + truncation: Optional[bool] = None + """Activates and controls truncation.""" + + +@dataclass_with_extra +class TableQuestionAnsweringInput(BaseInferenceType): + """Inputs for Table Question Answering inference""" + + inputs: TableQuestionAnsweringInputData + """One (table, question) pair to answer""" + parameters: Optional[TableQuestionAnsweringParameters] = None + """Additional inference parameters for Table Question Answering""" + + +@dataclass_with_extra +class TableQuestionAnsweringOutputElement(BaseInferenceType): + """Outputs of inference for the Table Question Answering task""" + + answer: str + """The answer of the question given the table. If there is an aggregator, the answer will be + preceded by `AGGREGATOR >`. + """ + cells: List[str] + """List of strings made up of the answer cell values.""" + coordinates: List[List[int]] + """Coordinates of the cells of the answers.""" + aggregator: Optional[str] = None + """If the model has an aggregator, this returns the aggregator.""" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text2text_generation.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text2text_generation.py new file mode 100644 index 0000000000000000000000000000000000000000..34ac74e21e8a30d889f1a251f648d4c365325be6 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text2text_generation.py @@ -0,0 +1,42 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any, Dict, Literal, Optional + +from .base import BaseInferenceType, dataclass_with_extra + + +Text2TextGenerationTruncationStrategy = Literal["do_not_truncate", "longest_first", "only_first", "only_second"] + + +@dataclass_with_extra +class Text2TextGenerationParameters(BaseInferenceType): + """Additional inference parameters for Text2text Generation""" + + clean_up_tokenization_spaces: Optional[bool] = None + """Whether to clean up the potential extra spaces in the text output.""" + generate_parameters: Optional[Dict[str, Any]] = None + """Additional parametrization of the text generation algorithm""" + truncation: Optional["Text2TextGenerationTruncationStrategy"] = None + """The truncation strategy to use""" + + +@dataclass_with_extra +class Text2TextGenerationInput(BaseInferenceType): + """Inputs for Text2text Generation inference""" + + inputs: str + """The input text data""" + parameters: Optional[Text2TextGenerationParameters] = None + """Additional inference parameters for Text2text Generation""" + + +@dataclass_with_extra +class Text2TextGenerationOutput(BaseInferenceType): + """Outputs of inference for the Text2text Generation task""" + + generated_text: Any + text2_text_generation_output_generated_text: Optional[str] = None + """The generated text.""" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text_classification.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..9a172b23f844fa58f757a644d52138a18e7b6ddb --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text_classification.py @@ -0,0 +1,41 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Literal, Optional + +from .base import BaseInferenceType, dataclass_with_extra + + +TextClassificationOutputTransform = Literal["sigmoid", "softmax", "none"] + + +@dataclass_with_extra +class TextClassificationParameters(BaseInferenceType): + """Additional inference parameters for Text Classification""" + + function_to_apply: Optional["TextClassificationOutputTransform"] = None + """The function to apply to the model outputs in order to retrieve the scores.""" + top_k: Optional[int] = None + """When specified, limits the output to the top K most probable classes.""" + + +@dataclass_with_extra +class TextClassificationInput(BaseInferenceType): + """Inputs for Text Classification inference""" + + inputs: str + """The text to classify""" + parameters: Optional[TextClassificationParameters] = None + """Additional inference parameters for Text Classification""" + + +@dataclass_with_extra +class TextClassificationOutputElement(BaseInferenceType): + """Outputs of inference for the Text Classification task""" + + label: str + """The predicted class label.""" + score: float + """The corresponding probability.""" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text_generation.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text_generation.py new file mode 100644 index 0000000000000000000000000000000000000000..9b79cc691dce3a6d42aef716d4a93a719f2d600c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text_generation.py @@ -0,0 +1,168 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any, List, Literal, Optional + +from .base import BaseInferenceType, dataclass_with_extra + + +TypeEnum = Literal["json", "regex", "json_schema"] + + +@dataclass_with_extra +class TextGenerationInputGrammarType(BaseInferenceType): + type: "TypeEnum" + value: Any + """A string that represents a [JSON Schema](https://json-schema.org/). + JSON Schema is a declarative language that allows to annotate JSON documents + with types and descriptions. + """ + + +@dataclass_with_extra +class TextGenerationInputGenerateParameters(BaseInferenceType): + adapter_id: Optional[str] = None + """Lora adapter id""" + best_of: Optional[int] = None + """Generate best_of sequences and return the one if the highest token logprobs.""" + decoder_input_details: Optional[bool] = None + """Whether to return decoder input token logprobs and ids.""" + details: Optional[bool] = None + """Whether to return generation details.""" + do_sample: Optional[bool] = None + """Activate logits sampling.""" + frequency_penalty: Optional[float] = None + """The parameter for frequency penalty. 1.0 means no penalty + Penalize new tokens based on their existing frequency in the text so far, + decreasing the model's likelihood to repeat the same line verbatim. + """ + grammar: Optional[TextGenerationInputGrammarType] = None + max_new_tokens: Optional[int] = None + """Maximum number of tokens to generate.""" + repetition_penalty: Optional[float] = None + """The parameter for repetition penalty. 1.0 means no penalty. + See [this paper](https://arxiv.org/pdf/1909.05858.pdf) for more details. + """ + return_full_text: Optional[bool] = None + """Whether to prepend the prompt to the generated text""" + seed: Optional[int] = None + """Random sampling seed.""" + stop: Optional[List[str]] = None + """Stop generating tokens if a member of `stop` is generated.""" + temperature: Optional[float] = None + """The value used to module the logits distribution.""" + top_k: Optional[int] = None + """The number of highest probability vocabulary tokens to keep for top-k-filtering.""" + top_n_tokens: Optional[int] = None + """The number of highest probability vocabulary tokens to keep for top-n-filtering.""" + top_p: Optional[float] = None + """Top-p value for nucleus sampling.""" + truncate: Optional[int] = None + """Truncate inputs tokens to the given size.""" + typical_p: Optional[float] = None + """Typical Decoding mass + See [Typical Decoding for Natural Language Generation](https://arxiv.org/abs/2202.00666) + for more information. + """ + watermark: Optional[bool] = None + """Watermarking with [A Watermark for Large Language + Models](https://arxiv.org/abs/2301.10226). + """ + + +@dataclass_with_extra +class TextGenerationInput(BaseInferenceType): + """Text Generation Input. + Auto-generated from TGI specs. + For more details, check out + https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts. + """ + + inputs: str + parameters: Optional[TextGenerationInputGenerateParameters] = None + stream: Optional[bool] = None + + +TextGenerationOutputFinishReason = Literal["length", "eos_token", "stop_sequence"] + + +@dataclass_with_extra +class TextGenerationOutputPrefillToken(BaseInferenceType): + id: int + logprob: float + text: str + + +@dataclass_with_extra +class TextGenerationOutputToken(BaseInferenceType): + id: int + logprob: float + special: bool + text: str + + +@dataclass_with_extra +class TextGenerationOutputBestOfSequence(BaseInferenceType): + finish_reason: "TextGenerationOutputFinishReason" + generated_text: str + generated_tokens: int + prefill: List[TextGenerationOutputPrefillToken] + tokens: List[TextGenerationOutputToken] + seed: Optional[int] = None + top_tokens: Optional[List[List[TextGenerationOutputToken]]] = None + + +@dataclass_with_extra +class TextGenerationOutputDetails(BaseInferenceType): + finish_reason: "TextGenerationOutputFinishReason" + generated_tokens: int + prefill: List[TextGenerationOutputPrefillToken] + tokens: List[TextGenerationOutputToken] + best_of_sequences: Optional[List[TextGenerationOutputBestOfSequence]] = None + seed: Optional[int] = None + top_tokens: Optional[List[List[TextGenerationOutputToken]]] = None + + +@dataclass_with_extra +class TextGenerationOutput(BaseInferenceType): + """Text Generation Output. + Auto-generated from TGI specs. + For more details, check out + https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts. + """ + + generated_text: str + details: Optional[TextGenerationOutputDetails] = None + + +@dataclass_with_extra +class TextGenerationStreamOutputStreamDetails(BaseInferenceType): + finish_reason: "TextGenerationOutputFinishReason" + generated_tokens: int + input_length: int + seed: Optional[int] = None + + +@dataclass_with_extra +class TextGenerationStreamOutputToken(BaseInferenceType): + id: int + logprob: float + special: bool + text: str + + +@dataclass_with_extra +class TextGenerationStreamOutput(BaseInferenceType): + """Text Generation Stream Output. + Auto-generated from TGI specs. + For more details, check out + https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts. + """ + + index: int + token: TextGenerationStreamOutputToken + details: Optional[TextGenerationStreamOutputStreamDetails] = None + generated_text: Optional[str] = None + top_tokens: Optional[List[TextGenerationStreamOutputToken]] = None diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text_to_audio.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text_to_audio.py new file mode 100644 index 0000000000000000000000000000000000000000..87af80a598af70800b8386f034c65de0b397479e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text_to_audio.py @@ -0,0 +1,99 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any, Literal, Optional, Union + +from .base import BaseInferenceType, dataclass_with_extra + + +TextToAudioEarlyStoppingEnum = Literal["never"] + + +@dataclass_with_extra +class TextToAudioGenerationParameters(BaseInferenceType): + """Parametrization of the text generation process""" + + do_sample: Optional[bool] = None + """Whether to use sampling instead of greedy decoding when generating new tokens.""" + early_stopping: Optional[Union[bool, "TextToAudioEarlyStoppingEnum"]] = None + """Controls the stopping condition for beam-based methods.""" + epsilon_cutoff: Optional[float] = None + """If set to float strictly between 0 and 1, only tokens with a conditional probability + greater than epsilon_cutoff will be sampled. In the paper, suggested values range from + 3e-4 to 9e-4, depending on the size of the model. See [Truncation Sampling as Language + Model Desmoothing](https://hf.co/papers/2210.15191) for more details. + """ + eta_cutoff: Optional[float] = None + """Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to + float strictly between 0 and 1, a token is only considered if it is greater than either + eta_cutoff or sqrt(eta_cutoff) * exp(-entropy(softmax(next_token_logits))). The latter + term is intuitively the expected next token probability, scaled by sqrt(eta_cutoff). In + the paper, suggested values range from 3e-4 to 2e-3, depending on the size of the model. + See [Truncation Sampling as Language Model Desmoothing](https://hf.co/papers/2210.15191) + for more details. + """ + max_length: Optional[int] = None + """The maximum length (in tokens) of the generated text, including the input.""" + max_new_tokens: Optional[int] = None + """The maximum number of tokens to generate. Takes precedence over max_length.""" + min_length: Optional[int] = None + """The minimum length (in tokens) of the generated text, including the input.""" + min_new_tokens: Optional[int] = None + """The minimum number of tokens to generate. Takes precedence over min_length.""" + num_beam_groups: Optional[int] = None + """Number of groups to divide num_beams into in order to ensure diversity among different + groups of beams. See [this paper](https://hf.co/papers/1610.02424) for more details. + """ + num_beams: Optional[int] = None + """Number of beams to use for beam search.""" + penalty_alpha: Optional[float] = None + """The value balances the model confidence and the degeneration penalty in contrastive + search decoding. + """ + temperature: Optional[float] = None + """The value used to modulate the next token probabilities.""" + top_k: Optional[int] = None + """The number of highest probability vocabulary tokens to keep for top-k-filtering.""" + top_p: Optional[float] = None + """If set to float < 1, only the smallest set of most probable tokens with probabilities + that add up to top_p or higher are kept for generation. + """ + typical_p: Optional[float] = None + """Local typicality measures how similar the conditional probability of predicting a target + token next is to the expected conditional probability of predicting a random token next, + given the partial text already generated. If set to float < 1, the smallest set of the + most locally typical tokens with probabilities that add up to typical_p or higher are + kept for generation. See [this paper](https://hf.co/papers/2202.00666) for more details. + """ + use_cache: Optional[bool] = None + """Whether the model should use the past last key/values attentions to speed up decoding""" + + +@dataclass_with_extra +class TextToAudioParameters(BaseInferenceType): + """Additional inference parameters for Text To Audio""" + + generation_parameters: Optional[TextToAudioGenerationParameters] = None + """Parametrization of the text generation process""" + + +@dataclass_with_extra +class TextToAudioInput(BaseInferenceType): + """Inputs for Text To Audio inference""" + + inputs: str + """The input text data""" + parameters: Optional[TextToAudioParameters] = None + """Additional inference parameters for Text To Audio""" + + +@dataclass_with_extra +class TextToAudioOutput(BaseInferenceType): + """Outputs of inference for the Text To Audio task""" + + audio: Any + """The generated audio waveform.""" + sampling_rate: float + """The sampling rate of the generated audio waveform.""" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text_to_image.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text_to_image.py new file mode 100644 index 0000000000000000000000000000000000000000..20c963731371339975019ca5d40c95303d79209b --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text_to_image.py @@ -0,0 +1,50 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any, Optional + +from .base import BaseInferenceType, dataclass_with_extra + + +@dataclass_with_extra +class TextToImageParameters(BaseInferenceType): + """Additional inference parameters for Text To Image""" + + guidance_scale: Optional[float] = None + """A higher guidance scale value encourages the model to generate images closely linked to + the text prompt, but values too high may cause saturation and other artifacts. + """ + height: Optional[int] = None + """The height in pixels of the output image""" + negative_prompt: Optional[str] = None + """One prompt to guide what NOT to include in image generation.""" + num_inference_steps: Optional[int] = None + """The number of denoising steps. More denoising steps usually lead to a higher quality + image at the expense of slower inference. + """ + scheduler: Optional[str] = None + """Override the scheduler with a compatible one.""" + seed: Optional[int] = None + """Seed for the random number generator.""" + width: Optional[int] = None + """The width in pixels of the output image""" + + +@dataclass_with_extra +class TextToImageInput(BaseInferenceType): + """Inputs for Text To Image inference""" + + inputs: str + """The input text data (sometimes called "prompt")""" + parameters: Optional[TextToImageParameters] = None + """Additional inference parameters for Text To Image""" + + +@dataclass_with_extra +class TextToImageOutput(BaseInferenceType): + """Outputs of inference for the Text To Image task""" + + image: Any + """The generated image returned as raw bytes in the payload.""" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text_to_speech.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text_to_speech.py new file mode 100644 index 0000000000000000000000000000000000000000..ce2db8f3f901cc99b5d2fcbb362c4b07b2a718e0 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text_to_speech.py @@ -0,0 +1,99 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any, Literal, Optional, Union + +from .base import BaseInferenceType, dataclass_with_extra + + +TextToSpeechEarlyStoppingEnum = Literal["never"] + + +@dataclass_with_extra +class TextToSpeechGenerationParameters(BaseInferenceType): + """Parametrization of the text generation process""" + + do_sample: Optional[bool] = None + """Whether to use sampling instead of greedy decoding when generating new tokens.""" + early_stopping: Optional[Union[bool, "TextToSpeechEarlyStoppingEnum"]] = None + """Controls the stopping condition for beam-based methods.""" + epsilon_cutoff: Optional[float] = None + """If set to float strictly between 0 and 1, only tokens with a conditional probability + greater than epsilon_cutoff will be sampled. In the paper, suggested values range from + 3e-4 to 9e-4, depending on the size of the model. See [Truncation Sampling as Language + Model Desmoothing](https://hf.co/papers/2210.15191) for more details. + """ + eta_cutoff: Optional[float] = None + """Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to + float strictly between 0 and 1, a token is only considered if it is greater than either + eta_cutoff or sqrt(eta_cutoff) * exp(-entropy(softmax(next_token_logits))). The latter + term is intuitively the expected next token probability, scaled by sqrt(eta_cutoff). In + the paper, suggested values range from 3e-4 to 2e-3, depending on the size of the model. + See [Truncation Sampling as Language Model Desmoothing](https://hf.co/papers/2210.15191) + for more details. + """ + max_length: Optional[int] = None + """The maximum length (in tokens) of the generated text, including the input.""" + max_new_tokens: Optional[int] = None + """The maximum number of tokens to generate. Takes precedence over max_length.""" + min_length: Optional[int] = None + """The minimum length (in tokens) of the generated text, including the input.""" + min_new_tokens: Optional[int] = None + """The minimum number of tokens to generate. Takes precedence over min_length.""" + num_beam_groups: Optional[int] = None + """Number of groups to divide num_beams into in order to ensure diversity among different + groups of beams. See [this paper](https://hf.co/papers/1610.02424) for more details. + """ + num_beams: Optional[int] = None + """Number of beams to use for beam search.""" + penalty_alpha: Optional[float] = None + """The value balances the model confidence and the degeneration penalty in contrastive + search decoding. + """ + temperature: Optional[float] = None + """The value used to modulate the next token probabilities.""" + top_k: Optional[int] = None + """The number of highest probability vocabulary tokens to keep for top-k-filtering.""" + top_p: Optional[float] = None + """If set to float < 1, only the smallest set of most probable tokens with probabilities + that add up to top_p or higher are kept for generation. + """ + typical_p: Optional[float] = None + """Local typicality measures how similar the conditional probability of predicting a target + token next is to the expected conditional probability of predicting a random token next, + given the partial text already generated. If set to float < 1, the smallest set of the + most locally typical tokens with probabilities that add up to typical_p or higher are + kept for generation. See [this paper](https://hf.co/papers/2202.00666) for more details. + """ + use_cache: Optional[bool] = None + """Whether the model should use the past last key/values attentions to speed up decoding""" + + +@dataclass_with_extra +class TextToSpeechParameters(BaseInferenceType): + """Additional inference parameters for Text To Speech""" + + generation_parameters: Optional[TextToSpeechGenerationParameters] = None + """Parametrization of the text generation process""" + + +@dataclass_with_extra +class TextToSpeechInput(BaseInferenceType): + """Inputs for Text To Speech inference""" + + inputs: str + """The input text data""" + parameters: Optional[TextToSpeechParameters] = None + """Additional inference parameters for Text To Speech""" + + +@dataclass_with_extra +class TextToSpeechOutput(BaseInferenceType): + """Outputs of inference for the Text To Speech task""" + + audio: Any + """The generated audio""" + sampling_rate: Optional[float] = None + """The sampling rate of the generated audio waveform.""" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text_to_video.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text_to_video.py new file mode 100644 index 0000000000000000000000000000000000000000..e54a1bc094e4aaf7132e502aa268bc052ab34f0a --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/text_to_video.py @@ -0,0 +1,46 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any, List, Optional + +from .base import BaseInferenceType, dataclass_with_extra + + +@dataclass_with_extra +class TextToVideoParameters(BaseInferenceType): + """Additional inference parameters for Text To Video""" + + guidance_scale: Optional[float] = None + """A higher guidance scale value encourages the model to generate videos closely linked to + the text prompt, but values too high may cause saturation and other artifacts. + """ + negative_prompt: Optional[List[str]] = None + """One or several prompt to guide what NOT to include in video generation.""" + num_frames: Optional[float] = None + """The num_frames parameter determines how many video frames are generated.""" + num_inference_steps: Optional[int] = None + """The number of denoising steps. More denoising steps usually lead to a higher quality + video at the expense of slower inference. + """ + seed: Optional[int] = None + """Seed for the random number generator.""" + + +@dataclass_with_extra +class TextToVideoInput(BaseInferenceType): + """Inputs for Text To Video inference""" + + inputs: str + """The input text data (sometimes called "prompt")""" + parameters: Optional[TextToVideoParameters] = None + """Additional inference parameters for Text To Video""" + + +@dataclass_with_extra +class TextToVideoOutput(BaseInferenceType): + """Outputs of inference for the Text To Video task""" + + video: Any + """The generated video returned as raw bytes in the payload.""" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/token_classification.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/token_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..e039b6a1db7dcd54dbc9434d3254da0770c6799e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/token_classification.py @@ -0,0 +1,51 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import List, Literal, Optional + +from .base import BaseInferenceType, dataclass_with_extra + + +TokenClassificationAggregationStrategy = Literal["none", "simple", "first", "average", "max"] + + +@dataclass_with_extra +class TokenClassificationParameters(BaseInferenceType): + """Additional inference parameters for Token Classification""" + + aggregation_strategy: Optional["TokenClassificationAggregationStrategy"] = None + """The strategy used to fuse tokens based on model predictions""" + ignore_labels: Optional[List[str]] = None + """A list of labels to ignore""" + stride: Optional[int] = None + """The number of overlapping tokens between chunks when splitting the input text.""" + + +@dataclass_with_extra +class TokenClassificationInput(BaseInferenceType): + """Inputs for Token Classification inference""" + + inputs: str + """The input text data""" + parameters: Optional[TokenClassificationParameters] = None + """Additional inference parameters for Token Classification""" + + +@dataclass_with_extra +class TokenClassificationOutputElement(BaseInferenceType): + """Outputs of inference for the Token Classification task""" + + end: int + """The character position in the input where this group ends.""" + score: float + """The associated score / probability""" + start: int + """The character position in the input where this group begins.""" + word: str + """The corresponding text""" + entity: Optional[str] = None + """The predicted label for a single token""" + entity_group: Optional[str] = None + """The predicted label for a group of one or more tokens""" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/translation.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/translation.py new file mode 100644 index 0000000000000000000000000000000000000000..df95b7dbb1f4ce5b80cec034e004bb6e71387be8 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/translation.py @@ -0,0 +1,49 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any, Dict, Literal, Optional + +from .base import BaseInferenceType, dataclass_with_extra + + +TranslationTruncationStrategy = Literal["do_not_truncate", "longest_first", "only_first", "only_second"] + + +@dataclass_with_extra +class TranslationParameters(BaseInferenceType): + """Additional inference parameters for Translation""" + + clean_up_tokenization_spaces: Optional[bool] = None + """Whether to clean up the potential extra spaces in the text output.""" + generate_parameters: Optional[Dict[str, Any]] = None + """Additional parametrization of the text generation algorithm.""" + src_lang: Optional[str] = None + """The source language of the text. Required for models that can translate from multiple + languages. + """ + tgt_lang: Optional[str] = None + """Target language to translate to. Required for models that can translate to multiple + languages. + """ + truncation: Optional["TranslationTruncationStrategy"] = None + """The truncation strategy to use.""" + + +@dataclass_with_extra +class TranslationInput(BaseInferenceType): + """Inputs for Translation inference""" + + inputs: str + """The text to translate.""" + parameters: Optional[TranslationParameters] = None + """Additional inference parameters for Translation""" + + +@dataclass_with_extra +class TranslationOutput(BaseInferenceType): + """Outputs of inference for the Translation task""" + + translation_text: str + """The translated text.""" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/video_classification.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/video_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..e1d7a15bb4ee5fa63aa6ebc3750191bd38549212 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/video_classification.py @@ -0,0 +1,45 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any, Literal, Optional + +from .base import BaseInferenceType, dataclass_with_extra + + +VideoClassificationOutputTransform = Literal["sigmoid", "softmax", "none"] + + +@dataclass_with_extra +class VideoClassificationParameters(BaseInferenceType): + """Additional inference parameters for Video Classification""" + + frame_sampling_rate: Optional[int] = None + """The sampling rate used to select frames from the video.""" + function_to_apply: Optional["VideoClassificationOutputTransform"] = None + """The function to apply to the model outputs in order to retrieve the scores.""" + num_frames: Optional[int] = None + """The number of sampled frames to consider for classification.""" + top_k: Optional[int] = None + """When specified, limits the output to the top K most probable classes.""" + + +@dataclass_with_extra +class VideoClassificationInput(BaseInferenceType): + """Inputs for Video Classification inference""" + + inputs: Any + """The input video data""" + parameters: Optional[VideoClassificationParameters] = None + """Additional inference parameters for Video Classification""" + + +@dataclass_with_extra +class VideoClassificationOutputElement(BaseInferenceType): + """Outputs of inference for the Video Classification task""" + + label: str + """The predicted class label.""" + score: float + """The corresponding probability.""" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/visual_question_answering.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/visual_question_answering.py new file mode 100644 index 0000000000000000000000000000000000000000..d368f1621289bc11a17be3e590cf8a040019d455 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/visual_question_answering.py @@ -0,0 +1,49 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import Any, Optional + +from .base import BaseInferenceType, dataclass_with_extra + + +@dataclass_with_extra +class VisualQuestionAnsweringInputData(BaseInferenceType): + """One (image, question) pair to answer""" + + image: Any + """The image.""" + question: str + """The question to answer based on the image.""" + + +@dataclass_with_extra +class VisualQuestionAnsweringParameters(BaseInferenceType): + """Additional inference parameters for Visual Question Answering""" + + top_k: Optional[int] = None + """The number of answers to return (will be chosen by order of likelihood). Note that we + return less than topk answers if there are not enough options available within the + context. + """ + + +@dataclass_with_extra +class VisualQuestionAnsweringInput(BaseInferenceType): + """Inputs for Visual Question Answering inference""" + + inputs: VisualQuestionAnsweringInputData + """One (image, question) pair to answer""" + parameters: Optional[VisualQuestionAnsweringParameters] = None + """Additional inference parameters for Visual Question Answering""" + + +@dataclass_with_extra +class VisualQuestionAnsweringOutputElement(BaseInferenceType): + """Outputs of inference for the Visual Question Answering task""" + + score: float + """The associated score / probability""" + answer: Optional[str] = None + """The answer to the question""" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/zero_shot_classification.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/zero_shot_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..47b32492e358edcc0de6aa09d53635b0a8156b25 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/zero_shot_classification.py @@ -0,0 +1,45 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import List, Optional + +from .base import BaseInferenceType, dataclass_with_extra + + +@dataclass_with_extra +class ZeroShotClassificationParameters(BaseInferenceType): + """Additional inference parameters for Zero Shot Classification""" + + candidate_labels: List[str] + """The set of possible class labels to classify the text into.""" + hypothesis_template: Optional[str] = None + """The sentence used in conjunction with `candidate_labels` to attempt the text + classification by replacing the placeholder with the candidate labels. + """ + multi_label: Optional[bool] = None + """Whether multiple candidate labels can be true. If false, the scores are normalized such + that the sum of the label likelihoods for each sequence is 1. If true, the labels are + considered independent and probabilities are normalized for each candidate. + """ + + +@dataclass_with_extra +class ZeroShotClassificationInput(BaseInferenceType): + """Inputs for Zero Shot Classification inference""" + + inputs: str + """The text to classify""" + parameters: ZeroShotClassificationParameters + """Additional inference parameters for Zero Shot Classification""" + + +@dataclass_with_extra +class ZeroShotClassificationOutputElement(BaseInferenceType): + """Outputs of inference for the Zero Shot Classification task""" + + label: str + """The predicted class label.""" + score: float + """The corresponding probability.""" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/zero_shot_image_classification.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/zero_shot_image_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..998d66b6b4e3356f0f09a0ad25ebdaf2e76cd03f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/zero_shot_image_classification.py @@ -0,0 +1,40 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import List, Optional + +from .base import BaseInferenceType, dataclass_with_extra + + +@dataclass_with_extra +class ZeroShotImageClassificationParameters(BaseInferenceType): + """Additional inference parameters for Zero Shot Image Classification""" + + candidate_labels: List[str] + """The candidate labels for this image""" + hypothesis_template: Optional[str] = None + """The sentence used in conjunction with `candidate_labels` to attempt the image + classification by replacing the placeholder with the candidate labels. + """ + + +@dataclass_with_extra +class ZeroShotImageClassificationInput(BaseInferenceType): + """Inputs for Zero Shot Image Classification inference""" + + inputs: str + """The input image data to classify as a base64-encoded string.""" + parameters: ZeroShotImageClassificationParameters + """Additional inference parameters for Zero Shot Image Classification""" + + +@dataclass_with_extra +class ZeroShotImageClassificationOutputElement(BaseInferenceType): + """Outputs of inference for the Zero Shot Image Classification task""" + + label: str + """The predicted class label.""" + score: float + """The corresponding probability.""" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/zero_shot_object_detection.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/zero_shot_object_detection.py new file mode 100644 index 0000000000000000000000000000000000000000..8ef76b5fcb93e8126266e4b1464934d01024b1b7 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_generated/types/zero_shot_object_detection.py @@ -0,0 +1,52 @@ +# Inference code generated from the JSON schema spec in @huggingface/tasks. +# +# See: +# - script: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-codegen.ts +# - specs: https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks. +from typing import List + +from .base import BaseInferenceType, dataclass_with_extra + + +@dataclass_with_extra +class ZeroShotObjectDetectionParameters(BaseInferenceType): + """Additional inference parameters for Zero Shot Object Detection""" + + candidate_labels: List[str] + """The candidate labels for this image""" + + +@dataclass_with_extra +class ZeroShotObjectDetectionInput(BaseInferenceType): + """Inputs for Zero Shot Object Detection inference""" + + inputs: str + """The input image data as a base64-encoded string.""" + parameters: ZeroShotObjectDetectionParameters + """Additional inference parameters for Zero Shot Object Detection""" + + +@dataclass_with_extra +class ZeroShotObjectDetectionBoundingBox(BaseInferenceType): + """The predicted bounding box. Coordinates are relative to the top left corner of the input + image. + """ + + xmax: int + xmin: int + ymax: int + ymin: int + + +@dataclass_with_extra +class ZeroShotObjectDetectionOutputElement(BaseInferenceType): + """Outputs of inference for the Zero Shot Object Detection task""" + + box: ZeroShotObjectDetectionBoundingBox + """The predicted bounding box. Coordinates are relative to the top left corner of the input + image. + """ + label: str + """A candidate label""" + score: float + """The associated score / probability""" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3e69c8b7f16ec5fe4b09a6296609ce58e299d71c Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/_cli_hacks.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/_cli_hacks.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..19170d6ab84f7f9975db5d2baff2ab59e79bf40f Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/_cli_hacks.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/agent.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/agent.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1729123405c75abb2bc55e43a6a030892997ad8e Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/agent.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/cli.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/cli.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e708f2bfc2d8ecf59fbd2e155632266a6b4d7320 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/cli.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/constants.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/constants.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..19fdea19b2a6522a5da8ad547e01633176750b96 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/constants.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/mcp_client.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/mcp_client.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..da2f943f23a50d205b4b135220cadbe0374be87d Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/mcp_client.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/types.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/types.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f6700e07e5a41d386adb067f5e216060c6f4ea16 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/types.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/utils.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..19034bddc153a157c6c6d261ff177c1c3d70c880 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_mcp/__pycache__/utils.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..79d2bd75c8329f73bf466cd6b14467579595d180 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__init__.py @@ -0,0 +1,231 @@ +from typing import Dict, Literal, Optional, Union + +from huggingface_hub.inference._providers.featherless_ai import ( + FeatherlessConversationalTask, + FeatherlessTextGenerationTask, +) +from huggingface_hub.utils import logging + +from ._common import TaskProviderHelper, _fetch_inference_provider_mapping +from .black_forest_labs import BlackForestLabsTextToImageTask +from .cerebras import CerebrasConversationalTask +from .clarifai import ClarifaiConversationalTask +from .cohere import CohereConversationalTask +from .fal_ai import ( + FalAIAutomaticSpeechRecognitionTask, + FalAIImageToImageTask, + FalAIImageToVideoTask, + FalAITextToImageTask, + FalAITextToSpeechTask, + FalAITextToVideoTask, +) +from .fireworks_ai import FireworksAIConversationalTask +from .groq import GroqConversationalTask +from .hf_inference import ( + HFInferenceBinaryInputTask, + HFInferenceConversational, + HFInferenceFeatureExtractionTask, + HFInferenceTask, +) +from .hyperbolic import HyperbolicTextGenerationTask, HyperbolicTextToImageTask +from .nebius import ( + NebiusConversationalTask, + NebiusFeatureExtractionTask, + NebiusTextGenerationTask, + NebiusTextToImageTask, +) +from .novita import NovitaConversationalTask, NovitaTextGenerationTask, NovitaTextToVideoTask +from .nscale import NscaleConversationalTask, NscaleTextToImageTask +from .openai import OpenAIConversationalTask +from .publicai import PublicAIConversationalTask +from .replicate import ReplicateImageToImageTask, ReplicateTask, ReplicateTextToImageTask, ReplicateTextToSpeechTask +from .sambanova import SambanovaConversationalTask, SambanovaFeatureExtractionTask +from .scaleway import ScalewayConversationalTask, ScalewayFeatureExtractionTask +from .together import TogetherConversationalTask, TogetherTextGenerationTask, TogetherTextToImageTask +from .zai_org import ZaiConversationalTask + + +logger = logging.get_logger(__name__) + + +PROVIDER_T = Literal[ + "black-forest-labs", + "cerebras", + "clarifai", + "cohere", + "fal-ai", + "featherless-ai", + "fireworks-ai", + "groq", + "hf-inference", + "hyperbolic", + "nebius", + "novita", + "nscale", + "openai", + "publicai", + "replicate", + "sambanova", + "scaleway", + "together", + "zai-org", +] + +PROVIDER_OR_POLICY_T = Union[PROVIDER_T, Literal["auto"]] + +PROVIDERS: Dict[PROVIDER_T, Dict[str, TaskProviderHelper]] = { + "black-forest-labs": { + "text-to-image": BlackForestLabsTextToImageTask(), + }, + "cerebras": { + "conversational": CerebrasConversationalTask(), + }, + "clarifai": { + "conversational": ClarifaiConversationalTask(), + }, + "cohere": { + "conversational": CohereConversationalTask(), + }, + "fal-ai": { + "automatic-speech-recognition": FalAIAutomaticSpeechRecognitionTask(), + "text-to-image": FalAITextToImageTask(), + "text-to-speech": FalAITextToSpeechTask(), + "text-to-video": FalAITextToVideoTask(), + "image-to-video": FalAIImageToVideoTask(), + "image-to-image": FalAIImageToImageTask(), + }, + "featherless-ai": { + "conversational": FeatherlessConversationalTask(), + "text-generation": FeatherlessTextGenerationTask(), + }, + "fireworks-ai": { + "conversational": FireworksAIConversationalTask(), + }, + "groq": { + "conversational": GroqConversationalTask(), + }, + "hf-inference": { + "text-to-image": HFInferenceTask("text-to-image"), + "conversational": HFInferenceConversational(), + "text-generation": HFInferenceTask("text-generation"), + "text-classification": HFInferenceTask("text-classification"), + "question-answering": HFInferenceTask("question-answering"), + "audio-classification": HFInferenceBinaryInputTask("audio-classification"), + "automatic-speech-recognition": HFInferenceBinaryInputTask("automatic-speech-recognition"), + "fill-mask": HFInferenceTask("fill-mask"), + "feature-extraction": HFInferenceFeatureExtractionTask(), + "image-classification": HFInferenceBinaryInputTask("image-classification"), + "image-segmentation": HFInferenceBinaryInputTask("image-segmentation"), + "document-question-answering": HFInferenceTask("document-question-answering"), + "image-to-text": HFInferenceBinaryInputTask("image-to-text"), + "object-detection": HFInferenceBinaryInputTask("object-detection"), + "audio-to-audio": HFInferenceBinaryInputTask("audio-to-audio"), + "zero-shot-image-classification": HFInferenceBinaryInputTask("zero-shot-image-classification"), + "zero-shot-classification": HFInferenceTask("zero-shot-classification"), + "image-to-image": HFInferenceBinaryInputTask("image-to-image"), + "sentence-similarity": HFInferenceTask("sentence-similarity"), + "table-question-answering": HFInferenceTask("table-question-answering"), + "tabular-classification": HFInferenceTask("tabular-classification"), + "text-to-speech": HFInferenceTask("text-to-speech"), + "token-classification": HFInferenceTask("token-classification"), + "translation": HFInferenceTask("translation"), + "summarization": HFInferenceTask("summarization"), + "visual-question-answering": HFInferenceBinaryInputTask("visual-question-answering"), + }, + "hyperbolic": { + "text-to-image": HyperbolicTextToImageTask(), + "conversational": HyperbolicTextGenerationTask("conversational"), + "text-generation": HyperbolicTextGenerationTask("text-generation"), + }, + "nebius": { + "text-to-image": NebiusTextToImageTask(), + "conversational": NebiusConversationalTask(), + "text-generation": NebiusTextGenerationTask(), + "feature-extraction": NebiusFeatureExtractionTask(), + }, + "novita": { + "text-generation": NovitaTextGenerationTask(), + "conversational": NovitaConversationalTask(), + "text-to-video": NovitaTextToVideoTask(), + }, + "nscale": { + "conversational": NscaleConversationalTask(), + "text-to-image": NscaleTextToImageTask(), + }, + "openai": { + "conversational": OpenAIConversationalTask(), + }, + "publicai": { + "conversational": PublicAIConversationalTask(), + }, + "replicate": { + "image-to-image": ReplicateImageToImageTask(), + "text-to-image": ReplicateTextToImageTask(), + "text-to-speech": ReplicateTextToSpeechTask(), + "text-to-video": ReplicateTask("text-to-video"), + }, + "sambanova": { + "conversational": SambanovaConversationalTask(), + "feature-extraction": SambanovaFeatureExtractionTask(), + }, + "scaleway": { + "conversational": ScalewayConversationalTask(), + "feature-extraction": ScalewayFeatureExtractionTask(), + }, + "together": { + "text-to-image": TogetherTextToImageTask(), + "conversational": TogetherConversationalTask(), + "text-generation": TogetherTextGenerationTask(), + }, + "zai-org": { + "conversational": ZaiConversationalTask(), + }, +} + + +def get_provider_helper( + provider: Optional[PROVIDER_OR_POLICY_T], task: str, model: Optional[str] +) -> TaskProviderHelper: + """Get provider helper instance by name and task. + + Args: + provider (`str`, *optional*): name of the provider, or "auto" to automatically select the provider for the model. + task (`str`): Name of the task + model (`str`, *optional*): Name of the model + Returns: + TaskProviderHelper: Helper instance for the specified provider and task + + Raises: + ValueError: If provider or task is not supported + """ + + if (model is None and provider in (None, "auto")) or ( + model is not None and model.startswith(("http://", "https://")) + ): + provider = "hf-inference" + + if provider is None: + logger.info( + "Defaulting to 'auto' which will select the first provider available for the model, sorted by the user's order in https://hf.co/settings/inference-providers." + ) + provider = "auto" + + if provider == "auto": + if model is None: + raise ValueError("Specifying a model is required when provider is 'auto'") + provider_mapping = _fetch_inference_provider_mapping(model) + provider = next(iter(provider_mapping)).provider + + provider_tasks = PROVIDERS.get(provider) # type: ignore + if provider_tasks is None: + raise ValueError( + f"Provider '{provider}' not supported. Available values: 'auto' or any provider from {list(PROVIDERS.keys())}." + "Passing 'auto' (default value) will automatically select the first provider available for the model, sorted " + "by the user's order in https://hf.co/settings/inference-providers." + ) + + if task not in provider_tasks: + raise ValueError( + f"Task '{task}' not supported for provider '{provider}'. Available tasks: {list(provider_tasks.keys())}" + ) + return provider_tasks[task] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bce28adef9c5c7622bf33bb64e86bd89981bcf2a Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/_common.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/_common.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..77f08f95f0af84d3f02dcff695a493fc76b619e2 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/_common.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/black_forest_labs.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/black_forest_labs.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..82802318376e422794c686a2f28a1fa5161af437 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/black_forest_labs.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/cerebras.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/cerebras.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5fb4555ba2c6dadcd18a38f75d806613d2442dd7 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/cerebras.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/cohere.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/cohere.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..52899afd252bef6b46dd9f03bd7e7618c9c67c6a Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/cohere.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/fal_ai.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/fal_ai.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..89aa9ad8d8db28395c5e146e9c028151260d83b9 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/fal_ai.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/featherless_ai.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/featherless_ai.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..46e3ff458b7f60f71e45322511e2ab48faf3b450 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/featherless_ai.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/fireworks_ai.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/fireworks_ai.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7c72e5e99979e7fe7f77199a58dd6586f663c704 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/fireworks_ai.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/groq.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/groq.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..950330155b6beedaa75e6d44394c67b9b7a0e6ce Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/groq.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/hf_inference.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/hf_inference.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..76827390cf8c60fa607c7ee1dc65091d7b6dfbc6 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/hf_inference.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/hyperbolic.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/hyperbolic.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..09f4602ca60edf02ea9030b05e1df2723e785104 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/hyperbolic.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/nebius.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/nebius.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..05a4dcad7fe550c6428e6e19d289b9b662daf092 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/nebius.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/novita.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/novita.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..22ef371bf5e8dd3887f98b1fcbc5ca9f3beb7e3b Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/novita.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/nscale.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/nscale.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6f099b195a7260f1379d085ef70a090ebb82b8f7 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/nscale.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/openai.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/openai.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7ded2aab341e03ddef0b2c6b2b14d9b3b0d30e0e Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/openai.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/publicai.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/publicai.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..558244845307588565b4351f81bf969e0d46e65b Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/publicai.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/replicate.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/replicate.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ce0deb998ac29d8738a41cf454075d18da6614aa Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/replicate.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/sambanova.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/sambanova.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9929e3a1ba1996fc4b504a510c33dbddd271a8dd Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/sambanova.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/scaleway.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/scaleway.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8456788ebed2c1efa2b29d766dd1d810b6087c45 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/scaleway.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/together.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/together.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..386a76eb6706e130d9e838ae45ada469956deb6b Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/together.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/zai_org.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/zai_org.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8a676aec1f7f5e6cd3a1656c216aba5aede38a85 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/__pycache__/zai_org.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/black_forest_labs.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/black_forest_labs.py new file mode 100644 index 0000000000000000000000000000000000000000..a5d96832256e3505d503a7d23bbcee76e485561a --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/black_forest_labs.py @@ -0,0 +1,69 @@ +import time +from typing import Any, Dict, Optional, Union + +from huggingface_hub.hf_api import InferenceProviderMapping +from huggingface_hub.inference._common import RequestParameters, _as_dict +from huggingface_hub.inference._providers._common import TaskProviderHelper, filter_none +from huggingface_hub.utils import logging +from huggingface_hub.utils._http import get_session + + +logger = logging.get_logger(__name__) + +MAX_POLLING_ATTEMPTS = 6 +POLLING_INTERVAL = 1.0 + + +class BlackForestLabsTextToImageTask(TaskProviderHelper): + def __init__(self): + super().__init__(provider="black-forest-labs", base_url="https://api.us1.bfl.ai", task="text-to-image") + + def _prepare_headers(self, headers: Dict, api_key: str) -> Dict[str, Any]: + headers = super()._prepare_headers(headers, api_key) + if not api_key.startswith("hf_"): + _ = headers.pop("authorization") + headers["X-Key"] = api_key + return headers + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + return f"/v1/{mapped_model}" + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping + ) -> Optional[Dict]: + parameters = filter_none(parameters) + if "num_inference_steps" in parameters: + parameters["steps"] = parameters.pop("num_inference_steps") + if "guidance_scale" in parameters: + parameters["guidance"] = parameters.pop("guidance_scale") + + return {"prompt": inputs, **parameters} + + def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any: + """ + Polling mechanism for Black Forest Labs since the API is asynchronous. + """ + url = _as_dict(response).get("polling_url") + session = get_session() + for _ in range(MAX_POLLING_ATTEMPTS): + time.sleep(POLLING_INTERVAL) + + response = session.get(url, headers={"Content-Type": "application/json"}) # type: ignore + response.raise_for_status() # type: ignore + response_json: Dict = response.json() # type: ignore + status = response_json.get("status") + logger.info( + f"Polling generation result from {url}. Current status: {status}. " + f"Will retry after {POLLING_INTERVAL} seconds if not ready." + ) + + if ( + status == "Ready" + and isinstance(response_json.get("result"), dict) + and (sample_url := response_json["result"].get("sample")) + ): + image_resp = session.get(sample_url) + image_resp.raise_for_status() + return image_resp.content + + raise TimeoutError(f"Failed to get the image URL after {MAX_POLLING_ATTEMPTS} attempts.") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/cerebras.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/cerebras.py new file mode 100644 index 0000000000000000000000000000000000000000..a9b9c3aacb3e134a8e755297c15ece198ffe633d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/cerebras.py @@ -0,0 +1,6 @@ +from ._common import BaseConversationalTask + + +class CerebrasConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider="cerebras", base_url="https://api.cerebras.ai") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/clarifai.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/clarifai.py new file mode 100644 index 0000000000000000000000000000000000000000..5f118b7fc9a8dafb01305758791191ccef045a5d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/clarifai.py @@ -0,0 +1,13 @@ +from ._common import BaseConversationalTask + + +_PROVIDER = "clarifai" +_BASE_URL = "https://api.clarifai.com" + + +class ClarifaiConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL) + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + return "/v2/ext/openai/v1/chat/completions" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/cohere.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/cohere.py new file mode 100644 index 0000000000000000000000000000000000000000..a5e9191caec50b0e659dddceba3e817a4ac28307 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/cohere.py @@ -0,0 +1,32 @@ +from typing import Any, Dict, Optional + +from huggingface_hub.hf_api import InferenceProviderMapping + +from ._common import BaseConversationalTask + + +_PROVIDER = "cohere" +_BASE_URL = "https://api.cohere.com" + + +class CohereConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL) + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + return "/compatibility/v1/chat/completions" + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping + ) -> Optional[Dict]: + payload = super()._prepare_payload_as_dict(inputs, parameters, provider_mapping_info) + response_format = parameters.get("response_format") + if isinstance(response_format, dict) and response_format.get("type") == "json_schema": + json_schema_details = response_format.get("json_schema") + if isinstance(json_schema_details, dict) and "schema" in json_schema_details: + payload["response_format"] = { # type: ignore [index] + "type": "json_object", + "schema": json_schema_details["schema"], + } + + return payload diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/fal_ai.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/fal_ai.py new file mode 100644 index 0000000000000000000000000000000000000000..bc2c41d04f811f6a6508ea6abf84593add31ef42 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/fal_ai.py @@ -0,0 +1,248 @@ +import base64 +import time +from abc import ABC +from typing import Any, Dict, Optional, Union +from urllib.parse import urlparse + +from huggingface_hub import constants +from huggingface_hub.hf_api import InferenceProviderMapping +from huggingface_hub.inference._common import RequestParameters, _as_dict, _as_url +from huggingface_hub.inference._providers._common import TaskProviderHelper, filter_none +from huggingface_hub.utils import get_session, hf_raise_for_status +from huggingface_hub.utils.logging import get_logger + + +logger = get_logger(__name__) + +# Arbitrary polling interval +_POLLING_INTERVAL = 0.5 + + +class FalAITask(TaskProviderHelper, ABC): + def __init__(self, task: str): + super().__init__(provider="fal-ai", base_url="https://fal.run", task=task) + + def _prepare_headers(self, headers: Dict, api_key: str) -> Dict[str, Any]: + headers = super()._prepare_headers(headers, api_key) + if not api_key.startswith("hf_"): + headers["authorization"] = f"Key {api_key}" + return headers + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + return f"/{mapped_model}" + + +class FalAIQueueTask(TaskProviderHelper, ABC): + def __init__(self, task: str): + super().__init__(provider="fal-ai", base_url="https://queue.fal.run", task=task) + + def _prepare_headers(self, headers: Dict, api_key: str) -> Dict[str, Any]: + headers = super()._prepare_headers(headers, api_key) + if not api_key.startswith("hf_"): + headers["authorization"] = f"Key {api_key}" + return headers + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + if api_key.startswith("hf_"): + # Use the queue subdomain for HF routing + return f"/{mapped_model}?_subdomain=queue" + return f"/{mapped_model}" + + def get_response( + self, + response: Union[bytes, Dict], + request_params: Optional[RequestParameters] = None, + ) -> Any: + response_dict = _as_dict(response) + + request_id = response_dict.get("request_id") + if not request_id: + raise ValueError("No request ID found in the response") + if request_params is None: + raise ValueError( + f"A `RequestParameters` object should be provided to get {self.task} responses with Fal AI." + ) + + # extract the base url and query params + parsed_url = urlparse(request_params.url) + # a bit hacky way to concatenate the provider name without parsing `parsed_url.path` + base_url = f"{parsed_url.scheme}://{parsed_url.netloc}{'/fal-ai' if parsed_url.netloc == 'router.huggingface.co' else ''}" + query_param = f"?{parsed_url.query}" if parsed_url.query else "" + + # extracting the provider model id for status and result urls + # from the response as it might be different from the mapped model in `request_params.url` + model_id = urlparse(response_dict.get("response_url")).path + status_url = f"{base_url}{str(model_id)}/status{query_param}" + result_url = f"{base_url}{str(model_id)}{query_param}" + + status = response_dict.get("status") + logger.info("Generating the output.. this can take several minutes.") + while status != "COMPLETED": + time.sleep(_POLLING_INTERVAL) + status_response = get_session().get(status_url, headers=request_params.headers) + hf_raise_for_status(status_response) + status = status_response.json().get("status") + + return get_session().get(result_url, headers=request_params.headers).json() + + +class FalAIAutomaticSpeechRecognitionTask(FalAITask): + def __init__(self): + super().__init__("automatic-speech-recognition") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping + ) -> Optional[Dict]: + if isinstance(inputs, str) and inputs.startswith(("http://", "https://")): + # If input is a URL, pass it directly + audio_url = inputs + else: + # If input is a file path, read it first + if isinstance(inputs, str): + with open(inputs, "rb") as f: + inputs = f.read() + + audio_b64 = base64.b64encode(inputs).decode() + content_type = "audio/mpeg" + audio_url = f"data:{content_type};base64,{audio_b64}" + + return {"audio_url": audio_url, **filter_none(parameters)} + + def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any: + text = _as_dict(response)["text"] + if not isinstance(text, str): + raise ValueError(f"Unexpected output format from FalAI API. Expected string, got {type(text)}.") + return text + + +class FalAITextToImageTask(FalAITask): + def __init__(self): + super().__init__("text-to-image") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping + ) -> Optional[Dict]: + payload: Dict[str, Any] = { + "prompt": inputs, + **filter_none(parameters), + } + if "width" in payload and "height" in payload: + payload["image_size"] = { + "width": payload.pop("width"), + "height": payload.pop("height"), + } + if provider_mapping_info.adapter_weights_path is not None: + lora_path = constants.HUGGINGFACE_CO_URL_TEMPLATE.format( + repo_id=provider_mapping_info.hf_model_id, + revision="main", + filename=provider_mapping_info.adapter_weights_path, + ) + payload["loras"] = [{"path": lora_path, "scale": 1}] + if provider_mapping_info.provider_id == "fal-ai/lora": + # little hack: fal requires the base model for stable-diffusion-based loras but not for flux-based + # See payloads in https://fal.ai/models/fal-ai/lora/api vs https://fal.ai/models/fal-ai/flux-lora/api + payload["model_name"] = "stabilityai/stable-diffusion-xl-base-1.0" + + return payload + + def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any: + url = _as_dict(response)["images"][0]["url"] + return get_session().get(url).content + + +class FalAITextToSpeechTask(FalAITask): + def __init__(self): + super().__init__("text-to-speech") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping + ) -> Optional[Dict]: + return {"text": inputs, **filter_none(parameters)} + + def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any: + url = _as_dict(response)["audio"]["url"] + return get_session().get(url).content + + +class FalAITextToVideoTask(FalAIQueueTask): + def __init__(self): + super().__init__("text-to-video") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping + ) -> Optional[Dict]: + return {"prompt": inputs, **filter_none(parameters)} + + def get_response( + self, + response: Union[bytes, Dict], + request_params: Optional[RequestParameters] = None, + ) -> Any: + output = super().get_response(response, request_params) + url = _as_dict(output)["video"]["url"] + return get_session().get(url).content + + +class FalAIImageToImageTask(FalAIQueueTask): + def __init__(self): + super().__init__("image-to-image") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping + ) -> Optional[Dict]: + image_url = _as_url(inputs, default_mime_type="image/jpeg") + if "target_size" in parameters: + parameters["image_size"] = parameters.pop("target_size") + payload: Dict[str, Any] = { + "image_url": image_url, + **filter_none(parameters), + } + if provider_mapping_info.adapter_weights_path is not None: + lora_path = constants.HUGGINGFACE_CO_URL_TEMPLATE.format( + repo_id=provider_mapping_info.hf_model_id, + revision="main", + filename=provider_mapping_info.adapter_weights_path, + ) + payload["loras"] = [{"path": lora_path, "scale": 1}] + + return payload + + def get_response( + self, + response: Union[bytes, Dict], + request_params: Optional[RequestParameters] = None, + ) -> Any: + output = super().get_response(response, request_params) + url = _as_dict(output)["images"][0]["url"] + return get_session().get(url).content + + +class FalAIImageToVideoTask(FalAIQueueTask): + def __init__(self): + super().__init__("image-to-video") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping + ) -> Optional[Dict]: + image_url = _as_url(inputs, default_mime_type="image/jpeg") + payload: Dict[str, Any] = { + "image_url": image_url, + **filter_none(parameters), + } + if provider_mapping_info.adapter_weights_path is not None: + lora_path = constants.HUGGINGFACE_CO_URL_TEMPLATE.format( + repo_id=provider_mapping_info.hf_model_id, + revision="main", + filename=provider_mapping_info.adapter_weights_path, + ) + payload["loras"] = [{"path": lora_path, "scale": 1}] + return payload + + def get_response( + self, + response: Union[bytes, Dict], + request_params: Optional[RequestParameters] = None, + ) -> Any: + output = super().get_response(response, request_params) + url = _as_dict(output)["video"]["url"] + return get_session().get(url).content diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/featherless_ai.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/featherless_ai.py new file mode 100644 index 0000000000000000000000000000000000000000..6ad1c48134f5c990b6ac4fca5ff919f4cc0d2373 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/featherless_ai.py @@ -0,0 +1,38 @@ +from typing import Any, Dict, Optional, Union + +from huggingface_hub.hf_api import InferenceProviderMapping +from huggingface_hub.inference._common import RequestParameters, _as_dict + +from ._common import BaseConversationalTask, BaseTextGenerationTask, filter_none + + +_PROVIDER = "featherless-ai" +_BASE_URL = "https://api.featherless.ai" + + +class FeatherlessTextGenerationTask(BaseTextGenerationTask): + def __init__(self): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL) + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping + ) -> Optional[Dict]: + params = filter_none(parameters.copy()) + params["max_tokens"] = params.pop("max_new_tokens", None) + + return {"prompt": inputs, **params, "model": provider_mapping_info.provider_id} + + def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any: + output = _as_dict(response)["choices"][0] + return { + "generated_text": output["text"], + "details": { + "finish_reason": output.get("finish_reason"), + "seed": output.get("seed"), + }, + } + + +class FeatherlessConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/fireworks_ai.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/fireworks_ai.py new file mode 100644 index 0000000000000000000000000000000000000000..b4cc19a5700047f6516b2784d9785a99d7e32451 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/fireworks_ai.py @@ -0,0 +1,27 @@ +from typing import Any, Dict, Optional + +from huggingface_hub.hf_api import InferenceProviderMapping + +from ._common import BaseConversationalTask + + +class FireworksAIConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider="fireworks-ai", base_url="https://api.fireworks.ai") + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + return "/inference/v1/chat/completions" + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping + ) -> Optional[Dict]: + payload = super()._prepare_payload_as_dict(inputs, parameters, provider_mapping_info) + response_format = parameters.get("response_format") + if isinstance(response_format, dict) and response_format.get("type") == "json_schema": + json_schema_details = response_format.get("json_schema") + if isinstance(json_schema_details, dict) and "schema" in json_schema_details: + payload["response_format"] = { # type: ignore [index] + "type": "json_object", + "schema": json_schema_details["schema"], + } + return payload diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/groq.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/groq.py new file mode 100644 index 0000000000000000000000000000000000000000..11e677504e89bc02b966e7d37d9e11f1b94b297f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/groq.py @@ -0,0 +1,9 @@ +from ._common import BaseConversationalTask + + +class GroqConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider="groq", base_url="https://api.groq.com") + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + return "/openai/v1/chat/completions" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/hf_inference.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/hf_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..d90d00c4f3e5b93029ed979df6e310635a639d93 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/hf_inference.py @@ -0,0 +1,228 @@ +import json +from functools import lru_cache +from pathlib import Path +from typing import Any, Dict, Optional, Union +from urllib.parse import urlparse, urlunparse + +from huggingface_hub import constants +from huggingface_hub.hf_api import InferenceProviderMapping +from huggingface_hub.inference._common import ( + MimeBytes, + RequestParameters, + _b64_encode, + _bytes_to_dict, + _open_as_mime_bytes, +) +from huggingface_hub.inference._providers._common import TaskProviderHelper, filter_none +from huggingface_hub.utils import build_hf_headers, get_session, get_token, hf_raise_for_status + + +class HFInferenceTask(TaskProviderHelper): + """Base class for HF Inference API tasks.""" + + def __init__(self, task: str): + super().__init__( + provider="hf-inference", + base_url=constants.INFERENCE_PROXY_TEMPLATE.format(provider="hf-inference"), + task=task, + ) + + def _prepare_api_key(self, api_key: Optional[str]) -> str: + # special case: for HF Inference we allow not providing an API key + return api_key or get_token() # type: ignore[return-value] + + def _prepare_mapping_info(self, model: Optional[str]) -> InferenceProviderMapping: + if model is not None and model.startswith(("http://", "https://")): + return InferenceProviderMapping( + provider="hf-inference", providerId=model, hf_model_id=model, task=self.task, status="live" + ) + model_id = model if model is not None else _fetch_recommended_models().get(self.task) + if model_id is None: + raise ValueError( + f"Task {self.task} has no recommended model for HF Inference. Please specify a model" + " explicitly. Visit https://huggingface.co/tasks for more info." + ) + _check_supported_task(model_id, self.task) + return InferenceProviderMapping( + provider="hf-inference", providerId=model_id, hf_model_id=model_id, task=self.task, status="live" + ) + + def _prepare_url(self, api_key: str, mapped_model: str) -> str: + # hf-inference provider can handle URLs (e.g. Inference Endpoints or TGI deployment) + if mapped_model.startswith(("http://", "https://")): + return mapped_model + return ( + # Feature-extraction and sentence-similarity are the only cases where we handle models with several tasks. + f"{self.base_url}/models/{mapped_model}/pipeline/{self.task}" + if self.task in ("feature-extraction", "sentence-similarity") + # Otherwise, we use the default endpoint + else f"{self.base_url}/models/{mapped_model}" + ) + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping + ) -> Optional[Dict]: + if isinstance(inputs, bytes): + raise ValueError(f"Unexpected binary input for task {self.task}.") + if isinstance(inputs, Path): + raise ValueError(f"Unexpected path input for task {self.task} (got {inputs})") + return filter_none({"inputs": inputs, "parameters": parameters}) + + +class HFInferenceBinaryInputTask(HFInferenceTask): + def _prepare_payload_as_dict( + self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping + ) -> Optional[Dict]: + return None + + def _prepare_payload_as_bytes( + self, + inputs: Any, + parameters: Dict, + provider_mapping_info: InferenceProviderMapping, + extra_payload: Optional[Dict], + ) -> Optional[MimeBytes]: + parameters = filter_none(parameters) + extra_payload = extra_payload or {} + has_parameters = len(parameters) > 0 or len(extra_payload) > 0 + + # Raise if not a binary object or a local path or a URL. + if not isinstance(inputs, (bytes, Path)) and not isinstance(inputs, str): + raise ValueError(f"Expected binary inputs or a local path or a URL. Got {inputs}") + + # Send inputs as raw content when no parameters are provided + if not has_parameters: + return _open_as_mime_bytes(inputs) + + # Otherwise encode as b64 + return MimeBytes( + json.dumps({"inputs": _b64_encode(inputs), "parameters": parameters, **extra_payload}).encode("utf-8"), + mime_type="application/json", + ) + + +class HFInferenceConversational(HFInferenceTask): + def __init__(self): + super().__init__("conversational") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping + ) -> Optional[Dict]: + payload = filter_none(parameters) + mapped_model = provider_mapping_info.provider_id + payload_model = parameters.get("model") or mapped_model + + if payload_model is None or payload_model.startswith(("http://", "https://")): + payload_model = "dummy" + + response_format = parameters.get("response_format") + if isinstance(response_format, dict) and response_format.get("type") == "json_schema": + payload["response_format"] = { + "type": "json_object", + "value": response_format["json_schema"]["schema"], + } + return {**payload, "model": payload_model, "messages": inputs} + + def _prepare_url(self, api_key: str, mapped_model: str) -> str: + base_url = ( + mapped_model + if mapped_model.startswith(("http://", "https://")) + else f"{constants.INFERENCE_PROXY_TEMPLATE.format(provider='hf-inference')}/models/{mapped_model}" + ) + return _build_chat_completion_url(base_url) + + +def _build_chat_completion_url(model_url: str) -> str: + parsed = urlparse(model_url) + path = parsed.path.rstrip("/") + + # If the path already ends with /chat/completions, we're done! + if path.endswith("/chat/completions"): + return model_url + + # Append /chat/completions if not already present + if path.endswith("/v1"): + new_path = path + "/chat/completions" + # If path was empty or just "/", set the full path + elif not path: + new_path = "/v1/chat/completions" + # Append /v1/chat/completions if not already present + else: + new_path = path + "/v1/chat/completions" + + # Reconstruct the URL with the new path and original query parameters. + new_parsed = parsed._replace(path=new_path) + return str(urlunparse(new_parsed)) + + +@lru_cache(maxsize=1) +def _fetch_recommended_models() -> Dict[str, Optional[str]]: + response = get_session().get(f"{constants.ENDPOINT}/api/tasks", headers=build_hf_headers()) + hf_raise_for_status(response) + return {task: next(iter(details["widgetModels"]), None) for task, details in response.json().items()} + + +@lru_cache(maxsize=None) +def _check_supported_task(model: str, task: str) -> None: + from huggingface_hub.hf_api import HfApi + + model_info = HfApi().model_info(model) + pipeline_tag = model_info.pipeline_tag + tags = model_info.tags or [] + is_conversational = "conversational" in tags + if task in ("text-generation", "conversational"): + if pipeline_tag == "text-generation": + # text-generation + conversational tag -> both tasks allowed + if is_conversational: + return + # text-generation without conversational tag -> only text-generation allowed + if task == "text-generation": + return + raise ValueError(f"Model '{model}' doesn't support task '{task}'.") + + if pipeline_tag == "text2text-generation": + if task == "text-generation": + return + raise ValueError(f"Model '{model}' doesn't support task '{task}'.") + + if pipeline_tag == "image-text-to-text": + if is_conversational and task == "conversational": + return # Only conversational allowed if tagged as conversational + raise ValueError("Non-conversational image-text-to-text task is not supported.") + + if ( + task in ("feature-extraction", "sentence-similarity") + and pipeline_tag in ("feature-extraction", "sentence-similarity") + and task in tags + ): + # feature-extraction and sentence-similarity are interchangeable for HF Inference + return + + # For all other tasks, just check pipeline tag + if pipeline_tag != task: + raise ValueError( + f"Model '{model}' doesn't support task '{task}'. Supported tasks: '{pipeline_tag}', got: '{task}'" + ) + return + + +class HFInferenceFeatureExtractionTask(HFInferenceTask): + def __init__(self): + super().__init__("feature-extraction") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping + ) -> Optional[Dict]: + if isinstance(inputs, bytes): + raise ValueError(f"Unexpected binary input for task {self.task}.") + if isinstance(inputs, Path): + raise ValueError(f"Unexpected path input for task {self.task} (got {inputs})") + + # Parameters are sent at root-level for feature-extraction task + # See specs: https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/src/tasks/feature-extraction/spec/input.json + return {"inputs": inputs, **filter_none(parameters)} + + def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any: + if isinstance(response, bytes): + return _bytes_to_dict(response) + return response diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/hyperbolic.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/hyperbolic.py new file mode 100644 index 0000000000000000000000000000000000000000..6dcb14cc275f6b80db5643361b9dfd3cbf8d91a2 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/hyperbolic.py @@ -0,0 +1,47 @@ +import base64 +from typing import Any, Dict, Optional, Union + +from huggingface_hub.hf_api import InferenceProviderMapping +from huggingface_hub.inference._common import RequestParameters, _as_dict +from huggingface_hub.inference._providers._common import BaseConversationalTask, TaskProviderHelper, filter_none + + +class HyperbolicTextToImageTask(TaskProviderHelper): + def __init__(self): + super().__init__(provider="hyperbolic", base_url="https://api.hyperbolic.xyz", task="text-to-image") + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + return "/v1/images/generations" + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping + ) -> Optional[Dict]: + mapped_model = provider_mapping_info.provider_id + parameters = filter_none(parameters) + if "num_inference_steps" in parameters: + parameters["steps"] = parameters.pop("num_inference_steps") + if "guidance_scale" in parameters: + parameters["cfg_scale"] = parameters.pop("guidance_scale") + # For Hyperbolic, the width and height are required parameters + if "width" not in parameters: + parameters["width"] = 512 + if "height" not in parameters: + parameters["height"] = 512 + return {"prompt": inputs, "model_name": mapped_model, **parameters} + + def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any: + response_dict = _as_dict(response) + return base64.b64decode(response_dict["images"][0]["image"]) + + +class HyperbolicTextGenerationTask(BaseConversationalTask): + """ + Special case for Hyperbolic, where text-generation task is handled as a conversational task. + """ + + def __init__(self, task: str): + super().__init__( + provider="hyperbolic", + base_url="https://api.hyperbolic.xyz", + ) + self.task = task diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/nebius.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/nebius.py new file mode 100644 index 0000000000000000000000000000000000000000..85ad67c4c8835d7fb8bfe5f36e426614174a66ba --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/nebius.py @@ -0,0 +1,83 @@ +import base64 +from typing import Any, Dict, Optional, Union + +from huggingface_hub.hf_api import InferenceProviderMapping +from huggingface_hub.inference._common import RequestParameters, _as_dict +from huggingface_hub.inference._providers._common import ( + BaseConversationalTask, + BaseTextGenerationTask, + TaskProviderHelper, + filter_none, +) + + +class NebiusTextGenerationTask(BaseTextGenerationTask): + def __init__(self): + super().__init__(provider="nebius", base_url="https://api.studio.nebius.ai") + + def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any: + output = _as_dict(response)["choices"][0] + return { + "generated_text": output["text"], + "details": { + "finish_reason": output.get("finish_reason"), + "seed": output.get("seed"), + }, + } + + +class NebiusConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider="nebius", base_url="https://api.studio.nebius.ai") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping + ) -> Optional[Dict]: + payload = super()._prepare_payload_as_dict(inputs, parameters, provider_mapping_info) + response_format = parameters.get("response_format") + if isinstance(response_format, dict) and response_format.get("type") == "json_schema": + json_schema_details = response_format.get("json_schema") + if isinstance(json_schema_details, dict) and "schema" in json_schema_details: + payload["guided_json"] = json_schema_details["schema"] # type: ignore [index] + return payload + + +class NebiusTextToImageTask(TaskProviderHelper): + def __init__(self): + super().__init__(task="text-to-image", provider="nebius", base_url="https://api.studio.nebius.ai") + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + return "/v1/images/generations" + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping + ) -> Optional[Dict]: + mapped_model = provider_mapping_info.provider_id + parameters = filter_none(parameters) + if "guidance_scale" in parameters: + parameters.pop("guidance_scale") + if parameters.get("response_format") not in ("b64_json", "url"): + parameters["response_format"] = "b64_json" + + return {"prompt": inputs, **parameters, "model": mapped_model} + + def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any: + response_dict = _as_dict(response) + return base64.b64decode(response_dict["data"][0]["b64_json"]) + + +class NebiusFeatureExtractionTask(TaskProviderHelper): + def __init__(self): + super().__init__(task="feature-extraction", provider="nebius", base_url="https://api.studio.nebius.ai") + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + return "/v1/embeddings" + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping + ) -> Optional[Dict]: + return {"input": inputs, "model": provider_mapping_info.provider_id} + + def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any: + embeddings = _as_dict(response)["data"] + return [embedding["embedding"] for embedding in embeddings] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/novita.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/novita.py new file mode 100644 index 0000000000000000000000000000000000000000..44adc9017b456f487513cde251086075d84b69f0 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/novita.py @@ -0,0 +1,69 @@ +from typing import Any, Dict, Optional, Union + +from huggingface_hub.hf_api import InferenceProviderMapping +from huggingface_hub.inference._common import RequestParameters, _as_dict +from huggingface_hub.inference._providers._common import ( + BaseConversationalTask, + BaseTextGenerationTask, + TaskProviderHelper, + filter_none, +) +from huggingface_hub.utils import get_session + + +_PROVIDER = "novita" +_BASE_URL = "https://api.novita.ai" + + +class NovitaTextGenerationTask(BaseTextGenerationTask): + def __init__(self): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL) + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + # there is no v1/ route for novita + return "/v3/openai/completions" + + def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any: + output = _as_dict(response)["choices"][0] + return { + "generated_text": output["text"], + "details": { + "finish_reason": output.get("finish_reason"), + "seed": output.get("seed"), + }, + } + + +class NovitaConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL) + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + # there is no v1/ route for novita + return "/v3/openai/chat/completions" + + +class NovitaTextToVideoTask(TaskProviderHelper): + def __init__(self): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL, task="text-to-video") + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + return f"/v3/hf/{mapped_model}" + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping + ) -> Optional[Dict]: + return {"prompt": inputs, **filter_none(parameters)} + + def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any: + response_dict = _as_dict(response) + if not ( + isinstance(response_dict, dict) + and "video" in response_dict + and isinstance(response_dict["video"], dict) + and "video_url" in response_dict["video"] + ): + raise ValueError("Expected response format: { 'video': { 'video_url': string } }") + + video_url = response_dict["video"]["video_url"] + return get_session().get(video_url).content diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/nscale.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/nscale.py new file mode 100644 index 0000000000000000000000000000000000000000..ce5b20e354e246e93a7dd9831e4acf69ebcfad63 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/nscale.py @@ -0,0 +1,44 @@ +import base64 +from typing import Any, Dict, Optional, Union + +from huggingface_hub.hf_api import InferenceProviderMapping +from huggingface_hub.inference._common import RequestParameters, _as_dict + +from ._common import BaseConversationalTask, TaskProviderHelper, filter_none + + +class NscaleConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider="nscale", base_url="https://inference.api.nscale.com") + + +class NscaleTextToImageTask(TaskProviderHelper): + def __init__(self): + super().__init__(provider="nscale", base_url="https://inference.api.nscale.com", task="text-to-image") + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + return "/v1/images/generations" + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping + ) -> Optional[Dict]: + mapped_model = provider_mapping_info.provider_id + # Combine all parameters except inputs and parameters + parameters = filter_none(parameters) + if "width" in parameters and "height" in parameters: + parameters["size"] = f"{parameters.pop('width')}x{parameters.pop('height')}" + if "num_inference_steps" in parameters: + parameters.pop("num_inference_steps") + if "cfg_scale" in parameters: + parameters.pop("cfg_scale") + payload = { + "response_format": "b64_json", + "prompt": inputs, + "model": mapped_model, + **parameters, + } + return payload + + def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any: + response_dict = _as_dict(response) + return base64.b64decode(response_dict["data"][0]["b64_json"]) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/openai.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/openai.py new file mode 100644 index 0000000000000000000000000000000000000000..7a554093c173ea8f664cb7fbd9616ce3a08ce78c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/openai.py @@ -0,0 +1,25 @@ +from typing import Optional + +from huggingface_hub.hf_api import InferenceProviderMapping +from huggingface_hub.inference._providers._common import BaseConversationalTask + + +class OpenAIConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider="openai", base_url="https://api.openai.com") + + def _prepare_api_key(self, api_key: Optional[str]) -> str: + if api_key is None: + raise ValueError("You must provide an api_key to work with OpenAI API.") + if api_key.startswith("hf_"): + raise ValueError( + "OpenAI provider is not available through Hugging Face routing, please use your own OpenAI API key." + ) + return api_key + + def _prepare_mapping_info(self, model: Optional[str]) -> InferenceProviderMapping: + if model is None: + raise ValueError("Please provide an OpenAI model ID, e.g. `gpt-4o` or `o1`.") + return InferenceProviderMapping( + provider="openai", providerId=model, task="conversational", status="live", hf_model_id=model + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/publicai.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/publicai.py new file mode 100644 index 0000000000000000000000000000000000000000..4c88528e4f1e2eefaf6be9315c490db19ff5ca1e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/publicai.py @@ -0,0 +1,6 @@ +from ._common import BaseConversationalTask + + +class PublicAIConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider="publicai", base_url="https://api.publicai.co") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/replicate.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/replicate.py new file mode 100644 index 0000000000000000000000000000000000000000..139582cc801eaf0bdd93e006df404432f2375fb3 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/replicate.py @@ -0,0 +1,90 @@ +from typing import Any, Dict, Optional, Union + +from huggingface_hub.hf_api import InferenceProviderMapping +from huggingface_hub.inference._common import RequestParameters, _as_dict, _as_url +from huggingface_hub.inference._providers._common import TaskProviderHelper, filter_none +from huggingface_hub.utils import get_session + + +_PROVIDER = "replicate" +_BASE_URL = "https://api.replicate.com" + + +class ReplicateTask(TaskProviderHelper): + def __init__(self, task: str): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL, task=task) + + def _prepare_headers(self, headers: Dict, api_key: str) -> Dict[str, Any]: + headers = super()._prepare_headers(headers, api_key) + headers["Prefer"] = "wait" + return headers + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + if ":" in mapped_model: + return "/v1/predictions" + return f"/v1/models/{mapped_model}/predictions" + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping + ) -> Optional[Dict]: + mapped_model = provider_mapping_info.provider_id + payload: Dict[str, Any] = {"input": {"prompt": inputs, **filter_none(parameters)}} + if ":" in mapped_model: + version = mapped_model.split(":", 1)[1] + payload["version"] = version + return payload + + def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any: + response_dict = _as_dict(response) + if response_dict.get("output") is None: + raise TimeoutError( + f"Inference request timed out after 60 seconds. No output generated for model {response_dict.get('model')}" + "The model might be in cold state or starting up. Please try again later." + ) + output_url = ( + response_dict["output"] if isinstance(response_dict["output"], str) else response_dict["output"][0] + ) + return get_session().get(output_url).content + + +class ReplicateTextToImageTask(ReplicateTask): + def __init__(self): + super().__init__("text-to-image") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping + ) -> Optional[Dict]: + payload: Dict = super()._prepare_payload_as_dict(inputs, parameters, provider_mapping_info) # type: ignore[assignment] + if provider_mapping_info.adapter_weights_path is not None: + payload["input"]["lora_weights"] = f"https://huggingface.co/{provider_mapping_info.hf_model_id}" + return payload + + +class ReplicateTextToSpeechTask(ReplicateTask): + def __init__(self): + super().__init__("text-to-speech") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping + ) -> Optional[Dict]: + payload: Dict = super()._prepare_payload_as_dict(inputs, parameters, provider_mapping_info) # type: ignore[assignment] + payload["input"]["text"] = payload["input"].pop("prompt") # rename "prompt" to "text" for TTS + return payload + + +class ReplicateImageToImageTask(ReplicateTask): + def __init__(self): + super().__init__("image-to-image") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping + ) -> Optional[Dict]: + image_url = _as_url(inputs, default_mime_type="image/jpeg") + + payload: Dict[str, Any] = {"input": {"input_image": image_url, **filter_none(parameters)}} + + mapped_model = provider_mapping_info.provider_id + if ":" in mapped_model: + version = mapped_model.split(":", 1)[1] + payload["version"] = version + return payload diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/sambanova.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/sambanova.py new file mode 100644 index 0000000000000000000000000000000000000000..ed96fb766ce49003b605bda8ef8ee34da0ebe2f4 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/sambanova.py @@ -0,0 +1,42 @@ +from typing import Any, Dict, Optional, Union + +from huggingface_hub.hf_api import InferenceProviderMapping +from huggingface_hub.inference._common import RequestParameters, _as_dict +from huggingface_hub.inference._providers._common import BaseConversationalTask, TaskProviderHelper, filter_none + + +class SambanovaConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider="sambanova", base_url="https://api.sambanova.ai") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping + ) -> Optional[Dict]: + response_format_config = parameters.get("response_format") + if isinstance(response_format_config, dict): + if response_format_config.get("type") == "json_schema": + json_schema_config = response_format_config.get("json_schema", {}) + strict = json_schema_config.get("strict") + if isinstance(json_schema_config, dict) and (strict is True or strict is None): + json_schema_config["strict"] = False + + payload = super()._prepare_payload_as_dict(inputs, parameters, provider_mapping_info) + return payload + + +class SambanovaFeatureExtractionTask(TaskProviderHelper): + def __init__(self): + super().__init__(provider="sambanova", base_url="https://api.sambanova.ai", task="feature-extraction") + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + return "/v1/embeddings" + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping + ) -> Optional[Dict]: + parameters = filter_none(parameters) + return {"input": inputs, "model": provider_mapping_info.provider_id, **parameters} + + def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any: + embeddings = _as_dict(response)["data"] + return [embedding["embedding"] for embedding in embeddings] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/scaleway.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/scaleway.py new file mode 100644 index 0000000000000000000000000000000000000000..cfdd75416f1a11f3f4908d1c29541920cba76d79 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/scaleway.py @@ -0,0 +1,28 @@ +from typing import Any, Dict, Optional, Union + +from huggingface_hub.inference._common import RequestParameters, _as_dict + +from ._common import BaseConversationalTask, InferenceProviderMapping, TaskProviderHelper, filter_none + + +class ScalewayConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider="scaleway", base_url="https://api.scaleway.ai") + + +class ScalewayFeatureExtractionTask(TaskProviderHelper): + def __init__(self): + super().__init__(provider="scaleway", base_url="https://api.scaleway.ai", task="feature-extraction") + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + return "/v1/embeddings" + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping + ) -> Optional[Dict]: + parameters = filter_none(parameters) + return {"input": inputs, "model": provider_mapping_info.provider_id, **parameters} + + def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any: + embeddings = _as_dict(response)["data"] + return [embedding["embedding"] for embedding in embeddings] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/together.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/together.py new file mode 100644 index 0000000000000000000000000000000000000000..de166b7baf8d50b255f29cf8cc9b9d3fa639646e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/together.py @@ -0,0 +1,88 @@ +import base64 +from abc import ABC +from typing import Any, Dict, Optional, Union + +from huggingface_hub.hf_api import InferenceProviderMapping +from huggingface_hub.inference._common import RequestParameters, _as_dict +from huggingface_hub.inference._providers._common import ( + BaseConversationalTask, + BaseTextGenerationTask, + TaskProviderHelper, + filter_none, +) + + +_PROVIDER = "together" +_BASE_URL = "https://api.together.xyz" + + +class TogetherTask(TaskProviderHelper, ABC): + """Base class for Together API tasks.""" + + def __init__(self, task: str): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL, task=task) + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + if self.task == "text-to-image": + return "/v1/images/generations" + elif self.task == "conversational": + return "/v1/chat/completions" + elif self.task == "text-generation": + return "/v1/completions" + raise ValueError(f"Unsupported task '{self.task}' for Together API.") + + +class TogetherTextGenerationTask(BaseTextGenerationTask): + def __init__(self): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL) + + def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any: + output = _as_dict(response)["choices"][0] + return { + "generated_text": output["text"], + "details": { + "finish_reason": output.get("finish_reason"), + "seed": output.get("seed"), + }, + } + + +class TogetherConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider=_PROVIDER, base_url=_BASE_URL) + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping + ) -> Optional[Dict]: + payload = super()._prepare_payload_as_dict(inputs, parameters, provider_mapping_info) + response_format = parameters.get("response_format") + if isinstance(response_format, dict) and response_format.get("type") == "json_schema": + json_schema_details = response_format.get("json_schema") + if isinstance(json_schema_details, dict) and "schema" in json_schema_details: + payload["response_format"] = { # type: ignore [index] + "type": "json_object", + "schema": json_schema_details["schema"], + } + + return payload + + +class TogetherTextToImageTask(TogetherTask): + def __init__(self): + super().__init__("text-to-image") + + def _prepare_payload_as_dict( + self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping + ) -> Optional[Dict]: + mapped_model = provider_mapping_info.provider_id + parameters = filter_none(parameters) + if "num_inference_steps" in parameters: + parameters["steps"] = parameters.pop("num_inference_steps") + if "guidance_scale" in parameters: + parameters["guidance"] = parameters.pop("guidance_scale") + + return {"prompt": inputs, "response_format": "base64", **parameters, "model": mapped_model} + + def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any: + response_dict = _as_dict(response) + return base64.b64decode(response_dict["data"][0]["b64_json"]) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/zai_org.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/zai_org.py new file mode 100644 index 0000000000000000000000000000000000000000..d6f4c42b5abc78a98474b2f8899d6b30a4a58f8d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/inference/_providers/zai_org.py @@ -0,0 +1,17 @@ +from typing import Any, Dict + +from huggingface_hub.inference._providers._common import BaseConversationalTask + + +class ZaiConversationalTask(BaseConversationalTask): + def __init__(self): + super().__init__(provider="zai-org", base_url="https://api.z.ai") + + def _prepare_headers(self, headers: Dict, api_key: str) -> Dict[str, Any]: + headers = super()._prepare_headers(headers, api_key) + headers["Accept-Language"] = "en-US,en" + headers["x-source-channel"] = "hugging_face" + return headers + + def _prepare_route(self, mapped_model: str, api_key: str) -> str: + return "/api/paas/v4/chat/completions" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/serialization/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/serialization/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8949a22a5f65ab29b7df65aa6a9df9bce0544b7e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/serialization/__init__.py @@ -0,0 +1,27 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ruff: noqa: F401 +"""Contains helpers to serialize tensors.""" + +from ._base import StateDictSplit, split_state_dict_into_shards_factory +from ._tensorflow import get_tf_storage_size, split_tf_state_dict_into_shards +from ._torch import ( + get_torch_storage_id, + get_torch_storage_size, + load_state_dict_from_file, + load_torch_model, + save_torch_model, + save_torch_state_dict, + split_torch_state_dict_into_shards, +) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/serialization/_base.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/serialization/_base.py new file mode 100644 index 0000000000000000000000000000000000000000..b79c82f5dba58d252b5c3a7345f0df09794b55ce --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/serialization/_base.py @@ -0,0 +1,207 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains helpers to split tensors into shards.""" + +from dataclasses import dataclass, field +from typing import Any, Callable, Dict, List, Optional, TypeVar, Union + +from .. import logging + + +TensorT = TypeVar("TensorT") +TensorSizeFn_T = Callable[[TensorT], int] +StorageIDFn_T = Callable[[TensorT], Optional[Any]] + +MAX_SHARD_SIZE = "5GB" +SIZE_UNITS = { + "TB": 10**12, + "GB": 10**9, + "MB": 10**6, + "KB": 10**3, +} + + +logger = logging.get_logger(__file__) + + +@dataclass +class StateDictSplit: + is_sharded: bool = field(init=False) + metadata: Dict[str, Any] + filename_to_tensors: Dict[str, List[str]] + tensor_to_filename: Dict[str, str] + + def __post_init__(self): + self.is_sharded = len(self.filename_to_tensors) > 1 + + +def split_state_dict_into_shards_factory( + state_dict: Dict[str, TensorT], + *, + get_storage_size: TensorSizeFn_T, + filename_pattern: str, + get_storage_id: StorageIDFn_T = lambda tensor: None, + max_shard_size: Union[int, str] = MAX_SHARD_SIZE, +) -> StateDictSplit: + """ + Split a model state dictionary in shards so that each shard is smaller than a given size. + + The shards are determined by iterating through the `state_dict` in the order of its keys. There is no optimization + made to make each shard as close as possible to the maximum size passed. For example, if the limit is 10GB and we + have tensors of sizes [6GB, 6GB, 2GB, 6GB, 2GB, 2GB] they will get sharded as [6GB], [6+2GB], [6+2+2GB] and not + [6+2+2GB], [6+2GB], [6GB]. + + > [!WARNING] + > If one of the model's tensor is bigger than `max_shard_size`, it will end up in its own shard which will have a + > size greater than `max_shard_size`. + + Args: + state_dict (`Dict[str, Tensor]`): + The state dictionary to save. + get_storage_size (`Callable[[Tensor], int]`): + A function that returns the size of a tensor when saved on disk in bytes. + get_storage_id (`Callable[[Tensor], Optional[Any]]`, *optional*): + A function that returns a unique identifier to a tensor storage. Multiple different tensors can share the + same underlying storage. This identifier is guaranteed to be unique and constant for this tensor's storage + during its lifetime. Two tensor storages with non-overlapping lifetimes may have the same id. + filename_pattern (`str`, *optional*): + The pattern to generate the files names in which the model will be saved. Pattern must be a string that + can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix` + max_shard_size (`int` or `str`, *optional*): + The maximum size of each shard, in bytes. Defaults to 5GB. + + Returns: + [`StateDictSplit`]: A `StateDictSplit` object containing the shards and the index to retrieve them. + """ + storage_id_to_tensors: Dict[Any, List[str]] = {} + + shard_list: List[Dict[str, TensorT]] = [] + current_shard: Dict[str, TensorT] = {} + current_shard_size = 0 + total_size = 0 + + if isinstance(max_shard_size, str): + max_shard_size = parse_size_to_int(max_shard_size) + + for key, tensor in state_dict.items(): + # when bnb serialization is used the weights in the state dict can be strings + # check: https://github.com/huggingface/transformers/pull/24416 for more details + if isinstance(tensor, str): + logger.info("Skipping tensor %s as it is a string (bnb serialization)", key) + continue + + # If a `tensor` shares the same underlying storage as another tensor, we put `tensor` in the same `block` + storage_id = get_storage_id(tensor) + if storage_id is not None: + if storage_id in storage_id_to_tensors: + # We skip this tensor for now and will reassign to correct shard later + storage_id_to_tensors[storage_id].append(key) + continue + else: + # This is the first tensor with this storage_id, we create a new entry + # in the storage_id_to_tensors dict => we will assign the shard id later + storage_id_to_tensors[storage_id] = [key] + + # Compute tensor size + tensor_size = get_storage_size(tensor) + + # If this tensor is bigger than the maximal size, we put it in its own shard + if tensor_size > max_shard_size: + total_size += tensor_size + shard_list.append({key: tensor}) + continue + + # If this tensor is going to tip up over the maximal size, we split. + # Current shard already has some tensors, we add it to the list of shards and create a new one. + if current_shard_size + tensor_size > max_shard_size: + shard_list.append(current_shard) + current_shard = {} + current_shard_size = 0 + + # Add the tensor to the current shard + current_shard[key] = tensor + current_shard_size += tensor_size + total_size += tensor_size + + # Add the last shard + if len(current_shard) > 0: + shard_list.append(current_shard) + nb_shards = len(shard_list) + + # Loop over the tensors that share the same storage and assign them together + for storage_id, keys in storage_id_to_tensors.items(): + # Let's try to find the shard where the first tensor of this storage is and put all tensors in the same shard + for shard in shard_list: + if keys[0] in shard: + for key in keys: + shard[key] = state_dict[key] + break + + # If we only have one shard, we return it => no need to build the index + if nb_shards == 1: + filename = filename_pattern.format(suffix="") + return StateDictSplit( + metadata={"total_size": total_size}, + filename_to_tensors={filename: list(state_dict.keys())}, + tensor_to_filename={key: filename for key in state_dict.keys()}, + ) + + # Now that each tensor is assigned to a shard, let's assign a filename to each shard + tensor_name_to_filename = {} + filename_to_tensors = {} + for idx, shard in enumerate(shard_list): + filename = filename_pattern.format(suffix=f"-{idx + 1:05d}-of-{nb_shards:05d}") + for key in shard: + tensor_name_to_filename[key] = filename + filename_to_tensors[filename] = list(shard.keys()) + + # Build the index and return + return StateDictSplit( + metadata={"total_size": total_size}, + filename_to_tensors=filename_to_tensors, + tensor_to_filename=tensor_name_to_filename, + ) + + +def parse_size_to_int(size_as_str: str) -> int: + """ + Parse a size expressed as a string with digits and unit (like `"5MB"`) to an integer (in bytes). + + Supported units are "TB", "GB", "MB", "KB". + + Args: + size_as_str (`str`): The size to convert. Will be directly returned if an `int`. + + Example: + + ```py + >>> parse_size_to_int("5MB") + 5000000 + ``` + """ + size_as_str = size_as_str.strip() + + # Parse unit + unit = size_as_str[-2:].upper() + if unit not in SIZE_UNITS: + raise ValueError(f"Unit '{unit}' not supported. Supported units are TB, GB, MB, KB. Got '{size_as_str}'.") + multiplier = SIZE_UNITS[unit] + + # Parse value + try: + value = float(size_as_str[:-2].strip()) + except ValueError as e: + raise ValueError(f"Could not parse the size value from '{size_as_str}': {e}") from e + + return int(value * multiplier) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/serialization/_dduf.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/serialization/_dduf.py new file mode 100644 index 0000000000000000000000000000000000000000..a1debadb3ac8a45716f0359b932dc065f09edb84 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/serialization/_dduf.py @@ -0,0 +1,387 @@ +import json +import logging +import mmap +import os +import shutil +import zipfile +from contextlib import contextmanager +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, Generator, Iterable, Tuple, Union + +from ..errors import DDUFCorruptedFileError, DDUFExportError, DDUFInvalidEntryNameError + + +logger = logging.getLogger(__name__) + +DDUF_ALLOWED_ENTRIES = { + # Allowed file extensions in a DDUF file + ".json", + ".model", + ".safetensors", + ".txt", +} + +DDUF_FOLDER_REQUIRED_ENTRIES = { + # Each folder must contain at least one of these entries + "config.json", + "tokenizer_config.json", + "preprocessor_config.json", + "scheduler_config.json", +} + + +@dataclass +class DDUFEntry: + """Object representing a file entry in a DDUF file. + + See [`read_dduf_file`] for how to read a DDUF file. + + Attributes: + filename (str): + The name of the file in the DDUF archive. + offset (int): + The offset of the file in the DDUF archive. + length (int): + The length of the file in the DDUF archive. + dduf_path (str): + The path to the DDUF archive (for internal use). + """ + + filename: str + length: int + offset: int + + dduf_path: Path = field(repr=False) + + @contextmanager + def as_mmap(self) -> Generator[bytes, None, None]: + """Open the file as a memory-mapped file. + + Useful to load safetensors directly from the file. + + Example: + ```py + >>> import safetensors.torch + >>> with entry.as_mmap() as mm: + ... tensors = safetensors.torch.load(mm) + ``` + """ + with self.dduf_path.open("rb") as f: + with mmap.mmap(f.fileno(), length=0, access=mmap.ACCESS_READ) as mm: + yield mm[self.offset : self.offset + self.length] + + def read_text(self, encoding: str = "utf-8") -> str: + """Read the file as text. + + Useful for '.txt' and '.json' entries. + + Example: + ```py + >>> import json + >>> index = json.loads(entry.read_text()) + ``` + """ + with self.dduf_path.open("rb") as f: + f.seek(self.offset) + return f.read(self.length).decode(encoding=encoding) + + +def read_dduf_file(dduf_path: Union[os.PathLike, str]) -> Dict[str, DDUFEntry]: + """ + Read a DDUF file and return a dictionary of entries. + + Only the metadata is read, the data is not loaded in memory. + + Args: + dduf_path (`str` or `os.PathLike`): + The path to the DDUF file to read. + + Returns: + `Dict[str, DDUFEntry]`: + A dictionary of [`DDUFEntry`] indexed by filename. + + Raises: + - [`DDUFCorruptedFileError`]: If the DDUF file is corrupted (i.e. doesn't follow the DDUF format). + + Example: + ```python + >>> import json + >>> import safetensors.torch + >>> from huggingface_hub import read_dduf_file + + # Read DDUF metadata + >>> dduf_entries = read_dduf_file("FLUX.1-dev.dduf") + + # Returns a mapping filename <> DDUFEntry + >>> dduf_entries["model_index.json"] + DDUFEntry(filename='model_index.json', offset=66, length=587) + + # Load model index as JSON + >>> json.loads(dduf_entries["model_index.json"].read_text()) + {'_class_name': 'FluxPipeline', '_diffusers_version': '0.32.0.dev0', '_name_or_path': 'black-forest-labs/FLUX.1-dev', ... + + # Load VAE weights using safetensors + >>> with dduf_entries["vae/diffusion_pytorch_model.safetensors"].as_mmap() as mm: + ... state_dict = safetensors.torch.load(mm) + ``` + """ + entries = {} + dduf_path = Path(dduf_path) + logger.info(f"Reading DDUF file {dduf_path}") + with zipfile.ZipFile(str(dduf_path), "r") as zf: + for info in zf.infolist(): + logger.debug(f"Reading entry {info.filename}") + if info.compress_type != zipfile.ZIP_STORED: + raise DDUFCorruptedFileError("Data must not be compressed in DDUF file.") + + try: + _validate_dduf_entry_name(info.filename) + except DDUFInvalidEntryNameError as e: + raise DDUFCorruptedFileError(f"Invalid entry name in DDUF file: {info.filename}") from e + + offset = _get_data_offset(zf, info) + + entries[info.filename] = DDUFEntry( + filename=info.filename, offset=offset, length=info.file_size, dduf_path=dduf_path + ) + + # Consistency checks on the DDUF file + if "model_index.json" not in entries: + raise DDUFCorruptedFileError("Missing required 'model_index.json' entry in DDUF file.") + index = json.loads(entries["model_index.json"].read_text()) + _validate_dduf_structure(index, entries.keys()) + + logger.info(f"Done reading DDUF file {dduf_path}. Found {len(entries)} entries") + return entries + + +def export_entries_as_dduf( + dduf_path: Union[str, os.PathLike], entries: Iterable[Tuple[str, Union[str, Path, bytes]]] +) -> None: + """Write a DDUF file from an iterable of entries. + + This is a lower-level helper than [`export_folder_as_dduf`] that allows more flexibility when serializing data. + In particular, you don't need to save the data on disk before exporting it in the DDUF file. + + Args: + dduf_path (`str` or `os.PathLike`): + The path to the DDUF file to write. + entries (`Iterable[Tuple[str, Union[str, Path, bytes]]]`): + An iterable of entries to write in the DDUF file. Each entry is a tuple with the filename and the content. + The filename should be the path to the file in the DDUF archive. + The content can be a string or a pathlib.Path representing a path to a file on the local disk or directly the content as bytes. + + Raises: + - [`DDUFExportError`]: If anything goes wrong during the export (e.g. invalid entry name, missing 'model_index.json', etc.). + + Example: + ```python + # Export specific files from the local disk. + >>> from huggingface_hub import export_entries_as_dduf + >>> export_entries_as_dduf( + ... dduf_path="stable-diffusion-v1-4-FP16.dduf", + ... entries=[ # List entries to add to the DDUF file (here, only FP16 weights) + ... ("model_index.json", "path/to/model_index.json"), + ... ("vae/config.json", "path/to/vae/config.json"), + ... ("vae/diffusion_pytorch_model.fp16.safetensors", "path/to/vae/diffusion_pytorch_model.fp16.safetensors"), + ... ("text_encoder/config.json", "path/to/text_encoder/config.json"), + ... ("text_encoder/model.fp16.safetensors", "path/to/text_encoder/model.fp16.safetensors"), + ... # ... add more entries here + ... ] + ... ) + ``` + + ```python + # Export state_dicts one by one from a loaded pipeline + >>> from diffusers import DiffusionPipeline + >>> from typing import Generator, Tuple + >>> import safetensors.torch + >>> from huggingface_hub import export_entries_as_dduf + >>> pipe = DiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4") + ... # ... do some work with the pipeline + + >>> def as_entries(pipe: DiffusionPipeline) -> Generator[Tuple[str, bytes], None, None]: + ... # Build an generator that yields the entries to add to the DDUF file. + ... # The first element of the tuple is the filename in the DDUF archive (must use UNIX separator!). The second element is the content of the file. + ... # Entries will be evaluated lazily when the DDUF file is created (only 1 entry is loaded in memory at a time) + ... yield "vae/config.json", pipe.vae.to_json_string().encode() + ... yield "vae/diffusion_pytorch_model.safetensors", safetensors.torch.save(pipe.vae.state_dict()) + ... yield "text_encoder/config.json", pipe.text_encoder.config.to_json_string().encode() + ... yield "text_encoder/model.safetensors", safetensors.torch.save(pipe.text_encoder.state_dict()) + ... # ... add more entries here + + >>> export_entries_as_dduf(dduf_path="stable-diffusion-v1-4.dduf", entries=as_entries(pipe)) + ``` + """ + logger.info(f"Exporting DDUF file '{dduf_path}'") + filenames = set() + index = None + with zipfile.ZipFile(str(dduf_path), "w", zipfile.ZIP_STORED) as archive: + for filename, content in entries: + if filename in filenames: + raise DDUFExportError(f"Can't add duplicate entry: {filename}") + filenames.add(filename) + + if filename == "model_index.json": + try: + index = json.loads(_load_content(content).decode()) + except json.JSONDecodeError as e: + raise DDUFExportError("Failed to parse 'model_index.json'.") from e + + try: + filename = _validate_dduf_entry_name(filename) + except DDUFInvalidEntryNameError as e: + raise DDUFExportError(f"Invalid entry name: {filename}") from e + logger.debug(f"Adding entry '{filename}' to DDUF file") + _dump_content_in_archive(archive, filename, content) + + # Consistency checks on the DDUF file + if index is None: + raise DDUFExportError("Missing required 'model_index.json' entry in DDUF file.") + try: + _validate_dduf_structure(index, filenames) + except DDUFCorruptedFileError as e: + raise DDUFExportError("Invalid DDUF file structure.") from e + + logger.info(f"Done writing DDUF file {dduf_path}") + + +def export_folder_as_dduf(dduf_path: Union[str, os.PathLike], folder_path: Union[str, os.PathLike]) -> None: + """ + Export a folder as a DDUF file. + + AUses [`export_entries_as_dduf`] under the hood. + + Args: + dduf_path (`str` or `os.PathLike`): + The path to the DDUF file to write. + folder_path (`str` or `os.PathLike`): + The path to the folder containing the diffusion model. + + Example: + ```python + >>> from huggingface_hub import export_folder_as_dduf + >>> export_folder_as_dduf(dduf_path="FLUX.1-dev.dduf", folder_path="path/to/FLUX.1-dev") + ``` + """ + folder_path = Path(folder_path) + + def _iterate_over_folder() -> Iterable[Tuple[str, Path]]: + for path in Path(folder_path).glob("**/*"): + if not path.is_file(): + continue + if path.suffix not in DDUF_ALLOWED_ENTRIES: + logger.debug(f"Skipping file '{path}' (file type not allowed)") + continue + path_in_archive = path.relative_to(folder_path) + if len(path_in_archive.parts) >= 3: + logger.debug(f"Skipping file '{path}' (nested directories not allowed)") + continue + yield path_in_archive.as_posix(), path + + export_entries_as_dduf(dduf_path, _iterate_over_folder()) + + +def _dump_content_in_archive(archive: zipfile.ZipFile, filename: str, content: Union[str, os.PathLike, bytes]) -> None: + with archive.open(filename, "w", force_zip64=True) as archive_fh: + if isinstance(content, (str, Path)): + content_path = Path(content) + with content_path.open("rb") as content_fh: + shutil.copyfileobj(content_fh, archive_fh, 1024 * 1024 * 8) # type: ignore[misc] + elif isinstance(content, bytes): + archive_fh.write(content) + else: + raise DDUFExportError(f"Invalid content type for {filename}. Must be str, Path or bytes.") + + +def _load_content(content: Union[str, Path, bytes]) -> bytes: + """Load the content of an entry as bytes. + + Used only for small checks (not to dump content into archive). + """ + if isinstance(content, (str, Path)): + return Path(content).read_bytes() + elif isinstance(content, bytes): + return content + else: + raise DDUFExportError(f"Invalid content type. Must be str, Path or bytes. Got {type(content)}.") + + +def _validate_dduf_entry_name(entry_name: str) -> str: + if "." + entry_name.split(".")[-1] not in DDUF_ALLOWED_ENTRIES: + raise DDUFInvalidEntryNameError(f"File type not allowed: {entry_name}") + if "\\" in entry_name: + raise DDUFInvalidEntryNameError(f"Entry names must use UNIX separators ('/'). Got {entry_name}.") + entry_name = entry_name.strip("/") + if entry_name.count("/") > 1: + raise DDUFInvalidEntryNameError(f"DDUF only supports 1 level of directory. Got {entry_name}.") + return entry_name + + +def _validate_dduf_structure(index: Any, entry_names: Iterable[str]) -> None: + """ + Consistency checks on the DDUF file structure. + + Rules: + - The 'model_index.json' entry is required and must contain a dictionary. + - Each folder name must correspond to an entry in 'model_index.json'. + - Each folder must contain at least a config file ('config.json', 'tokenizer_config.json', 'preprocessor_config.json', 'scheduler_config.json'). + + Args: + index (Any): + The content of the 'model_index.json' entry. + entry_names (Iterable[str]): + The list of entry names in the DDUF file. + + Raises: + - [`DDUFCorruptedFileError`]: If the DDUF file is corrupted (i.e. doesn't follow the DDUF format). + """ + if not isinstance(index, dict): + raise DDUFCorruptedFileError(f"Invalid 'model_index.json' content. Must be a dictionary. Got {type(index)}.") + + dduf_folders = {entry.split("/")[0] for entry in entry_names if "/" in entry} + for folder in dduf_folders: + if folder not in index: + raise DDUFCorruptedFileError(f"Missing required entry '{folder}' in 'model_index.json'.") + if not any(f"{folder}/{required_entry}" in entry_names for required_entry in DDUF_FOLDER_REQUIRED_ENTRIES): + raise DDUFCorruptedFileError( + f"Missing required file in folder '{folder}'. Must contains at least one of {DDUF_FOLDER_REQUIRED_ENTRIES}." + ) + + +def _get_data_offset(zf: zipfile.ZipFile, info: zipfile.ZipInfo) -> int: + """ + Calculate the data offset for a file in a ZIP archive. + + Args: + zf (`zipfile.ZipFile`): + The opened ZIP file. Must be opened in read mode. + info (`zipfile.ZipInfo`): + The file info. + + Returns: + int: The offset of the file data in the ZIP archive. + """ + if zf.fp is None: + raise DDUFCorruptedFileError("ZipFile object must be opened in read mode.") + + # Step 1: Get the local file header offset + header_offset = info.header_offset + + # Step 2: Read the local file header + zf.fp.seek(header_offset) + local_file_header = zf.fp.read(30) # Fixed-size part of the local header + + if len(local_file_header) < 30: + raise DDUFCorruptedFileError("Incomplete local file header.") + + # Step 3: Parse the header fields to calculate the start of file data + # Local file header: https://en.wikipedia.org/wiki/ZIP_(file_format)#File_headers + filename_len = int.from_bytes(local_file_header[26:28], "little") + extra_field_len = int.from_bytes(local_file_header[28:30], "little") + + # Data offset is after the fixed header, filename, and extra fields + data_offset = header_offset + 30 + filename_len + extra_field_len + + return data_offset diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/serialization/_tensorflow.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/serialization/_tensorflow.py new file mode 100644 index 0000000000000000000000000000000000000000..1173e34a28b2d7f9d879e01ffdae8ce09e9d5b5c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/serialization/_tensorflow.py @@ -0,0 +1,92 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains tensorflow-specific helpers.""" + +import math +import re +from typing import TYPE_CHECKING, Dict, Union + +from .. import constants +from ._base import MAX_SHARD_SIZE, StateDictSplit, split_state_dict_into_shards_factory + + +if TYPE_CHECKING: + import tensorflow as tf + + +def split_tf_state_dict_into_shards( + state_dict: Dict[str, "tf.Tensor"], + *, + filename_pattern: str = constants.TF2_WEIGHTS_FILE_PATTERN, + max_shard_size: Union[int, str] = MAX_SHARD_SIZE, +) -> StateDictSplit: + """ + Split a model state dictionary in shards so that each shard is smaller than a given size. + + The shards are determined by iterating through the `state_dict` in the order of its keys. There is no optimization + made to make each shard as close as possible to the maximum size passed. For example, if the limit is 10GB and we + have tensors of sizes [6GB, 6GB, 2GB, 6GB, 2GB, 2GB] they will get sharded as [6GB], [6+2GB], [6+2+2GB] and not + [6+2+2GB], [6+2GB], [6GB]. + + > [!WARNING] + > If one of the model's tensor is bigger than `max_shard_size`, it will end up in its own shard which will have a + > size greater than `max_shard_size`. + + Args: + state_dict (`Dict[str, Tensor]`): + The state dictionary to save. + filename_pattern (`str`, *optional*): + The pattern to generate the files names in which the model will be saved. Pattern must be a string that + can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix` + Defaults to `"tf_model{suffix}.h5"`. + max_shard_size (`int` or `str`, *optional*): + The maximum size of each shard, in bytes. Defaults to 5GB. + + Returns: + [`StateDictSplit`]: A `StateDictSplit` object containing the shards and the index to retrieve them. + """ + return split_state_dict_into_shards_factory( + state_dict, + max_shard_size=max_shard_size, + filename_pattern=filename_pattern, + get_storage_size=get_tf_storage_size, + ) + + +def get_tf_storage_size(tensor: "tf.Tensor") -> int: + # Return `math.ceil` since dtype byte size can be a float (e.g., 0.125 for tf.bool). + # Better to overestimate than underestimate. + return math.ceil(tensor.numpy().size * _dtype_byte_size_tf(tensor.dtype)) + + +def _dtype_byte_size_tf(dtype) -> float: + """ + Returns the size (in bytes) occupied by one parameter of type `dtype`. + Taken from https://github.com/huggingface/transformers/blob/74d9d0cebb0263a3f8ab9c280569170cc74651d0/src/transformers/modeling_tf_utils.py#L608. + NOTE: why not `tensor.numpy().nbytes`? + Example: + ```py + >>> _dtype_byte_size(tf.float32) + 4 + ``` + """ + import tensorflow as tf + + if dtype == tf.bool: + return 1 / 8 + bit_search = re.search(r"[^\d](\d+)$", dtype.name) + if bit_search is None: + raise ValueError(f"`dtype` is not a valid dtype: {dtype}.") + bit_size = int(bit_search.groups()[0]) + return bit_size // 8 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/serialization/_torch.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/serialization/_torch.py new file mode 100644 index 0000000000000000000000000000000000000000..e24d46ab4e14415104922681cd64944a33a3d9ab --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/serialization/_torch.py @@ -0,0 +1,1015 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains pytorch-specific helpers.""" + +import importlib +import json +import os +import re +from collections import defaultdict, namedtuple +from functools import lru_cache +from pathlib import Path +from typing import TYPE_CHECKING, Any, Dict, Iterable, List, NamedTuple, Optional, Set, Tuple, Union + +from packaging import version + +from .. import constants, logging +from ._base import MAX_SHARD_SIZE, StateDictSplit, split_state_dict_into_shards_factory + + +logger = logging.get_logger(__file__) + +if TYPE_CHECKING: + import torch + +# SAVING + + +def save_torch_model( + model: "torch.nn.Module", + save_directory: Union[str, Path], + *, + filename_pattern: Optional[str] = None, + force_contiguous: bool = True, + max_shard_size: Union[int, str] = MAX_SHARD_SIZE, + metadata: Optional[Dict[str, str]] = None, + safe_serialization: bool = True, + is_main_process: bool = True, + shared_tensors_to_discard: Optional[List[str]] = None, +): + """ + Saves a given torch model to disk, handling sharding and shared tensors issues. + + See also [`save_torch_state_dict`] to save a state dict with more flexibility. + + For more information about tensor sharing, check out [this guide](https://huggingface.co/docs/safetensors/torch_shared_tensors). + + The model state dictionary is split into shards so that each shard is smaller than a given size. The shards are + saved in the `save_directory` with the given `filename_pattern`. If the model is too big to fit in a single shard, + an index file is saved in the `save_directory` to indicate where each tensor is saved. This helper uses + [`split_torch_state_dict_into_shards`] under the hood. If `safe_serialization` is `True`, the shards are saved as + safetensors (the default). Otherwise, the shards are saved as pickle. + + Before saving the model, the `save_directory` is cleaned from any previous shard files. + + > [!WARNING] + > If one of the model's tensor is bigger than `max_shard_size`, it will end up in its own shard which will have a + > size greater than `max_shard_size`. + + > [!WARNING] + > If your model is a `transformers.PreTrainedModel`, you should pass `model._tied_weights_keys` as `shared_tensors_to_discard` to properly handle shared tensors saving. This ensures the correct duplicate tensors are discarded during saving. + + Args: + model (`torch.nn.Module`): + The model to save on disk. + save_directory (`str` or `Path`): + The directory in which the model will be saved. + filename_pattern (`str`, *optional*): + The pattern to generate the files names in which the model will be saved. Pattern must be a string that + can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix` + Defaults to `"model{suffix}.safetensors"` or `pytorch_model{suffix}.bin` depending on `safe_serialization` + parameter. + force_contiguous (`boolean`, *optional*): + Forcing the state_dict to be saved as contiguous tensors. This has no effect on the correctness of the + model, but it could potentially change performance if the layout of the tensor was chosen specifically for + that reason. Defaults to `True`. + max_shard_size (`int` or `str`, *optional*): + The maximum size of each shard, in bytes. Defaults to 5GB. + metadata (`Dict[str, str]`, *optional*): + Extra information to save along with the model. Some metadata will be added for each dropped tensors. + This information will not be enough to recover the entire shared structure but might help understanding + things. + safe_serialization (`bool`, *optional*): + Whether to save as safetensors, which is the default behavior. If `False`, the shards are saved as pickle. + Safe serialization is recommended for security reasons. Saving as pickle is deprecated and will be removed + in a future version. + is_main_process (`bool`, *optional*): + Whether the process calling this is the main process or not. Useful when in distributed training like + TPUs and need to call this function from all processes. In this case, set `is_main_process=True` only on + the main process to avoid race conditions. Defaults to True. + shared_tensors_to_discard (`List[str]`, *optional*): + List of tensor names to drop when saving shared tensors. If not provided and shared tensors are + detected, it will drop the first name alphabetically. + + Example: + + ```py + >>> from huggingface_hub import save_torch_model + >>> model = ... # A PyTorch model + + # Save state dict to "path/to/folder". The model will be split into shards of 5GB each and saved as safetensors. + >>> save_torch_model(model, "path/to/folder") + + # Load model back + >>> from huggingface_hub import load_torch_model # TODO + >>> load_torch_model(model, "path/to/folder") + >>> + ``` + """ + save_torch_state_dict( + state_dict=model.state_dict(), + filename_pattern=filename_pattern, + force_contiguous=force_contiguous, + max_shard_size=max_shard_size, + metadata=metadata, + safe_serialization=safe_serialization, + save_directory=save_directory, + is_main_process=is_main_process, + shared_tensors_to_discard=shared_tensors_to_discard, + ) + + +def save_torch_state_dict( + state_dict: Dict[str, "torch.Tensor"], + save_directory: Union[str, Path], + *, + filename_pattern: Optional[str] = None, + force_contiguous: bool = True, + max_shard_size: Union[int, str] = MAX_SHARD_SIZE, + metadata: Optional[Dict[str, str]] = None, + safe_serialization: bool = True, + is_main_process: bool = True, + shared_tensors_to_discard: Optional[List[str]] = None, +) -> None: + """ + Save a model state dictionary to the disk, handling sharding and shared tensors issues. + + See also [`save_torch_model`] to directly save a PyTorch model. + + For more information about tensor sharing, check out [this guide](https://huggingface.co/docs/safetensors/torch_shared_tensors). + + The model state dictionary is split into shards so that each shard is smaller than a given size. The shards are + saved in the `save_directory` with the given `filename_pattern`. If the model is too big to fit in a single shard, + an index file is saved in the `save_directory` to indicate where each tensor is saved. This helper uses + [`split_torch_state_dict_into_shards`] under the hood. If `safe_serialization` is `True`, the shards are saved as + safetensors (the default). Otherwise, the shards are saved as pickle. + + Before saving the model, the `save_directory` is cleaned from any previous shard files. + + > [!WARNING] + > If one of the model's tensor is bigger than `max_shard_size`, it will end up in its own shard which will have a + > size greater than `max_shard_size`. + + > [!WARNING] + > If your model is a `transformers.PreTrainedModel`, you should pass `model._tied_weights_keys` as `shared_tensors_to_discard` to properly handle shared tensors saving. This ensures the correct duplicate tensors are discarded during saving. + + Args: + state_dict (`Dict[str, torch.Tensor]`): + The state dictionary to save. + save_directory (`str` or `Path`): + The directory in which the model will be saved. + filename_pattern (`str`, *optional*): + The pattern to generate the files names in which the model will be saved. Pattern must be a string that + can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix` + Defaults to `"model{suffix}.safetensors"` or `pytorch_model{suffix}.bin` depending on `safe_serialization` + parameter. + force_contiguous (`boolean`, *optional*): + Forcing the state_dict to be saved as contiguous tensors. This has no effect on the correctness of the + model, but it could potentially change performance if the layout of the tensor was chosen specifically for + that reason. Defaults to `True`. + max_shard_size (`int` or `str`, *optional*): + The maximum size of each shard, in bytes. Defaults to 5GB. + metadata (`Dict[str, str]`, *optional*): + Extra information to save along with the model. Some metadata will be added for each dropped tensors. + This information will not be enough to recover the entire shared structure but might help understanding + things. + safe_serialization (`bool`, *optional*): + Whether to save as safetensors, which is the default behavior. If `False`, the shards are saved as pickle. + Safe serialization is recommended for security reasons. Saving as pickle is deprecated and will be removed + in a future version. + is_main_process (`bool`, *optional*): + Whether the process calling this is the main process or not. Useful when in distributed training like + TPUs and need to call this function from all processes. In this case, set `is_main_process=True` only on + the main process to avoid race conditions. Defaults to True. + shared_tensors_to_discard (`List[str]`, *optional*): + List of tensor names to drop when saving shared tensors. If not provided and shared tensors are + detected, it will drop the first name alphabetically. + + Example: + + ```py + >>> from huggingface_hub import save_torch_state_dict + >>> model = ... # A PyTorch model + + # Save state dict to "path/to/folder". The model will be split into shards of 5GB each and saved as safetensors. + >>> state_dict = model_to_save.state_dict() + >>> save_torch_state_dict(state_dict, "path/to/folder") + ``` + """ + save_directory = str(save_directory) + + if filename_pattern is None: + filename_pattern = ( + constants.SAFETENSORS_WEIGHTS_FILE_PATTERN + if safe_serialization + else constants.PYTORCH_WEIGHTS_FILE_PATTERN + ) + + if metadata is None: + metadata = {} + if safe_serialization: + try: + from safetensors.torch import save_file as save_file_fn + except ImportError as e: + raise ImportError( + "Please install `safetensors` to use safe serialization. " + "You can install it with `pip install safetensors`." + ) from e + # Clean state dict for safetensors + state_dict = _clean_state_dict_for_safetensors( + state_dict, + metadata, + force_contiguous=force_contiguous, + shared_tensors_to_discard=shared_tensors_to_discard, + ) + else: + from torch import save as save_file_fn # type: ignore[assignment, no-redef] + + logger.warning( + "You are using unsafe serialization. Due to security reasons, it is recommended not to load " + "pickled models from untrusted sources. If you intend to share your model, we strongly recommend " + "using safe serialization by installing `safetensors` with `pip install safetensors`." + ) + # Split dict + state_dict_split = split_torch_state_dict_into_shards( + state_dict, filename_pattern=filename_pattern, max_shard_size=max_shard_size + ) + + # Only main process should clean up existing files to avoid race conditions in distributed environment + if is_main_process: + existing_files_regex = re.compile(filename_pattern.format(suffix=r"(-\d{5}-of-\d{5})?") + r"(\.index\.json)?") + for filename in os.listdir(save_directory): + if existing_files_regex.match(filename): + try: + logger.debug(f"Removing existing file '{filename}' from folder.") + os.remove(os.path.join(save_directory, filename)) + except Exception as e: + logger.warning( + f"Error when trying to remove existing '{filename}' from folder: {e}. Continuing..." + ) + + # Save each shard + per_file_metadata = {"format": "pt"} + if not state_dict_split.is_sharded: + per_file_metadata.update(metadata) + safe_file_kwargs = {"metadata": per_file_metadata} if safe_serialization else {} + for filename, tensors in state_dict_split.filename_to_tensors.items(): + shard = {tensor: state_dict[tensor] for tensor in tensors} + save_file_fn(shard, os.path.join(save_directory, filename), **safe_file_kwargs) # ty: ignore[invalid-argument-type] + logger.debug(f"Shard saved to {filename}") + + # Save the index (if any) + if state_dict_split.is_sharded: + index_path = filename_pattern.format(suffix="") + ".index.json" + index = { + "metadata": {**state_dict_split.metadata, **metadata}, + "weight_map": state_dict_split.tensor_to_filename, + } + with open(os.path.join(save_directory, index_path), "w") as f: + json.dump(index, f, indent=2) + logger.info( + f"The model is bigger than the maximum size per checkpoint ({max_shard_size}). " + f"Model weighs have been saved in {len(state_dict_split.filename_to_tensors)} checkpoint shards. " + f"You can find where each parameters has been saved in the index located at {index_path}." + ) + + logger.info(f"Model weights successfully saved to {save_directory}!") + + +def split_torch_state_dict_into_shards( + state_dict: Dict[str, "torch.Tensor"], + *, + filename_pattern: str = constants.SAFETENSORS_WEIGHTS_FILE_PATTERN, + max_shard_size: Union[int, str] = MAX_SHARD_SIZE, +) -> StateDictSplit: + """ + Split a model state dictionary in shards so that each shard is smaller than a given size. + + The shards are determined by iterating through the `state_dict` in the order of its keys. There is no optimization + made to make each shard as close as possible to the maximum size passed. For example, if the limit is 10GB and we + have tensors of sizes [6GB, 6GB, 2GB, 6GB, 2GB, 2GB] they will get sharded as [6GB], [6+2GB], [6+2+2GB] and not + [6+2+2GB], [6+2GB], [6GB]. + + + > [!TIP] + > To save a model state dictionary to the disk, see [`save_torch_state_dict`]. This helper uses + > `split_torch_state_dict_into_shards` under the hood. + + > [!WARNING] + > If one of the model's tensor is bigger than `max_shard_size`, it will end up in its own shard which will have a + > size greater than `max_shard_size`. + + Args: + state_dict (`Dict[str, torch.Tensor]`): + The state dictionary to save. + filename_pattern (`str`, *optional*): + The pattern to generate the files names in which the model will be saved. Pattern must be a string that + can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix` + Defaults to `"model{suffix}.safetensors"`. + max_shard_size (`int` or `str`, *optional*): + The maximum size of each shard, in bytes. Defaults to 5GB. + + Returns: + [`StateDictSplit`]: A `StateDictSplit` object containing the shards and the index to retrieve them. + + Example: + ```py + >>> import json + >>> import os + >>> from safetensors.torch import save_file as safe_save_file + >>> from huggingface_hub import split_torch_state_dict_into_shards + + >>> def save_state_dict(state_dict: Dict[str, torch.Tensor], save_directory: str): + ... state_dict_split = split_torch_state_dict_into_shards(state_dict) + ... for filename, tensors in state_dict_split.filename_to_tensors.items(): + ... shard = {tensor: state_dict[tensor] for tensor in tensors} + ... safe_save_file( + ... shard, + ... os.path.join(save_directory, filename), + ... metadata={"format": "pt"}, + ... ) + ... if state_dict_split.is_sharded: + ... index = { + ... "metadata": state_dict_split.metadata, + ... "weight_map": state_dict_split.tensor_to_filename, + ... } + ... with open(os.path.join(save_directory, "model.safetensors.index.json"), "w") as f: + ... f.write(json.dumps(index, indent=2)) + ``` + """ + return split_state_dict_into_shards_factory( + state_dict, + max_shard_size=max_shard_size, + filename_pattern=filename_pattern, + get_storage_size=get_torch_storage_size, + get_storage_id=get_torch_storage_id, + ) + + +# LOADING + + +def load_torch_model( + model: "torch.nn.Module", + checkpoint_path: Union[str, os.PathLike], + *, + strict: bool = False, + safe: bool = True, + weights_only: bool = False, + map_location: Optional[Union[str, "torch.device"]] = None, + mmap: bool = False, + filename_pattern: Optional[str] = None, +) -> NamedTuple: + """ + Load a checkpoint into a model, handling both sharded and non-sharded checkpoints. + + Args: + model (`torch.nn.Module`): + The model in which to load the checkpoint. + checkpoint_path (`str` or `os.PathLike`): + Path to either the checkpoint file or directory containing the checkpoint(s). + strict (`bool`, *optional*, defaults to `False`): + Whether to strictly enforce that the keys in the model state dict match the keys in the checkpoint. + safe (`bool`, *optional*, defaults to `True`): + If `safe` is True, the safetensors files will be loaded. If `safe` is False, the function + will first attempt to load safetensors files if they are available, otherwise it will fall back to loading + pickle files. `filename_pattern` parameter takes precedence over `safe` parameter. + weights_only (`bool`, *optional*, defaults to `False`): + If True, only loads the model weights without optimizer states and other metadata. + Only supported in PyTorch >= 1.13. + map_location (`str` or `torch.device`, *optional*): + A `torch.device` object, string or a dict specifying how to remap storage locations. It + indicates the location where all tensors should be loaded. + mmap (`bool`, *optional*, defaults to `False`): + Whether to use memory-mapped file loading. Memory mapping can improve loading performance + for large models in PyTorch >= 2.1.0 with zipfile-based checkpoints. + filename_pattern (`str`, *optional*): + The pattern to look for the index file. Pattern must be a string that + can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix` + Defaults to `"model{suffix}.safetensors"`. + Returns: + `NamedTuple`: A named tuple with `missing_keys` and `unexpected_keys` fields. + - `missing_keys` is a list of str containing the missing keys, i.e. keys that are in the model but not in the checkpoint. + - `unexpected_keys` is a list of str containing the unexpected keys, i.e. keys that are in the checkpoint but not in the model. + + Raises: + [`FileNotFoundError`](https://docs.python.org/3/library/exceptions.html#FileNotFoundError) + If the checkpoint file or directory does not exist. + [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError) + If safetensors or torch is not installed when trying to load a .safetensors file or a PyTorch checkpoint respectively. + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If the checkpoint path is invalid or if the checkpoint format cannot be determined. + + Example: + ```python + >>> from huggingface_hub import load_torch_model + >>> model = ... # A PyTorch model + >>> load_torch_model(model, "path/to/checkpoint") + ``` + """ + checkpoint_path = Path(checkpoint_path) + + if not checkpoint_path.exists(): + raise ValueError(f"Checkpoint path {checkpoint_path} does not exist") + # 1. Check if checkpoint is a single file + if checkpoint_path.is_file(): + state_dict = load_state_dict_from_file( + checkpoint_file=checkpoint_path, + map_location=map_location, + weights_only=weights_only, + ) + return model.load_state_dict(state_dict, strict=strict) + + # 2. If not, checkpoint_path is a directory + if filename_pattern is None: + filename_pattern = constants.SAFETENSORS_WEIGHTS_FILE_PATTERN + index_path = checkpoint_path / (filename_pattern.format(suffix="") + ".index.json") + # Only fallback to pickle format if safetensors index is not found and safe is False. + if not index_path.is_file() and not safe: + filename_pattern = constants.PYTORCH_WEIGHTS_FILE_PATTERN + + index_path = checkpoint_path / (filename_pattern.format(suffix="") + ".index.json") + + if index_path.is_file(): + return _load_sharded_checkpoint( + model=model, + save_directory=checkpoint_path, + strict=strict, + weights_only=weights_only, + filename_pattern=filename_pattern, + ) + + # Look for single model file + model_files = list(checkpoint_path.glob("*.safetensors" if safe else "*.bin")) + if len(model_files) == 1: + state_dict = load_state_dict_from_file( + checkpoint_file=model_files[0], + map_location=map_location, + weights_only=weights_only, + mmap=mmap, + ) + return model.load_state_dict(state_dict, strict=strict) + + raise ValueError( + f"Directory '{checkpoint_path}' does not contain a valid checkpoint. " + "Expected either a sharded checkpoint with an index file, or a single model file." + ) + + +def _load_sharded_checkpoint( + model: "torch.nn.Module", + save_directory: os.PathLike, + *, + strict: bool = False, + weights_only: bool = False, + filename_pattern: str = constants.SAFETENSORS_WEIGHTS_FILE_PATTERN, +) -> NamedTuple: + """ + Loads a sharded checkpoint into a model. This is the same as + [`torch.nn.Module.load_state_dict`](https://pytorch.org/docs/stable/generated/torch.nn.Module.html?highlight=load_state_dict#torch.nn.Module.load_state_dict) + but for a sharded checkpoint. Each shard is loaded one by one and removed from memory after being loaded into the model. + + Args: + model (`torch.nn.Module`): + The model in which to load the checkpoint. + save_directory (`str` or `os.PathLike`): + A path to a folder containing the sharded checkpoint. + strict (`bool`, *optional*, defaults to `False`): + Whether to strictly enforce that the keys in the model state dict match the keys in the sharded checkpoint. + weights_only (`bool`, *optional*, defaults to `False`): + If True, only loads the model weights without optimizer states and other metadata. + Only supported in PyTorch >= 1.13. + filename_pattern (`str`, *optional*, defaults to `"model{suffix}.safetensors"`): + The pattern to look for the index file. Pattern must be a string that + can be formatted with `filename_pattern.format(suffix=...)` and must contain the keyword `suffix` + Defaults to `"model{suffix}.safetensors"`. + + Returns: + `NamedTuple`: A named tuple with `missing_keys` and `unexpected_keys` fields, + - `missing_keys` is a list of str containing the missing keys + - `unexpected_keys` is a list of str containing the unexpected keys + """ + + # 1. Load and validate index file + # The index file contains mapping of parameter names to shard files + index_path = filename_pattern.format(suffix="") + ".index.json" + index_file = os.path.join(save_directory, index_path) + with open(index_file, "r", encoding="utf-8") as f: + index = json.load(f) + + # 2. Validate keys if in strict mode + # This is done before loading any shards to fail fast + if strict: + _validate_keys_for_strict_loading(model, index["weight_map"].keys()) + + # 3. Load each shard using `load_state_dict` + # Get unique shard files (multiple parameters can be in same shard) + shard_files = list(set(index["weight_map"].values())) + for shard_file in shard_files: + # Load shard into memory + shard_path = os.path.join(save_directory, shard_file) + state_dict = load_state_dict_from_file( + shard_path, + map_location="cpu", + weights_only=weights_only, + ) + # Update model with parameters from this shard + model.load_state_dict(state_dict, strict=strict) + # Explicitly remove the state dict from memory + del state_dict + + # 4. Return compatibility info + loaded_keys = set(index["weight_map"].keys()) + model_keys = set(model.state_dict().keys()) + return _IncompatibleKeys( + missing_keys=list(model_keys - loaded_keys), unexpected_keys=list(loaded_keys - model_keys) + ) + + +def load_state_dict_from_file( + checkpoint_file: Union[str, os.PathLike], + map_location: Optional[Union[str, "torch.device"]] = None, + weights_only: bool = False, + mmap: bool = False, +) -> Union[Dict[str, "torch.Tensor"], Any]: + """ + Loads a checkpoint file, handling both safetensors and pickle checkpoint formats. + + Args: + checkpoint_file (`str` or `os.PathLike`): + Path to the checkpoint file to load. Can be either a safetensors or pickle (`.bin`) checkpoint. + map_location (`str` or `torch.device`, *optional*): + A `torch.device` object, string or a dict specifying how to remap storage locations. It + indicates the location where all tensors should be loaded. + weights_only (`bool`, *optional*, defaults to `False`): + If True, only loads the model weights without optimizer states and other metadata. + Only supported for pickle (`.bin`) checkpoints with PyTorch >= 1.13. Has no effect when + loading safetensors files. + mmap (`bool`, *optional*, defaults to `False`): + Whether to use memory-mapped file loading. Memory mapping can improve loading performance + for large models in PyTorch >= 2.1.0 with zipfile-based checkpoints. Has no effect when + loading safetensors files, as the `safetensors` library uses memory mapping by default. + + Returns: + `Union[Dict[str, "torch.Tensor"], Any]`: The loaded checkpoint. + - For safetensors files: always returns a dictionary mapping parameter names to tensors. + - For pickle files: returns any Python object that was pickled (commonly a state dict, but could be + an entire model, optimizer state, or any other Python object). + + Raises: + [`FileNotFoundError`](https://docs.python.org/3/library/exceptions.html#FileNotFoundError) + If the checkpoint file does not exist. + [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError) + If safetensors or torch is not installed when trying to load a .safetensors file or a PyTorch checkpoint respectively. + [`OSError`](https://docs.python.org/3/library/exceptions.html#OSError) + If the checkpoint file format is invalid or if git-lfs files are not properly downloaded. + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If the checkpoint file path is empty or invalid. + + Example: + ```python + >>> from huggingface_hub import load_state_dict_from_file + + # Load a PyTorch checkpoint + >>> state_dict = load_state_dict_from_file("path/to/model.bin", map_location="cpu") + >>> model.load_state_dict(state_dict) + + # Load a safetensors checkpoint + >>> state_dict = load_state_dict_from_file("path/to/model.safetensors") + >>> model.load_state_dict(state_dict) + ``` + """ + checkpoint_path = Path(checkpoint_file) + + # Check if file exists and is a regular file (not a directory) + if not checkpoint_path.is_file(): + raise FileNotFoundError( + f"No checkpoint file found at '{checkpoint_path}'. Please verify the path is correct and " + "the file has been properly downloaded." + ) + + # Load safetensors checkpoint + if checkpoint_path.suffix == ".safetensors": + try: + from safetensors import safe_open + from safetensors.torch import load_file + except ImportError as e: + raise ImportError( + "Please install `safetensors` to load safetensors checkpoint. " + "You can install it with `pip install safetensors`." + ) from e + + # Check format of the archive + with safe_open(checkpoint_file, framework="pt") as f: # type: ignore[attr-defined] + metadata = f.metadata() + # see comment: https://github.com/huggingface/transformers/blob/3d213b57fe74302e5902d68ed9478c3ad1aaa713/src/transformers/modeling_utils.py#L3966 + if metadata is not None and metadata.get("format") not in ["pt", "mlx"]: + raise OSError( + f"The safetensors archive passed at {checkpoint_file} does not contain the valid metadata. Make sure " + "you save your model with the `save_torch_model` method." + ) + device = str(map_location.type) if map_location is not None and hasattr(map_location, "type") else map_location + # meta device is not supported with safetensors, falling back to CPU + if device == "meta": + logger.warning("Meta device is not supported with safetensors. Falling back to CPU device.") + device = "cpu" + return load_file(checkpoint_file, device=device) # type: ignore[arg-type] + # Otherwise, load from pickle + try: + import torch + from torch import load + except ImportError as e: + raise ImportError( + "Please install `torch` to load torch tensors. You can install it with `pip install torch`." + ) from e + # Add additional kwargs, mmap is only supported in torch >= 2.1.0 + additional_kwargs = {} + if version.parse(torch.__version__) >= version.parse("2.1.0"): + additional_kwargs["mmap"] = mmap + + # weights_only is only supported in torch >= 1.13.0 + if version.parse(torch.__version__) >= version.parse("1.13.0"): + additional_kwargs["weights_only"] = weights_only + + return load( + checkpoint_file, + map_location=map_location, + **additional_kwargs, + ) + + +# HELPERS + + +def _validate_keys_for_strict_loading( + model: "torch.nn.Module", + loaded_keys: Iterable[str], +) -> None: + """ + Validate that model keys match loaded keys when strict loading is enabled. + + Args: + model: The PyTorch model being loaded + loaded_keys: The keys present in the checkpoint + + Raises: + RuntimeError: If there are missing or unexpected keys in strict mode + """ + loaded_keys_set = set(loaded_keys) + model_keys = set(model.state_dict().keys()) + missing_keys = model_keys - loaded_keys_set # Keys in model but not in checkpoint + unexpected_keys = loaded_keys_set - model_keys # Keys in checkpoint but not in model + + if missing_keys or unexpected_keys: + error_message = f"Error(s) in loading state_dict for {model.__class__.__name__}" + if missing_keys: + str_missing_keys = ",".join([f'"{k}"' for k in sorted(missing_keys)]) + error_message += f"\nMissing key(s): {str_missing_keys}." + if unexpected_keys: + str_unexpected_keys = ",".join([f'"{k}"' for k in sorted(unexpected_keys)]) + error_message += f"\nUnexpected key(s): {str_unexpected_keys}." + raise RuntimeError(error_message) + + +def _get_unique_id(tensor: "torch.Tensor") -> Union[int, Tuple[Any, ...]]: + """Returns a unique id for plain tensor + or a (potentially nested) Tuple of unique id for the flattened Tensor + if the input is a wrapper tensor subclass Tensor + """ + + try: + from torch.distributed.tensor import DTensor + + if isinstance(tensor, DTensor): + local_tensor = tensor.to_local() + return local_tensor.storage().data_ptr() + except ImportError: + pass + + try: + # for torch 2.1 and above we can also handle tensor subclasses + from torch.utils._python_dispatch import is_traceable_wrapper_subclass + + if is_traceable_wrapper_subclass(tensor): + attrs, _ = tensor.__tensor_flatten__() # type: ignore[attr-defined] + return tuple(_get_unique_id(getattr(tensor, attr)) for attr in attrs) + + except ImportError: + # for torch version less than 2.1, we can fallback to original implementation + pass + + if tensor.device.type == "xla" and is_torch_tpu_available(): + # NOTE: xla tensors dont have storage + # use some other unique id to distinguish. + # this is a XLA tensor, it must be created using torch_xla's + # device. So the following import is safe: + import torch_xla # type: ignore[import] + + unique_id = torch_xla._XLAC._xla_get_tensor_id(tensor) + else: + unique_id = storage_ptr(tensor) + + return unique_id + + +def get_torch_storage_id(tensor: "torch.Tensor") -> Optional[Tuple["torch.device", Union[int, Tuple[Any, ...]], int]]: + """ + Return unique identifier to a tensor storage. + + Multiple different tensors can share the same underlying storage. This identifier is + guaranteed to be unique and constant for this tensor's storage during its lifetime. Two tensor storages with + non-overlapping lifetimes may have the same id. + In the case of meta tensors, we return None since we can't tell if they share the same storage. + + Taken from https://github.com/huggingface/transformers/blob/1ecf5f7c982d761b4daaa96719d162c324187c64/src/transformers/pytorch_utils.py#L278. + """ + if tensor.device.type == "meta": + return None + else: + return tensor.device, _get_unique_id(tensor), get_torch_storage_size(tensor) + + +def get_torch_storage_size(tensor: "torch.Tensor") -> int: + """ + Taken from https://github.com/huggingface/safetensors/blob/08db34094e9e59e2f9218f2df133b7b4aaff5a99/bindings/python/py_src/safetensors/torch.py#L31C1-L41C59 + """ + try: + from torch.distributed.tensor import DTensor + + if isinstance(tensor, DTensor): + # this returns the size of the FULL tensor in bytes + return tensor.nbytes + except ImportError: + pass + + try: + # for torch 2.1 and above we can also handle tensor subclasses + from torch.utils._python_dispatch import is_traceable_wrapper_subclass + + if is_traceable_wrapper_subclass(tensor): + attrs, _ = tensor.__tensor_flatten__() # type: ignore[attr-defined] + return sum(get_torch_storage_size(getattr(tensor, attr)) for attr in attrs) + except ImportError: + # for torch version less than 2.1, we can fallback to original implementation + pass + + try: + return tensor.untyped_storage().nbytes() + except AttributeError: + # Fallback for torch==1.10 + try: + return tensor.storage().size() * _get_dtype_size(tensor.dtype) + except NotImplementedError: + # Fallback for meta storage + # On torch >=2.0 this is the tensor size + return tensor.nelement() * _get_dtype_size(tensor.dtype) + + +@lru_cache() +def is_torch_tpu_available(check_device=True): + """ + Checks if `torch_xla` is installed and potentially if a TPU is in the environment + + Taken from https://github.com/huggingface/transformers/blob/1ecf5f7c982d761b4daaa96719d162c324187c64/src/transformers/utils/import_utils.py#L463. + """ + if importlib.util.find_spec("torch_xla") is not None: + if check_device: + # We need to check if `xla_device` can be found, will raise a RuntimeError if not + try: + import torch_xla.core.xla_model as xm # type: ignore[import] + + _ = xm.xla_device() + return True + except RuntimeError: + return False + return True + return False + + +def storage_ptr(tensor: "torch.Tensor") -> Union[int, Tuple[Any, ...]]: + """ + Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L11. + """ + try: + # for torch 2.1 and above we can also handle tensor subclasses + from torch.utils._python_dispatch import is_traceable_wrapper_subclass + + if is_traceable_wrapper_subclass(tensor): + return _get_unique_id(tensor) # type: ignore + except ImportError: + # for torch version less than 2.1, we can fallback to original implementation + pass + + try: + return tensor.untyped_storage().data_ptr() + except Exception: + # Fallback for torch==1.10 + try: + return tensor.storage().data_ptr() + except NotImplementedError: + # Fallback for meta storage + return 0 + + +def _clean_state_dict_for_safetensors( + state_dict: Dict[str, "torch.Tensor"], + metadata: Dict[str, str], + force_contiguous: bool = True, + shared_tensors_to_discard: Optional[List[str]] = None, +): + """Remove shared tensors from state_dict and update metadata accordingly (for reloading). + + Warning: `state_dict` and `metadata` are mutated in-place! + + Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L155. + """ + to_removes = _remove_duplicate_names(state_dict, discard_names=shared_tensors_to_discard) + for kept_name, to_remove_group in to_removes.items(): + for to_remove in to_remove_group: + if metadata is None: + metadata = {} + + if to_remove not in metadata: + # Do not override user data + metadata[to_remove] = kept_name + del state_dict[to_remove] + if force_contiguous: + state_dict = {k: v.contiguous() for k, v in state_dict.items()} + return state_dict + + +def _end_ptr(tensor: "torch.Tensor") -> int: + """ + Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L23. + """ + if tensor.nelement(): + stop = tensor.view(-1)[-1].data_ptr() + _get_dtype_size(tensor.dtype) + else: + stop = tensor.data_ptr() + return stop + + +def _filter_shared_not_shared(tensors: List[Set[str]], state_dict: Dict[str, "torch.Tensor"]) -> List[Set[str]]: + """ + Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L44 + """ + filtered_tensors = [] + for shared in tensors: + if len(shared) < 2: + filtered_tensors.append(shared) + continue + + areas = [] + for name in shared: + tensor = state_dict[name] + areas.append((tensor.data_ptr(), _end_ptr(tensor), name)) + areas.sort() + + _, last_stop, last_name = areas[0] + filtered_tensors.append({last_name}) + for start, stop, name in areas[1:]: + if start >= last_stop: + filtered_tensors.append({name}) + else: + filtered_tensors[-1].add(name) + last_stop = stop + + return filtered_tensors + + +def _find_shared_tensors(state_dict: Dict[str, "torch.Tensor"]) -> List[Set[str]]: + """ + Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L69. + """ + import torch + + tensors_dict = defaultdict(set) + for k, v in state_dict.items(): + if v.device != torch.device("meta") and storage_ptr(v) != 0 and get_torch_storage_size(v) != 0: + # Need to add device as key because of multiple GPU. + tensors_dict[(v.device, storage_ptr(v), get_torch_storage_size(v))].add(k) + tensors = list(sorted(tensors_dict.values())) + tensors = _filter_shared_not_shared(tensors, state_dict) + return tensors + + +def _is_complete(tensor: "torch.Tensor") -> bool: + """ + Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L80 + """ + try: + # for torch 2.1 and above we can also handle tensor subclasses + from torch.utils._python_dispatch import is_traceable_wrapper_subclass + + if is_traceable_wrapper_subclass(tensor): + attrs, _ = tensor.__tensor_flatten__() # type: ignore[attr-defined] + return all(_is_complete(getattr(tensor, attr)) for attr in attrs) + except ImportError: + # for torch version less than 2.1, we can fallback to original implementation + pass + + return tensor.data_ptr() == storage_ptr(tensor) and tensor.nelement() * _get_dtype_size( + tensor.dtype + ) == get_torch_storage_size(tensor) + + +def _remove_duplicate_names( + state_dict: Dict[str, "torch.Tensor"], + *, + preferred_names: Optional[List[str]] = None, + discard_names: Optional[List[str]] = None, +) -> Dict[str, List[str]]: + """ + Taken from https://github.com/huggingface/safetensors/blob/079781fd0dc455ba0fe851e2b4507c33d0c0d407/bindings/python/py_src/safetensors/torch.py#L80 + """ + if preferred_names is None: + preferred_names = [] + unique_preferred_names = set(preferred_names) + if discard_names is None: + discard_names = [] + unique_discard_names = set(discard_names) + + shareds = _find_shared_tensors(state_dict) + to_remove = defaultdict(list) + for shared in shareds: + complete_names = set([name for name in shared if _is_complete(state_dict[name])]) + if not complete_names: + raise RuntimeError( + "Error while trying to find names to remove to save state dict, but found no suitable name to keep" + f" for saving amongst: {shared}. None is covering the entire storage. Refusing to save/load the model" + " since you could be storing much more memory than needed. Please refer to" + " https://huggingface.co/docs/safetensors/torch_shared_tensors for more information. Or open an" + " issue." + ) + + keep_name = sorted(list(complete_names))[0] + + # Mechanism to preferentially select keys to keep + # coming from the on-disk file to allow + # loading models saved with a different choice + # of keep_name + preferred = complete_names.difference(unique_discard_names) + if preferred: + keep_name = sorted(list(preferred))[0] + + if unique_preferred_names: + preferred = unique_preferred_names.intersection(complete_names) + if preferred: + keep_name = sorted(list(preferred))[0] + for name in sorted(shared): + if name != keep_name: + to_remove[keep_name].append(name) + return to_remove + + +@lru_cache() +def _get_dtype_size(dtype: "torch.dtype") -> int: + """ + Taken from https://github.com/huggingface/safetensors/blob/08db34094e9e59e2f9218f2df133b7b4aaff5a99/bindings/python/py_src/safetensors/torch.py#L344 + """ + import torch + + # torch.float8 formats require 2.1; we do not support these dtypes on earlier versions + _float8_e4m3fn = getattr(torch, "float8_e4m3fn", None) + _float8_e5m2 = getattr(torch, "float8_e5m2", None) + _SIZE = { + torch.int64: 8, + torch.float32: 4, + torch.int32: 4, + torch.bfloat16: 2, + torch.float16: 2, + torch.int16: 2, + torch.uint8: 1, + torch.int8: 1, + torch.bool: 1, + torch.float64: 8, + _float8_e4m3fn: 1, + _float8_e5m2: 1, + } + return _SIZE[dtype] + + +class _IncompatibleKeys(namedtuple("IncompatibleKeys", ["missing_keys", "unexpected_keys"])): + """ + This is used to report missing and unexpected keys in the state dict. + Taken from https://github.com/pytorch/pytorch/blob/main/torch/nn/modules/module.py#L52. + + """ + + def __repr__(self) -> str: + if not self.missing_keys and not self.unexpected_keys: + return "" + return super().__repr__() + + __str__ = __repr__ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_cache_assets.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_cache_assets.py new file mode 100644 index 0000000000000000000000000000000000000000..e5d435df9b0bb0c67c0bcb5ef65711e9aef367f6 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_cache_assets.py @@ -0,0 +1,135 @@ +# coding=utf-8 +# Copyright 2019-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from pathlib import Path +from typing import Union + +from ..constants import HF_ASSETS_CACHE + + +def cached_assets_path( + library_name: str, + namespace: str = "default", + subfolder: str = "default", + *, + assets_dir: Union[str, Path, None] = None, +): + """Return a folder path to cache arbitrary files. + + `huggingface_hub` provides a canonical folder path to store assets. This is the + recommended way to integrate cache in a downstream library as it will benefit from + the builtins tools to scan and delete the cache properly. + + The distinction is made between files cached from the Hub and assets. Files from the + Hub are cached in a git-aware manner and entirely managed by `huggingface_hub`. See + [related documentation](https://huggingface.co/docs/huggingface_hub/how-to-cache). + All other files that a downstream library caches are considered to be "assets" + (files downloaded from external sources, extracted from a .tar archive, preprocessed + for training,...). + + Once the folder path is generated, it is guaranteed to exist and to be a directory. + The path is based on 3 levels of depth: the library name, a namespace and a + subfolder. Those 3 levels grants flexibility while allowing `huggingface_hub` to + expect folders when scanning/deleting parts of the assets cache. Within a library, + it is expected that all namespaces share the same subset of subfolder names but this + is not a mandatory rule. The downstream library has then full control on which file + structure to adopt within its cache. Namespace and subfolder are optional (would + default to a `"default/"` subfolder) but library name is mandatory as we want every + downstream library to manage its own cache. + + Expected tree: + ```text + assets/ + └── datasets/ + │ ├── SQuAD/ + │ │ ├── downloaded/ + │ │ ├── extracted/ + │ │ └── processed/ + │ ├── Helsinki-NLP--tatoeba_mt/ + │ ├── downloaded/ + │ ├── extracted/ + │ └── processed/ + └── transformers/ + ├── default/ + │ ├── something/ + ├── bert-base-cased/ + │ ├── default/ + │ └── training/ + hub/ + └── models--julien-c--EsperBERTo-small/ + ├── blobs/ + │ ├── (...) + │ ├── (...) + ├── refs/ + │ └── (...) + └── [ 128] snapshots/ + ├── 2439f60ef33a0d46d85da5001d52aeda5b00ce9f/ + │ ├── (...) + └── bbc77c8132af1cc5cf678da3f1ddf2de43606d48/ + └── (...) + ``` + + + Args: + library_name (`str`): + Name of the library that will manage the cache folder. Example: `"dataset"`. + namespace (`str`, *optional*, defaults to "default"): + Namespace to which the data belongs. Example: `"SQuAD"`. + subfolder (`str`, *optional*, defaults to "default"): + Subfolder in which the data will be stored. Example: `extracted`. + assets_dir (`str`, `Path`, *optional*): + Path to the folder where assets are cached. This must not be the same folder + where Hub files are cached. Defaults to `HF_HOME / "assets"` if not provided. + Can also be set with `HF_ASSETS_CACHE` environment variable. + + Returns: + Path to the cache folder (`Path`). + + Example: + ```py + >>> from huggingface_hub import cached_assets_path + + >>> cached_assets_path(library_name="datasets", namespace="SQuAD", subfolder="download") + PosixPath('/home/wauplin/.cache/huggingface/extra/datasets/SQuAD/download') + + >>> cached_assets_path(library_name="datasets", namespace="SQuAD", subfolder="extracted") + PosixPath('/home/wauplin/.cache/huggingface/extra/datasets/SQuAD/extracted') + + >>> cached_assets_path(library_name="datasets", namespace="Helsinki-NLP/tatoeba_mt") + PosixPath('/home/wauplin/.cache/huggingface/extra/datasets/Helsinki-NLP--tatoeba_mt/default') + + >>> cached_assets_path(library_name="datasets", assets_dir="/tmp/tmp123456") + PosixPath('/tmp/tmp123456/datasets/default/default') + ``` + """ + # Resolve assets_dir + if assets_dir is None: + assets_dir = HF_ASSETS_CACHE + assets_dir = Path(assets_dir).expanduser().resolve() + + # Avoid names that could create path issues + for part in (" ", "/", "\\"): + library_name = library_name.replace(part, "--") + namespace = namespace.replace(part, "--") + subfolder = subfolder.replace(part, "--") + + # Path to subfolder is created + path = assets_dir / library_name / namespace / subfolder + try: + path.mkdir(exist_ok=True, parents=True) + except (FileExistsError, NotADirectoryError): + raise ValueError(f"Corrupted assets folder: cannot create directory because of an existing file ({path}).") + + # Return + return path diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_cache_manager.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_cache_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..90d0e01f74812c5c3e65ba9313a155ee8e517927 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_cache_manager.py @@ -0,0 +1,866 @@ +# coding=utf-8 +# Copyright 2022-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains utilities to manage the HF cache directory.""" + +import os +import shutil +import time +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, FrozenSet, List, Literal, Optional, Set, Union + +from huggingface_hub.errors import CacheNotFound, CorruptedCacheException + +from ..commands._cli_utils import tabulate +from ..constants import HF_HUB_CACHE +from . import logging + + +logger = logging.get_logger(__name__) + +REPO_TYPE_T = Literal["model", "dataset", "space"] + +# List of OS-created helper files that need to be ignored +FILES_TO_IGNORE = [".DS_Store"] + + +@dataclass(frozen=True) +class CachedFileInfo: + """Frozen data structure holding information about a single cached file. + + Args: + file_name (`str`): + Name of the file. Example: `config.json`. + file_path (`Path`): + Path of the file in the `snapshots` directory. The file path is a symlink + referring to a blob in the `blobs` folder. + blob_path (`Path`): + Path of the blob file. This is equivalent to `file_path.resolve()`. + size_on_disk (`int`): + Size of the blob file in bytes. + blob_last_accessed (`float`): + Timestamp of the last time the blob file has been accessed (from any + revision). + blob_last_modified (`float`): + Timestamp of the last time the blob file has been modified/created. + + > [!WARNING] + > `blob_last_accessed` and `blob_last_modified` reliability can depend on the OS you + > are using. See [python documentation](https://docs.python.org/3/library/os.html#os.stat_result) + > for more details. + """ + + file_name: str + file_path: Path + blob_path: Path + size_on_disk: int + + blob_last_accessed: float + blob_last_modified: float + + @property + def blob_last_accessed_str(self) -> str: + """ + (property) Timestamp of the last time the blob file has been accessed (from any + revision), returned as a human-readable string. + + Example: "2 weeks ago". + """ + return _format_timesince(self.blob_last_accessed) + + @property + def blob_last_modified_str(self) -> str: + """ + (property) Timestamp of the last time the blob file has been modified, returned + as a human-readable string. + + Example: "2 weeks ago". + """ + return _format_timesince(self.blob_last_modified) + + @property + def size_on_disk_str(self) -> str: + """ + (property) Size of the blob file as a human-readable string. + + Example: "42.2K". + """ + return _format_size(self.size_on_disk) + + +@dataclass(frozen=True) +class CachedRevisionInfo: + """Frozen data structure holding information about a revision. + + A revision correspond to a folder in the `snapshots` folder and is populated with + the exact tree structure as the repo on the Hub but contains only symlinks. A + revision can be either referenced by 1 or more `refs` or be "detached" (no refs). + + Args: + commit_hash (`str`): + Hash of the revision (unique). + Example: `"9338f7b671827df886678df2bdd7cc7b4f36dffd"`. + snapshot_path (`Path`): + Path to the revision directory in the `snapshots` folder. It contains the + exact tree structure as the repo on the Hub. + files: (`FrozenSet[CachedFileInfo]`): + Set of [`~CachedFileInfo`] describing all files contained in the snapshot. + refs (`FrozenSet[str]`): + Set of `refs` pointing to this revision. If the revision has no `refs`, it + is considered detached. + Example: `{"main", "2.4.0"}` or `{"refs/pr/1"}`. + size_on_disk (`int`): + Sum of the blob file sizes that are symlink-ed by the revision. + last_modified (`float`): + Timestamp of the last time the revision has been created/modified. + + > [!WARNING] + > `last_accessed` cannot be determined correctly on a single revision as blob files + > are shared across revisions. + + > [!WARNING] + > `size_on_disk` is not necessarily the sum of all file sizes because of possible + > duplicated files. Besides, only blobs are taken into account, not the (negligible) + > size of folders and symlinks. + """ + + commit_hash: str + snapshot_path: Path + size_on_disk: int + files: FrozenSet[CachedFileInfo] + refs: FrozenSet[str] + + last_modified: float + + @property + def last_modified_str(self) -> str: + """ + (property) Timestamp of the last time the revision has been modified, returned + as a human-readable string. + + Example: "2 weeks ago". + """ + return _format_timesince(self.last_modified) + + @property + def size_on_disk_str(self) -> str: + """ + (property) Sum of the blob file sizes as a human-readable string. + + Example: "42.2K". + """ + return _format_size(self.size_on_disk) + + @property + def nb_files(self) -> int: + """ + (property) Total number of files in the revision. + """ + return len(self.files) + + +@dataclass(frozen=True) +class CachedRepoInfo: + """Frozen data structure holding information about a cached repository. + + Args: + repo_id (`str`): + Repo id of the repo on the Hub. Example: `"google/fleurs"`. + repo_type (`Literal["dataset", "model", "space"]`): + Type of the cached repo. + repo_path (`Path`): + Local path to the cached repo. + size_on_disk (`int`): + Sum of the blob file sizes in the cached repo. + nb_files (`int`): + Total number of blob files in the cached repo. + revisions (`FrozenSet[CachedRevisionInfo]`): + Set of [`~CachedRevisionInfo`] describing all revisions cached in the repo. + last_accessed (`float`): + Timestamp of the last time a blob file of the repo has been accessed. + last_modified (`float`): + Timestamp of the last time a blob file of the repo has been modified/created. + + > [!WARNING] + > `size_on_disk` is not necessarily the sum of all revisions sizes because of + > duplicated files. Besides, only blobs are taken into account, not the (negligible) + > size of folders and symlinks. + + > [!WARNING] + > `last_accessed` and `last_modified` reliability can depend on the OS you are using. + > See [python documentation](https://docs.python.org/3/library/os.html#os.stat_result) + > for more details. + """ + + repo_id: str + repo_type: REPO_TYPE_T + repo_path: Path + size_on_disk: int + nb_files: int + revisions: FrozenSet[CachedRevisionInfo] + + last_accessed: float + last_modified: float + + @property + def last_accessed_str(self) -> str: + """ + (property) Last time a blob file of the repo has been accessed, returned as a + human-readable string. + + Example: "2 weeks ago". + """ + return _format_timesince(self.last_accessed) + + @property + def last_modified_str(self) -> str: + """ + (property) Last time a blob file of the repo has been modified, returned as a + human-readable string. + + Example: "2 weeks ago". + """ + return _format_timesince(self.last_modified) + + @property + def size_on_disk_str(self) -> str: + """ + (property) Sum of the blob file sizes as a human-readable string. + + Example: "42.2K". + """ + return _format_size(self.size_on_disk) + + @property + def refs(self) -> Dict[str, CachedRevisionInfo]: + """ + (property) Mapping between `refs` and revision data structures. + """ + return {ref: revision for revision in self.revisions for ref in revision.refs} + + +@dataclass(frozen=True) +class DeleteCacheStrategy: + """Frozen data structure holding the strategy to delete cached revisions. + + This object is not meant to be instantiated programmatically but to be returned by + [`~utils.HFCacheInfo.delete_revisions`]. See documentation for usage example. + + Args: + expected_freed_size (`float`): + Expected freed size once strategy is executed. + blobs (`FrozenSet[Path]`): + Set of blob file paths to be deleted. + refs (`FrozenSet[Path]`): + Set of reference file paths to be deleted. + repos (`FrozenSet[Path]`): + Set of entire repo paths to be deleted. + snapshots (`FrozenSet[Path]`): + Set of snapshots to be deleted (directory of symlinks). + """ + + expected_freed_size: int + blobs: FrozenSet[Path] + refs: FrozenSet[Path] + repos: FrozenSet[Path] + snapshots: FrozenSet[Path] + + @property + def expected_freed_size_str(self) -> str: + """ + (property) Expected size that will be freed as a human-readable string. + + Example: "42.2K". + """ + return _format_size(self.expected_freed_size) + + def execute(self) -> None: + """Execute the defined strategy. + + > [!WARNING] + > If this method is interrupted, the cache might get corrupted. Deletion order is + > implemented so that references and symlinks are deleted before the actual blob + > files. + + > [!WARNING] + > This method is irreversible. If executed, cached files are erased and must be + > downloaded again. + """ + # Deletion order matters. Blobs are deleted in last so that the user can't end + # up in a state where a `ref`` refers to a missing snapshot or a snapshot + # symlink refers to a deleted blob. + + # Delete entire repos + for path in self.repos: + _try_delete_path(path, path_type="repo") + + # Delete snapshot directories + for path in self.snapshots: + _try_delete_path(path, path_type="snapshot") + + # Delete refs files + for path in self.refs: + _try_delete_path(path, path_type="ref") + + # Delete blob files + for path in self.blobs: + _try_delete_path(path, path_type="blob") + + logger.info(f"Cache deletion done. Saved {self.expected_freed_size_str}.") + + +@dataclass(frozen=True) +class HFCacheInfo: + """Frozen data structure holding information about the entire cache-system. + + This data structure is returned by [`scan_cache_dir`] and is immutable. + + Args: + size_on_disk (`int`): + Sum of all valid repo sizes in the cache-system. + repos (`FrozenSet[CachedRepoInfo]`): + Set of [`~CachedRepoInfo`] describing all valid cached repos found on the + cache-system while scanning. + warnings (`List[CorruptedCacheException]`): + List of [`~CorruptedCacheException`] that occurred while scanning the cache. + Those exceptions are captured so that the scan can continue. Corrupted repos + are skipped from the scan. + + > [!WARNING] + > Here `size_on_disk` is equal to the sum of all repo sizes (only blobs). However if + > some cached repos are corrupted, their sizes are not taken into account. + """ + + size_on_disk: int + repos: FrozenSet[CachedRepoInfo] + warnings: List[CorruptedCacheException] + + @property + def size_on_disk_str(self) -> str: + """ + (property) Sum of all valid repo sizes in the cache-system as a human-readable + string. + + Example: "42.2K". + """ + return _format_size(self.size_on_disk) + + def delete_revisions(self, *revisions: str) -> DeleteCacheStrategy: + """Prepare the strategy to delete one or more revisions cached locally. + + Input revisions can be any revision hash. If a revision hash is not found in the + local cache, a warning is thrown but no error is raised. Revisions can be from + different cached repos since hashes are unique across repos, + + Examples: + ```py + >>> from huggingface_hub import scan_cache_dir + >>> cache_info = scan_cache_dir() + >>> delete_strategy = cache_info.delete_revisions( + ... "81fd1d6e7847c99f5862c9fb81387956d99ec7aa" + ... ) + >>> print(f"Will free {delete_strategy.expected_freed_size_str}.") + Will free 7.9K. + >>> delete_strategy.execute() + Cache deletion done. Saved 7.9K. + ``` + + ```py + >>> from huggingface_hub import scan_cache_dir + >>> scan_cache_dir().delete_revisions( + ... "81fd1d6e7847c99f5862c9fb81387956d99ec7aa", + ... "e2983b237dccf3ab4937c97fa717319a9ca1a96d", + ... "6c0e6080953db56375760c0471a8c5f2929baf11", + ... ).execute() + Cache deletion done. Saved 8.6G. + ``` + + > [!WARNING] + > `delete_revisions` returns a [`~utils.DeleteCacheStrategy`] object that needs to + > be executed. The [`~utils.DeleteCacheStrategy`] is not meant to be modified but + > allows having a dry run before actually executing the deletion. + """ + hashes_to_delete: Set[str] = set(revisions) + + repos_with_revisions: Dict[CachedRepoInfo, Set[CachedRevisionInfo]] = defaultdict(set) + + for repo in self.repos: + for revision in repo.revisions: + if revision.commit_hash in hashes_to_delete: + repos_with_revisions[repo].add(revision) + hashes_to_delete.remove(revision.commit_hash) + + if len(hashes_to_delete) > 0: + logger.warning(f"Revision(s) not found - cannot delete them: {', '.join(hashes_to_delete)}") + + delete_strategy_blobs: Set[Path] = set() + delete_strategy_refs: Set[Path] = set() + delete_strategy_repos: Set[Path] = set() + delete_strategy_snapshots: Set[Path] = set() + delete_strategy_expected_freed_size = 0 + + for affected_repo, revisions_to_delete in repos_with_revisions.items(): + other_revisions = affected_repo.revisions - revisions_to_delete + + # If no other revisions, it means all revisions are deleted + # -> delete the entire cached repo + if len(other_revisions) == 0: + delete_strategy_repos.add(affected_repo.repo_path) + delete_strategy_expected_freed_size += affected_repo.size_on_disk + continue + + # Some revisions of the repo will be deleted but not all. We need to filter + # which blob files will not be linked anymore. + for revision_to_delete in revisions_to_delete: + # Snapshot dir + delete_strategy_snapshots.add(revision_to_delete.snapshot_path) + + # Refs dir + for ref in revision_to_delete.refs: + delete_strategy_refs.add(affected_repo.repo_path / "refs" / ref) + + # Blobs dir + for file in revision_to_delete.files: + if file.blob_path not in delete_strategy_blobs: + is_file_alone = True + for revision in other_revisions: + for rev_file in revision.files: + if file.blob_path == rev_file.blob_path: + is_file_alone = False + break + if not is_file_alone: + break + + # Blob file not referenced by remaining revisions -> delete + if is_file_alone: + delete_strategy_blobs.add(file.blob_path) + delete_strategy_expected_freed_size += file.size_on_disk + + # Return the strategy instead of executing it. + return DeleteCacheStrategy( + blobs=frozenset(delete_strategy_blobs), + refs=frozenset(delete_strategy_refs), + repos=frozenset(delete_strategy_repos), + snapshots=frozenset(delete_strategy_snapshots), + expected_freed_size=delete_strategy_expected_freed_size, + ) + + def export_as_table(self, *, verbosity: int = 0) -> str: + """Generate a table from the [`HFCacheInfo`] object. + + Pass `verbosity=0` to get a table with a single row per repo, with columns + "repo_id", "repo_type", "size_on_disk", "nb_files", "last_accessed", "last_modified", "refs", "local_path". + + Pass `verbosity=1` to get a table with a row per repo and revision (thus multiple rows can appear for a single repo), with columns + "repo_id", "repo_type", "revision", "size_on_disk", "nb_files", "last_modified", "refs", "local_path". + + Example: + ```py + >>> from huggingface_hub.utils import scan_cache_dir + + >>> hf_cache_info = scan_cache_dir() + HFCacheInfo(...) + + >>> print(hf_cache_info.export_as_table()) + REPO ID REPO TYPE SIZE ON DISK NB FILES LAST_ACCESSED LAST_MODIFIED REFS LOCAL PATH + --------------------------------------------------- --------- ------------ -------- ------------- ------------- ---- -------------------------------------------------------------------------------------------------- + roberta-base model 2.7M 5 1 day ago 1 week ago main ~/.cache/huggingface/hub/models--roberta-base + suno/bark model 8.8K 1 1 week ago 1 week ago main ~/.cache/huggingface/hub/models--suno--bark + t5-base model 893.8M 4 4 days ago 7 months ago main ~/.cache/huggingface/hub/models--t5-base + t5-large model 3.0G 4 5 weeks ago 5 months ago main ~/.cache/huggingface/hub/models--t5-large + + >>> print(hf_cache_info.export_as_table(verbosity=1)) + REPO ID REPO TYPE REVISION SIZE ON DISK NB FILES LAST_MODIFIED REFS LOCAL PATH + --------------------------------------------------- --------- ---------------------------------------- ------------ -------- ------------- ---- ----------------------------------------------------------------------------------------------------------------------------------------------------- + roberta-base model e2da8e2f811d1448a5b465c236feacd80ffbac7b 2.7M 5 1 week ago main ~/.cache/huggingface/hub/models--roberta-base/snapshots/e2da8e2f811d1448a5b465c236feacd80ffbac7b + suno/bark model 70a8a7d34168586dc5d028fa9666aceade177992 8.8K 1 1 week ago main ~/.cache/huggingface/hub/models--suno--bark/snapshots/70a8a7d34168586dc5d028fa9666aceade177992 + t5-base model a9723ea7f1b39c1eae772870f3b547bf6ef7e6c1 893.8M 4 7 months ago main ~/.cache/huggingface/hub/models--t5-base/snapshots/a9723ea7f1b39c1eae772870f3b547bf6ef7e6c1 + t5-large model 150ebc2c4b72291e770f58e6057481c8d2ed331a 3.0G 4 5 months ago main ~/.cache/huggingface/hub/models--t5-large/snapshots/150ebc2c4b72291e770f58e6057481c8d2ed331a + ``` + + Args: + verbosity (`int`, *optional*): + The verbosity level. Defaults to 0. + + Returns: + `str`: The table as a string. + """ + if verbosity == 0: + return tabulate( + rows=[ + [ + repo.repo_id, + repo.repo_type, + "{:>12}".format(repo.size_on_disk_str), + repo.nb_files, + repo.last_accessed_str, + repo.last_modified_str, + ", ".join(sorted(repo.refs)), + str(repo.repo_path), + ] + for repo in sorted(self.repos, key=lambda repo: repo.repo_path) + ], + headers=[ + "REPO ID", + "REPO TYPE", + "SIZE ON DISK", + "NB FILES", + "LAST_ACCESSED", + "LAST_MODIFIED", + "REFS", + "LOCAL PATH", + ], + ) + else: + return tabulate( + rows=[ + [ + repo.repo_id, + repo.repo_type, + revision.commit_hash, + "{:>12}".format(revision.size_on_disk_str), + revision.nb_files, + revision.last_modified_str, + ", ".join(sorted(revision.refs)), + str(revision.snapshot_path), + ] + for repo in sorted(self.repos, key=lambda repo: repo.repo_path) + for revision in sorted(repo.revisions, key=lambda revision: revision.commit_hash) + ], + headers=[ + "REPO ID", + "REPO TYPE", + "REVISION", + "SIZE ON DISK", + "NB FILES", + "LAST_MODIFIED", + "REFS", + "LOCAL PATH", + ], + ) + + +def scan_cache_dir(cache_dir: Optional[Union[str, Path]] = None) -> HFCacheInfo: + """Scan the entire HF cache-system and return a [`~HFCacheInfo`] structure. + + Use `scan_cache_dir` in order to programmatically scan your cache-system. The cache + will be scanned repo by repo. If a repo is corrupted, a [`~CorruptedCacheException`] + will be thrown internally but captured and returned in the [`~HFCacheInfo`] + structure. Only valid repos get a proper report. + + ```py + >>> from huggingface_hub import scan_cache_dir + + >>> hf_cache_info = scan_cache_dir() + HFCacheInfo( + size_on_disk=3398085269, + repos=frozenset({ + CachedRepoInfo( + repo_id='t5-small', + repo_type='model', + repo_path=PosixPath(...), + size_on_disk=970726914, + nb_files=11, + revisions=frozenset({ + CachedRevisionInfo( + commit_hash='d78aea13fa7ecd06c29e3e46195d6341255065d5', + size_on_disk=970726339, + snapshot_path=PosixPath(...), + files=frozenset({ + CachedFileInfo( + file_name='config.json', + size_on_disk=1197 + file_path=PosixPath(...), + blob_path=PosixPath(...), + ), + CachedFileInfo(...), + ... + }), + ), + CachedRevisionInfo(...), + ... + }), + ), + CachedRepoInfo(...), + ... + }), + warnings=[ + CorruptedCacheException("Snapshots dir doesn't exist in cached repo: ..."), + CorruptedCacheException(...), + ... + ], + ) + ``` + + You can also print a detailed report directly from the `hf` command line using: + ```text + > hf cache scan + REPO ID REPO TYPE SIZE ON DISK NB FILES REFS LOCAL PATH + --------------------------- --------- ------------ -------- ------------------- ------------------------------------------------------------------------- + glue dataset 116.3K 15 1.17.0, main, 2.4.0 /Users/lucain/.cache/huggingface/hub/datasets--glue + google/fleurs dataset 64.9M 6 main, refs/pr/1 /Users/lucain/.cache/huggingface/hub/datasets--google--fleurs + Jean-Baptiste/camembert-ner model 441.0M 7 main /Users/lucain/.cache/huggingface/hub/models--Jean-Baptiste--camembert-ner + bert-base-cased model 1.9G 13 main /Users/lucain/.cache/huggingface/hub/models--bert-base-cased + t5-base model 10.1K 3 main /Users/lucain/.cache/huggingface/hub/models--t5-base + t5-small model 970.7M 11 refs/pr/1, main /Users/lucain/.cache/huggingface/hub/models--t5-small + + Done in 0.0s. Scanned 6 repo(s) for a total of 3.4G. + Got 1 warning(s) while scanning. Use -vvv to print details. + ``` + + Args: + cache_dir (`str` or `Path`, `optional`): + Cache directory to cache. Defaults to the default HF cache directory. + + > [!WARNING] + > Raises: + > + > `CacheNotFound` + > If the cache directory does not exist. + > + > [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + > If the cache directory is a file, instead of a directory. + + Returns: a [`~HFCacheInfo`] object. + """ + if cache_dir is None: + cache_dir = HF_HUB_CACHE + + cache_dir = Path(cache_dir).expanduser().resolve() + if not cache_dir.exists(): + raise CacheNotFound( + f"Cache directory not found: {cache_dir}. Please use `cache_dir` argument or set `HF_HUB_CACHE` environment variable.", + cache_dir=cache_dir, + ) + + if cache_dir.is_file(): + raise ValueError( + f"Scan cache expects a directory but found a file: {cache_dir}. Please use `cache_dir` argument or set `HF_HUB_CACHE` environment variable." + ) + + repos: Set[CachedRepoInfo] = set() + warnings: List[CorruptedCacheException] = [] + for repo_path in cache_dir.iterdir(): + if repo_path.name == ".locks": # skip './.locks/' folder + continue + try: + repos.add(_scan_cached_repo(repo_path)) + except CorruptedCacheException as e: + warnings.append(e) + + return HFCacheInfo( + repos=frozenset(repos), + size_on_disk=sum(repo.size_on_disk for repo in repos), + warnings=warnings, + ) + + +def _scan_cached_repo(repo_path: Path) -> CachedRepoInfo: + """Scan a single cache repo and return information about it. + + Any unexpected behavior will raise a [`~CorruptedCacheException`]. + """ + if not repo_path.is_dir(): + raise CorruptedCacheException(f"Repo path is not a directory: {repo_path}") + + if "--" not in repo_path.name: + raise CorruptedCacheException(f"Repo path is not a valid HuggingFace cache directory: {repo_path}") + + repo_type, repo_id = repo_path.name.split("--", maxsplit=1) + repo_type = repo_type[:-1] # "models" -> "model" + repo_id = repo_id.replace("--", "/") # google/fleurs -> "google/fleurs" + + if repo_type not in {"dataset", "model", "space"}: + raise CorruptedCacheException( + f"Repo type must be `dataset`, `model` or `space`, found `{repo_type}` ({repo_path})." + ) + + blob_stats: Dict[Path, os.stat_result] = {} # Key is blob_path, value is blob stats + + snapshots_path = repo_path / "snapshots" + refs_path = repo_path / "refs" + + if not snapshots_path.exists() or not snapshots_path.is_dir(): + raise CorruptedCacheException(f"Snapshots dir doesn't exist in cached repo: {snapshots_path}") + + # Scan over `refs` directory + + # key is revision hash, value is set of refs + refs_by_hash: Dict[str, Set[str]] = defaultdict(set) + if refs_path.exists(): + # Example of `refs` directory + # ── refs + # ├── main + # └── refs + # └── pr + # └── 1 + if refs_path.is_file(): + raise CorruptedCacheException(f"Refs directory cannot be a file: {refs_path}") + + for ref_path in refs_path.glob("**/*"): + # glob("**/*") iterates over all files and directories -> skip directories + if ref_path.is_dir() or ref_path.name in FILES_TO_IGNORE: + continue + + ref_name = str(ref_path.relative_to(refs_path)) + with ref_path.open() as f: + commit_hash = f.read() + + refs_by_hash[commit_hash].add(ref_name) + + # Scan snapshots directory + cached_revisions: Set[CachedRevisionInfo] = set() + for revision_path in snapshots_path.iterdir(): + # Ignore OS-created helper files + if revision_path.name in FILES_TO_IGNORE: + continue + if revision_path.is_file(): + raise CorruptedCacheException(f"Snapshots folder corrupted. Found a file: {revision_path}") + + cached_files = set() + for file_path in revision_path.glob("**/*"): + # glob("**/*") iterates over all files and directories -> skip directories + if file_path.is_dir(): + continue + + blob_path = Path(file_path).resolve() + if not blob_path.exists(): + raise CorruptedCacheException(f"Blob missing (broken symlink): {blob_path}") + + if blob_path not in blob_stats: + blob_stats[blob_path] = blob_path.stat() + + cached_files.add( + CachedFileInfo( + file_name=file_path.name, + file_path=file_path, + size_on_disk=blob_stats[blob_path].st_size, + blob_path=blob_path, + blob_last_accessed=blob_stats[blob_path].st_atime, + blob_last_modified=blob_stats[blob_path].st_mtime, + ) + ) + + # Last modified is either the last modified blob file or the revision folder + # itself if it is empty + if len(cached_files) > 0: + revision_last_modified = max(blob_stats[file.blob_path].st_mtime for file in cached_files) + else: + revision_last_modified = revision_path.stat().st_mtime + + cached_revisions.add( + CachedRevisionInfo( + commit_hash=revision_path.name, + files=frozenset(cached_files), + refs=frozenset(refs_by_hash.pop(revision_path.name, set())), + size_on_disk=sum( + blob_stats[blob_path].st_size for blob_path in set(file.blob_path for file in cached_files) + ), + snapshot_path=revision_path, + last_modified=revision_last_modified, + ) + ) + + # Check that all refs referred to an existing revision + if len(refs_by_hash) > 0: + raise CorruptedCacheException( + f"Reference(s) refer to missing commit hashes: {dict(refs_by_hash)} ({repo_path})." + ) + + # Last modified is either the last modified blob file or the repo folder itself if + # no blob files has been found. Same for last accessed. + if len(blob_stats) > 0: + repo_last_accessed = max(stat.st_atime for stat in blob_stats.values()) + repo_last_modified = max(stat.st_mtime for stat in blob_stats.values()) + else: + repo_stats = repo_path.stat() + repo_last_accessed = repo_stats.st_atime + repo_last_modified = repo_stats.st_mtime + + # Build and return frozen structure + return CachedRepoInfo( + nb_files=len(blob_stats), + repo_id=repo_id, + repo_path=repo_path, + repo_type=repo_type, # type: ignore + revisions=frozenset(cached_revisions), + size_on_disk=sum(stat.st_size for stat in blob_stats.values()), + last_accessed=repo_last_accessed, + last_modified=repo_last_modified, + ) + + +def _format_size(num: int) -> str: + """Format size in bytes into a human-readable string. + + Taken from https://stackoverflow.com/a/1094933 + """ + num_f = float(num) + for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]: + if abs(num_f) < 1000.0: + return f"{num_f:3.1f}{unit}" + num_f /= 1000.0 + return f"{num_f:.1f}Y" + + +_TIMESINCE_CHUNKS = ( + # Label, divider, max value + ("second", 1, 60), + ("minute", 60, 60), + ("hour", 60 * 60, 24), + ("day", 60 * 60 * 24, 6), + ("week", 60 * 60 * 24 * 7, 6), + ("month", 60 * 60 * 24 * 30, 11), + ("year", 60 * 60 * 24 * 365, None), +) + + +def _format_timesince(ts: float) -> str: + """Format timestamp in seconds into a human-readable string, relative to now. + + Vaguely inspired by Django's `timesince` formatter. + """ + delta = time.time() - ts + if delta < 20: + return "a few seconds ago" + for label, divider, max_value in _TIMESINCE_CHUNKS: # noqa: B007 + value = round(delta / divider) + if max_value is not None and value <= max_value: + break + return f"{value} {label}{'s' if value > 1 else ''} ago" + + +def _try_delete_path(path: Path, path_type: str) -> None: + """Try to delete a local file or folder. + + If the path does not exists, error is logged as a warning and then ignored. + + Args: + path (`Path`) + Path to delete. Can be a file or a folder. + path_type (`str`) + What path are we deleting ? Only for logging purposes. Example: "snapshot". + """ + logger.info(f"Delete {path_type}: {path}") + try: + if path.is_file(): + os.remove(path) + else: + shutil.rmtree(path) + except FileNotFoundError: + logger.warning(f"Couldn't delete {path_type}: file not found ({path})", exc_info=True) + except PermissionError: + logger.warning(f"Couldn't delete {path_type}: permission denied ({path})", exc_info=True) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_chunk_utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_chunk_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fe8ecc9c94f9c09503761e734a005124d3291a52 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_chunk_utils.py @@ -0,0 +1,64 @@ +# coding=utf-8 +# Copyright 2022-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains a utility to iterate by chunks over an iterator.""" + +import itertools +from typing import Iterable, TypeVar + + +T = TypeVar("T") + + +def chunk_iterable(iterable: Iterable[T], chunk_size: int) -> Iterable[Iterable[T]]: + """Iterates over an iterator chunk by chunk. + + Taken from https://stackoverflow.com/a/8998040. + See also https://github.com/huggingface/huggingface_hub/pull/920#discussion_r938793088. + + Args: + iterable (`Iterable`): + The iterable on which we want to iterate. + chunk_size (`int`): + Size of the chunks. Must be a strictly positive integer (e.g. >0). + + Example: + + ```python + >>> from huggingface_hub.utils import chunk_iterable + + >>> for items in chunk_iterable(range(17), chunk_size=8): + ... print(items) + # [0, 1, 2, 3, 4, 5, 6, 7] + # [8, 9, 10, 11, 12, 13, 14, 15] + # [16] # smaller last chunk + ``` + + Raises: + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If `chunk_size` <= 0. + + > [!WARNING] + > The last chunk can be smaller than `chunk_size`. + """ + if not isinstance(chunk_size, int) or chunk_size <= 0: + raise ValueError("`chunk_size` must be a strictly positive integer (>0).") + + iterator = iter(iterable) + while True: + try: + next_item = next(iterator) + except StopIteration: + return + yield itertools.chain((next_item,), itertools.islice(iterator, chunk_size - 1)) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_datetime.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_datetime.py new file mode 100644 index 0000000000000000000000000000000000000000..1a7f44285d1c826006c97176ca66c3e9c33f61c0 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_datetime.py @@ -0,0 +1,67 @@ +# coding=utf-8 +# Copyright 2022-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains utilities to handle datetimes in Huggingface Hub.""" + +from datetime import datetime, timezone + + +def parse_datetime(date_string: str) -> datetime: + """ + Parses a date_string returned from the server to a datetime object. + + This parser is a weak-parser is the sense that it handles only a single format of + date_string. It is expected that the server format will never change. The + implementation depends only on the standard lib to avoid an external dependency + (python-dateutil). See full discussion about this decision on PR: + https://github.com/huggingface/huggingface_hub/pull/999. + + Example: + ```py + > parse_datetime('2022-08-19T07:19:38.123Z') + datetime.datetime(2022, 8, 19, 7, 19, 38, 123000, tzinfo=timezone.utc) + ``` + + Args: + date_string (`str`): + A string representing a datetime returned by the Hub server. + String is expected to follow '%Y-%m-%dT%H:%M:%S.%fZ' pattern. + + Returns: + A python datetime object. + + Raises: + :class:`ValueError`: + If `date_string` cannot be parsed. + """ + try: + # Normalize the string to always have 6 digits of fractional seconds + if date_string.endswith("Z"): + # Case 1: No decimal point (e.g., "2024-11-16T00:27:02Z") + if "." not in date_string: + # No fractional seconds - insert .000000 + date_string = date_string[:-1] + ".000000Z" + # Case 2: Has decimal point (e.g., "2022-08-19T07:19:38.123456789Z") + else: + # Get the fractional and base parts + base, fraction = date_string[:-1].split(".") + # fraction[:6] takes first 6 digits and :0<6 pads with zeros if less than 6 digits + date_string = f"{base}.{fraction[:6]:0<6}Z" + + return datetime.strptime(date_string, "%Y-%m-%dT%H:%M:%S.%fZ").replace(tzinfo=timezone.utc) + except ValueError as e: + raise ValueError( + f"Cannot parse '{date_string}' as a datetime. Date string is expected to" + " follow '%Y-%m-%dT%H:%M:%S.%fZ' pattern." + ) from e diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_deprecation.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_deprecation.py new file mode 100644 index 0000000000000000000000000000000000000000..4cb8d6e418c76accd1ecd61158b4bdd265e12f71 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_deprecation.py @@ -0,0 +1,136 @@ +import warnings +from functools import wraps +from inspect import Parameter, signature +from typing import Iterable, Optional + + +def _deprecate_positional_args(*, version: str): + """Decorator for methods that issues warnings for positional arguments. + Using the keyword-only argument syntax in pep 3102, arguments after the + * will issue a warning when passed as a positional argument. + + Args: + version (`str`): + The version when positional arguments will result in error. + """ + + def _inner_deprecate_positional_args(f): + sig = signature(f) + kwonly_args = [] + all_args = [] + for name, param in sig.parameters.items(): + if param.kind == Parameter.POSITIONAL_OR_KEYWORD: + all_args.append(name) + elif param.kind == Parameter.KEYWORD_ONLY: + kwonly_args.append(name) + + @wraps(f) + def inner_f(*args, **kwargs): + extra_args = len(args) - len(all_args) + if extra_args <= 0: + return f(*args, **kwargs) + # extra_args > 0 + args_msg = [ + f"{name}='{arg}'" if isinstance(arg, str) else f"{name}={arg}" + for name, arg in zip(kwonly_args[:extra_args], args[-extra_args:]) + ] + args_msg = ", ".join(args_msg) + warnings.warn( + f"Deprecated positional argument(s) used in '{f.__name__}': pass" + f" {args_msg} as keyword args. From version {version} passing these" + " as positional arguments will result in an error,", + FutureWarning, + ) + kwargs.update(zip(sig.parameters, args)) + return f(**kwargs) + + return inner_f + + return _inner_deprecate_positional_args + + +def _deprecate_arguments( + *, + version: str, + deprecated_args: Iterable[str], + custom_message: Optional[str] = None, +): + """Decorator to issue warnings when using deprecated arguments. + + TODO: could be useful to be able to set a custom error message. + + Args: + version (`str`): + The version when deprecated arguments will result in error. + deprecated_args (`List[str]`): + List of the arguments to be deprecated. + custom_message (`str`, *optional*): + Warning message that is raised. If not passed, a default warning message + will be created. + """ + + def _inner_deprecate_positional_args(f): + sig = signature(f) + + @wraps(f) + def inner_f(*args, **kwargs): + # Check for used deprecated arguments + used_deprecated_args = [] + for _, parameter in zip(args, sig.parameters.values()): + if parameter.name in deprecated_args: + used_deprecated_args.append(parameter.name) + for kwarg_name, kwarg_value in kwargs.items(): + if ( + # If argument is deprecated but still used + kwarg_name in deprecated_args + # And then the value is not the default value + and kwarg_value != sig.parameters[kwarg_name].default + ): + used_deprecated_args.append(kwarg_name) + + # Warn and proceed + if len(used_deprecated_args) > 0: + message = ( + f"Deprecated argument(s) used in '{f.__name__}':" + f" {', '.join(used_deprecated_args)}. Will not be supported from" + f" version '{version}'." + ) + if custom_message is not None: + message += "\n\n" + custom_message + warnings.warn(message, FutureWarning) + return f(*args, **kwargs) + + return inner_f + + return _inner_deprecate_positional_args + + +def _deprecate_method(*, version: str, message: Optional[str] = None): + """Decorator to issue warnings when using a deprecated method. + + Args: + version (`str`): + The version when deprecated arguments will result in error. + message (`str`, *optional*): + Warning message that is raised. If not passed, a default warning message + will be created. + """ + + def _inner_deprecate_method(f): + name = f.__name__ + if name == "__init__": + name = f.__qualname__.split(".")[0] # class name instead of method name + + @wraps(f) + def inner_f(*args, **kwargs): + warning_message = ( + f"'{name}' (from '{f.__module__}') is deprecated and will be removed from version '{version}'." + ) + if message is not None: + warning_message += " " + message + warnings.warn(warning_message, FutureWarning) + return f(*args, **kwargs) + + return inner_f + + return _inner_deprecate_method diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_experimental.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_experimental.py new file mode 100644 index 0000000000000000000000000000000000000000..40b0ed90ff8af6797758d59b93019498cd72f9ad --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_experimental.py @@ -0,0 +1,68 @@ +# coding=utf-8 +# Copyright 2023-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains utilities to flag a feature as "experimental" in Huggingface Hub.""" + +import warnings +from functools import wraps +from typing import Callable + +from .. import constants + + +def experimental(fn: Callable) -> Callable: + """Decorator to flag a feature as experimental. + + An experimental feature triggers a warning when used as it might be subject to breaking changes without prior notice + in the future. + + Warnings can be disabled by setting `HF_HUB_DISABLE_EXPERIMENTAL_WARNING=1` as environment variable. + + Args: + fn (`Callable`): + The function to flag as experimental. + + Returns: + `Callable`: The decorated function. + + Example: + + ```python + >>> from huggingface_hub.utils import experimental + + >>> @experimental + ... def my_function(): + ... print("Hello world!") + + >>> my_function() + UserWarning: 'my_function' is experimental and might be subject to breaking changes in the future without prior + notice. You can disable this warning by setting `HF_HUB_DISABLE_EXPERIMENTAL_WARNING=1` as environment variable. + Hello world! + ``` + """ + # For classes, put the "experimental" around the "__new__" method => __new__ will be removed in warning message + name = fn.__qualname__[: -len(".__new__")] if fn.__qualname__.endswith(".__new__") else fn.__qualname__ + + @wraps(fn) + def _inner_fn(*args, **kwargs): + if not constants.HF_HUB_DISABLE_EXPERIMENTAL_WARNING: + warnings.warn( + f"'{name}' is experimental and might be subject to breaking changes in the future without prior notice." + " You can disable this warning by setting `HF_HUB_DISABLE_EXPERIMENTAL_WARNING=1` as environment" + " variable.", + UserWarning, + ) + return fn(*args, **kwargs) + + return _inner_fn diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_fixes.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_fixes.py new file mode 100644 index 0000000000000000000000000000000000000000..560003b6222058b03791491b1ce70ea9d7a94404 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_fixes.py @@ -0,0 +1,133 @@ +# JSONDecodeError was introduced in requests=2.27 released in 2022. +# This allows us to support older requests for users +# More information: https://github.com/psf/requests/pull/5856 +try: + from requests import JSONDecodeError # type: ignore # noqa: F401 +except ImportError: + try: + from simplejson import JSONDecodeError # type: ignore # noqa: F401 + except ImportError: + from json import JSONDecodeError # type: ignore # noqa: F401 +import contextlib +import os +import shutil +import stat +import tempfile +import time +from functools import partial +from pathlib import Path +from typing import Callable, Generator, Optional, Union + +import yaml +from filelock import BaseFileLock, FileLock, SoftFileLock, Timeout + +from .. import constants +from . import logging + + +logger = logging.get_logger(__name__) + +# Wrap `yaml.dump` to set `allow_unicode=True` by default. +# +# Example: +# ```py +# >>> yaml.dump({"emoji": "👀", "some unicode": "日本か"}) +# 'emoji: "\\U0001F440"\nsome unicode: "\\u65E5\\u672C\\u304B"\n' +# +# >>> yaml_dump({"emoji": "👀", "some unicode": "日本か"}) +# 'emoji: "👀"\nsome unicode: "日本か"\n' +# ``` +yaml_dump: Callable[..., str] = partial(yaml.dump, stream=None, allow_unicode=True) # type: ignore + + +@contextlib.contextmanager +def SoftTemporaryDirectory( + suffix: Optional[str] = None, + prefix: Optional[str] = None, + dir: Optional[Union[Path, str]] = None, + **kwargs, +) -> Generator[Path, None, None]: + """ + Context manager to create a temporary directory and safely delete it. + + If tmp directory cannot be deleted normally, we set the WRITE permission and retry. + If cleanup still fails, we give up but don't raise an exception. This is equivalent + to `tempfile.TemporaryDirectory(..., ignore_cleanup_errors=True)` introduced in + Python 3.10. + + See https://www.scivision.dev/python-tempfile-permission-error-windows/. + """ + tmpdir = tempfile.TemporaryDirectory(prefix=prefix, suffix=suffix, dir=dir, **kwargs) + yield Path(tmpdir.name).resolve() + + try: + # First once with normal cleanup + shutil.rmtree(tmpdir.name) + except Exception: + # If failed, try to set write permission and retry + try: + shutil.rmtree(tmpdir.name, onerror=_set_write_permission_and_retry) + except Exception: + pass + + # And finally, cleanup the tmpdir. + # If it fails again, give up but do not throw error + try: + tmpdir.cleanup() + except Exception: + pass + + +def _set_write_permission_and_retry(func, path, excinfo): + os.chmod(path, stat.S_IWRITE) + func(path) + + +@contextlib.contextmanager +def WeakFileLock( + lock_file: Union[str, Path], *, timeout: Optional[float] = None +) -> Generator[BaseFileLock, None, None]: + """A filelock with some custom logic. + + This filelock is weaker than the default filelock in that: + 1. It won't raise an exception if release fails. + 2. It will default to a SoftFileLock if the filesystem does not support flock. + + An INFO log message is emitted every 10 seconds if the lock is not acquired immediately. + If a timeout is provided, a `filelock.Timeout` exception is raised if the lock is not acquired within the timeout. + """ + log_interval = constants.FILELOCK_LOG_EVERY_SECONDS + lock = FileLock(lock_file, timeout=log_interval) + start_time = time.time() + + while True: + elapsed_time = time.time() - start_time + if timeout is not None and elapsed_time >= timeout: + raise Timeout(str(lock_file)) + + try: + lock.acquire(timeout=min(log_interval, timeout - elapsed_time) if timeout else log_interval) + except Timeout: + logger.info( + f"Still waiting to acquire lock on {lock_file} (elapsed: {time.time() - start_time:.1f} seconds)" + ) + except NotImplementedError as e: + if "use SoftFileLock instead" in str(e): + logger.warning( + "FileSystem does not appear to support flock. Falling back to SoftFileLock for %s", lock_file + ) + lock = SoftFileLock(lock_file, timeout=log_interval) + continue + else: + break + + try: + yield lock + finally: + try: + lock.release() + except OSError: + try: + Path(lock_file).unlink() + except OSError: + pass diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_git_credential.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_git_credential.py new file mode 100644 index 0000000000000000000000000000000000000000..5ad84648a0093de6e6defc178e4ffffe985f50e4 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_git_credential.py @@ -0,0 +1,121 @@ +# coding=utf-8 +# Copyright 2022-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains utilities to manage Git credentials.""" + +import re +import subprocess +from typing import List, Optional + +from ..constants import ENDPOINT +from ._subprocess import run_interactive_subprocess, run_subprocess + + +GIT_CREDENTIAL_REGEX = re.compile( + r""" + ^\s* # start of line + credential\.helper # credential.helper value + \s*=\s* # separator + ([\w\-\/]+) # the helper name or absolute path (group 1) + (\s|$) # whitespace or end of line + """, + flags=re.MULTILINE | re.IGNORECASE | re.VERBOSE, +) + + +def list_credential_helpers(folder: Optional[str] = None) -> List[str]: + """Return the list of git credential helpers configured. + + See https://git-scm.com/docs/gitcredentials. + + Credentials are saved in all configured helpers (store, cache, macOS keychain,...). + Calls "`git credential approve`" internally. See https://git-scm.com/docs/git-credential. + + Args: + folder (`str`, *optional*): + The folder in which to check the configured helpers. + """ + try: + output = run_subprocess("git config --list", folder=folder).stdout + parsed = _parse_credential_output(output) + return parsed + except subprocess.CalledProcessError as exc: + raise EnvironmentError(exc.stderr) + + +def set_git_credential(token: str, username: str = "hf_user", folder: Optional[str] = None) -> None: + """Save a username/token pair in git credential for HF Hub registry. + + Credentials are saved in all configured helpers (store, cache, macOS keychain,...). + Calls "`git credential approve`" internally. See https://git-scm.com/docs/git-credential. + + Args: + username (`str`, defaults to `"hf_user"`): + A git username. Defaults to `"hf_user"`, the default user used in the Hub. + token (`str`, defaults to `"hf_user"`): + A git password. In practice, the User Access Token for the Hub. + See https://huggingface.co/settings/tokens. + folder (`str`, *optional*): + The folder in which to check the configured helpers. + """ + with run_interactive_subprocess("git credential approve", folder=folder) as ( + stdin, + _, + ): + stdin.write(f"url={ENDPOINT}\nusername={username.lower()}\npassword={token}\n\n") + stdin.flush() + + +def unset_git_credential(username: str = "hf_user", folder: Optional[str] = None) -> None: + """Erase credentials from git credential for HF Hub registry. + + Credentials are erased from the configured helpers (store, cache, macOS + keychain,...), if any. If `username` is not provided, any credential configured for + HF Hub endpoint is erased. + Calls "`git credential erase`" internally. See https://git-scm.com/docs/git-credential. + + Args: + username (`str`, defaults to `"hf_user"`): + A git username. Defaults to `"hf_user"`, the default user used in the Hub. + folder (`str`, *optional*): + The folder in which to check the configured helpers. + """ + with run_interactive_subprocess("git credential reject", folder=folder) as ( + stdin, + _, + ): + standard_input = f"url={ENDPOINT}\n" + if username is not None: + standard_input += f"username={username.lower()}\n" + standard_input += "\n" + + stdin.write(standard_input) + stdin.flush() + + +def _parse_credential_output(output: str) -> List[str]: + """Parse the output of `git credential fill` to extract the password. + + Args: + output (`str`): + The output of `git credential fill`. + """ + # NOTE: If user has set an helper for a custom URL, it will not we caught here. + # Example: `credential.https://huggingface.co.helper=store` + # See: https://github.com/huggingface/huggingface_hub/pull/1138#discussion_r1013324508 + return sorted( # Sort for nice printing + set( # Might have some duplicates + match[0] for match in GIT_CREDENTIAL_REGEX.findall(output) + ) + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_headers.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_headers.py new file mode 100644 index 0000000000000000000000000000000000000000..053a92a398f8734ee14cd67e4b514dfc350fcecd --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_headers.py @@ -0,0 +1,228 @@ +# coding=utf-8 +# Copyright 2022-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains utilities to handle headers to send in calls to Huggingface Hub.""" + +from typing import Dict, Optional, Union + +from huggingface_hub.errors import LocalTokenNotFoundError + +from .. import constants +from ._auth import get_token +from ._deprecation import _deprecate_arguments +from ._runtime import ( + get_fastai_version, + get_fastcore_version, + get_hf_hub_version, + get_python_version, + get_tf_version, + get_torch_version, + is_fastai_available, + is_fastcore_available, + is_tf_available, + is_torch_available, +) +from ._validators import validate_hf_hub_args + + +@_deprecate_arguments( + version="1.0", + deprecated_args="is_write_action", + custom_message="This argument is ignored and we let the server handle the permission error instead (if any).", +) +@validate_hf_hub_args +def build_hf_headers( + *, + token: Optional[Union[bool, str]] = None, + library_name: Optional[str] = None, + library_version: Optional[str] = None, + user_agent: Union[Dict, str, None] = None, + headers: Optional[Dict[str, str]] = None, + is_write_action: bool = False, +) -> Dict[str, str]: + """ + Build headers dictionary to send in a HF Hub call. + + By default, authorization token is always provided either from argument (explicit + use) or retrieved from the cache (implicit use). To explicitly avoid sending the + token to the Hub, set `token=False` or set the `HF_HUB_DISABLE_IMPLICIT_TOKEN` + environment variable. + + In case of an API call that requires write access, an error is thrown if token is + `None` or token is an organization token (starting with `"api_org***"`). + + In addition to the auth header, a user-agent is added to provide information about + the installed packages (versions of python, huggingface_hub, torch, tensorflow, + fastai and fastcore). + + Args: + token (`str`, `bool`, *optional*): + The token to be sent in authorization header for the Hub call: + - if a string, it is used as the Hugging Face token + - if `True`, the token is read from the machine (cache or env variable) + - if `False`, authorization header is not set + - if `None`, the token is read from the machine only except if + `HF_HUB_DISABLE_IMPLICIT_TOKEN` env variable is set. + library_name (`str`, *optional*): + The name of the library that is making the HTTP request. Will be added to + the user-agent header. + library_version (`str`, *optional*): + The version of the library that is making the HTTP request. Will be added + to the user-agent header. + user_agent (`str`, `dict`, *optional*): + The user agent info in the form of a dictionary or a single string. It will + be completed with information about the installed packages. + headers (`dict`, *optional*): + Additional headers to include in the request. Those headers take precedence + over the ones generated by this function. + is_write_action (`bool`): + Ignored and deprecated argument. + + Returns: + A `Dict` of headers to pass in your API call. + + Example: + ```py + >>> build_hf_headers(token="hf_***") # explicit token + {"authorization": "Bearer hf_***", "user-agent": ""} + + >>> build_hf_headers(token=True) # explicitly use cached token + {"authorization": "Bearer hf_***",...} + + >>> build_hf_headers(token=False) # explicitly don't use cached token + {"user-agent": ...} + + >>> build_hf_headers() # implicit use of the cached token + {"authorization": "Bearer hf_***",...} + + # HF_HUB_DISABLE_IMPLICIT_TOKEN=True # to set as env variable + >>> build_hf_headers() # token is not sent + {"user-agent": ...} + + >>> build_hf_headers(library_name="transformers", library_version="1.2.3") + {"authorization": ..., "user-agent": "transformers/1.2.3; hf_hub/0.10.2; python/3.10.4; tensorflow/1.55"} + ``` + + Raises: + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If organization token is passed and "write" access is required. + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If "write" access is required but token is not passed and not saved locally. + [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError) + If `token=True` but token is not saved locally. + """ + # Get auth token to send + token_to_send = get_token_to_send(token) + + # Combine headers + hf_headers = { + "user-agent": _http_user_agent( + library_name=library_name, + library_version=library_version, + user_agent=user_agent, + ) + } + if token_to_send is not None: + hf_headers["authorization"] = f"Bearer {token_to_send}" + if headers is not None: + hf_headers.update(headers) + return hf_headers + + +def get_token_to_send(token: Optional[Union[bool, str]]) -> Optional[str]: + """Select the token to send from either `token` or the cache.""" + # Case token is explicitly provided + if isinstance(token, str): + return token + + # Case token is explicitly forbidden + if token is False: + return None + + # Token is not provided: we get it from local cache + cached_token = get_token() + + # Case token is explicitly required + if token is True: + if cached_token is None: + raise LocalTokenNotFoundError( + "Token is required (`token=True`), but no token found. You" + " need to provide a token or be logged in to Hugging Face with" + " `hf auth login` or `huggingface_hub.login`. See" + " https://huggingface.co/settings/tokens." + ) + return cached_token + + # Case implicit use of the token is forbidden by env variable + if constants.HF_HUB_DISABLE_IMPLICIT_TOKEN: + return None + + # Otherwise: we use the cached token as the user has not explicitly forbidden it + return cached_token + + +def _http_user_agent( + *, + library_name: Optional[str] = None, + library_version: Optional[str] = None, + user_agent: Union[Dict, str, None] = None, +) -> str: + """Format a user-agent string containing information about the installed packages. + + Args: + library_name (`str`, *optional*): + The name of the library that is making the HTTP request. + library_version (`str`, *optional*): + The version of the library that is making the HTTP request. + user_agent (`str`, `dict`, *optional*): + The user agent info in the form of a dictionary or a single string. + + Returns: + The formatted user-agent string. + """ + if library_name is not None: + ua = f"{library_name}/{library_version}" + else: + ua = "unknown/None" + ua += f"; hf_hub/{get_hf_hub_version()}" + ua += f"; python/{get_python_version()}" + + if not constants.HF_HUB_DISABLE_TELEMETRY: + if is_torch_available(): + ua += f"; torch/{get_torch_version()}" + if is_tf_available(): + ua += f"; tensorflow/{get_tf_version()}" + if is_fastai_available(): + ua += f"; fastai/{get_fastai_version()}" + if is_fastcore_available(): + ua += f"; fastcore/{get_fastcore_version()}" + + if isinstance(user_agent, dict): + ua += "; " + "; ".join(f"{k}/{v}" for k, v in user_agent.items()) + elif isinstance(user_agent, str): + ua += "; " + user_agent + + # Retrieve user-agent origin headers from environment variable + origin = constants.HF_HUB_USER_AGENT_ORIGIN + if origin is not None: + ua += "; origin/" + origin + + return _deduplicate_user_agent(ua) + + +def _deduplicate_user_agent(user_agent: str) -> str: + """Deduplicate redundant information in the generated user-agent.""" + # Split around ";" > Strip whitespaces > Store as dict keys (ensure unicity) > format back as string + # Order is implicitly preserved by dictionary structure (see https://stackoverflow.com/a/53657523). + return "; ".join({key.strip(): None for key in user_agent.split(";")}.keys()) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_http.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_http.py new file mode 100644 index 0000000000000000000000000000000000000000..3471031b34b15efe9d5fc76077eac467bbb03500 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_http.py @@ -0,0 +1,638 @@ +# coding=utf-8 +# Copyright 2022-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains utilities to handle HTTP requests in Huggingface Hub.""" + +import io +import os +import re +import threading +import time +import uuid +from functools import lru_cache +from shlex import quote +from typing import Any, Callable, List, Optional, Tuple, Type, Union + +import requests +from requests import HTTPError, Response +from requests.adapters import HTTPAdapter +from requests.models import PreparedRequest + +from huggingface_hub.errors import OfflineModeIsEnabled + +from .. import constants +from ..errors import ( + BadRequestError, + DisabledRepoError, + EntryNotFoundError, + GatedRepoError, + HfHubHTTPError, + RepositoryNotFoundError, + RevisionNotFoundError, +) +from . import logging +from ._fixes import JSONDecodeError +from ._lfs import SliceFileObj +from ._typing import HTTP_METHOD_T + + +logger = logging.get_logger(__name__) + +# Both headers are used by the Hub to debug failed requests. +# `X_AMZN_TRACE_ID` is better as it also works to debug on Cloudfront and ALB. +# If `X_AMZN_TRACE_ID` is set, the Hub will use it as well. +X_AMZN_TRACE_ID = "X-Amzn-Trace-Id" +X_REQUEST_ID = "x-request-id" +X_AMZ_CF_ID = "x-amz-cf-id" + +REPO_API_REGEX = re.compile( + r""" + # staging or production endpoint + ^https://[^/]+ + ( + # on /api/repo_type/repo_id + /api/(models|datasets|spaces)/(.+) + | + # or /repo_id/resolve/revision/... + /(.+)/resolve/(.+) + ) + """, + flags=re.VERBOSE, +) + + +class UniqueRequestIdAdapter(HTTPAdapter): + X_AMZN_TRACE_ID = "X-Amzn-Trace-Id" + + def add_headers(self, request, **kwargs): + super().add_headers(request, **kwargs) + + # Add random request ID => easier for server-side debug + if X_AMZN_TRACE_ID not in request.headers: + request.headers[X_AMZN_TRACE_ID] = request.headers.get(X_REQUEST_ID) or str(uuid.uuid4()) + + # Add debug log + has_token = len(str(request.headers.get("authorization", ""))) > 0 + logger.debug( + f"Request {request.headers[X_AMZN_TRACE_ID]}: {request.method} {request.url} (authenticated: {has_token})" + ) + + def send(self, request: PreparedRequest, *args, **kwargs) -> Response: + """Catch any RequestException to append request id to the error message for debugging.""" + if constants.HF_DEBUG: + logger.debug(f"Send: {_curlify(request)}") + try: + return super().send(request, *args, **kwargs) + except requests.RequestException as e: + request_id = request.headers.get(X_AMZN_TRACE_ID) + if request_id is not None: + # Taken from https://stackoverflow.com/a/58270258 + e.args = (*e.args, f"(Request ID: {request_id})") + raise + + +class OfflineAdapter(HTTPAdapter): + def send(self, request: PreparedRequest, *args, **kwargs) -> Response: + raise OfflineModeIsEnabled( + f"Cannot reach {request.url}: offline mode is enabled. To disable it, please unset the `HF_HUB_OFFLINE` environment variable." + ) + + +def _default_backend_factory() -> requests.Session: + session = requests.Session() + if constants.HF_HUB_OFFLINE: + session.mount("http://", OfflineAdapter()) + session.mount("https://", OfflineAdapter()) + else: + session.mount("http://", UniqueRequestIdAdapter()) + session.mount("https://", UniqueRequestIdAdapter()) + return session + + +BACKEND_FACTORY_T = Callable[[], requests.Session] +_GLOBAL_BACKEND_FACTORY: BACKEND_FACTORY_T = _default_backend_factory + + +def configure_http_backend(backend_factory: BACKEND_FACTORY_T = _default_backend_factory) -> None: + """ + Configure the HTTP backend by providing a `backend_factory`. Any HTTP calls made by `huggingface_hub` will use a + Session object instantiated by this factory. This can be useful if you are running your scripts in a specific + environment requiring custom configuration (e.g. custom proxy or certifications). + + Use [`get_session`] to get a configured Session. Since `requests.Session` is not guaranteed to be thread-safe, + `huggingface_hub` creates 1 Session instance per thread. They are all instantiated using the same `backend_factory` + set in [`configure_http_backend`]. A LRU cache is used to cache the created sessions (and connections) between + calls. Max size is 128 to avoid memory leaks if thousands of threads are spawned. + + See [this issue](https://github.com/psf/requests/issues/2766) to know more about thread-safety in `requests`. + + Example: + ```py + import requests + from huggingface_hub import configure_http_backend, get_session + + # Create a factory function that returns a Session with configured proxies + def backend_factory() -> requests.Session: + session = requests.Session() + session.proxies = {"http": "http://10.10.1.10:3128", "https": "https://10.10.1.11:1080"} + return session + + # Set it as the default session factory + configure_http_backend(backend_factory=backend_factory) + + # In practice, this is mostly done internally in `huggingface_hub` + session = get_session() + ``` + """ + global _GLOBAL_BACKEND_FACTORY + _GLOBAL_BACKEND_FACTORY = backend_factory + reset_sessions() + + +def get_session() -> requests.Session: + """ + Get a `requests.Session` object, using the session factory from the user. + + Use [`get_session`] to get a configured Session. Since `requests.Session` is not guaranteed to be thread-safe, + `huggingface_hub` creates 1 Session instance per thread. They are all instantiated using the same `backend_factory` + set in [`configure_http_backend`]. A LRU cache is used to cache the created sessions (and connections) between + calls. Max size is 128 to avoid memory leaks if thousands of threads are spawned. + + See [this issue](https://github.com/psf/requests/issues/2766) to know more about thread-safety in `requests`. + + Example: + ```py + import requests + from huggingface_hub import configure_http_backend, get_session + + # Create a factory function that returns a Session with configured proxies + def backend_factory() -> requests.Session: + session = requests.Session() + session.proxies = {"http": "http://10.10.1.10:3128", "https": "https://10.10.1.11:1080"} + return session + + # Set it as the default session factory + configure_http_backend(backend_factory=backend_factory) + + # In practice, this is mostly done internally in `huggingface_hub` + session = get_session() + ``` + """ + return _get_session_from_cache(process_id=os.getpid(), thread_id=threading.get_ident()) + + +def reset_sessions() -> None: + """Reset the cache of sessions. + + Mostly used internally when sessions are reconfigured or an SSLError is raised. + See [`configure_http_backend`] for more details. + """ + _get_session_from_cache.cache_clear() + + +@lru_cache +def _get_session_from_cache(process_id: int, thread_id: int) -> requests.Session: + """ + Create a new session per thread using global factory. Using LRU cache (maxsize 128) to avoid memory leaks when + using thousands of threads. Cache is cleared when `configure_http_backend` is called. + """ + return _GLOBAL_BACKEND_FACTORY() + + +def http_backoff( + method: HTTP_METHOD_T, + url: str, + *, + max_retries: int = 5, + base_wait_time: float = 1, + max_wait_time: float = 8, + retry_on_exceptions: Union[Type[Exception], Tuple[Type[Exception], ...]] = ( + requests.Timeout, + requests.ConnectionError, + requests.exceptions.ChunkedEncodingError, + ), + retry_on_status_codes: Union[int, Tuple[int, ...]] = (500, 502, 503, 504), + **kwargs, +) -> Response: + """Wrapper around requests to retry calls on an endpoint, with exponential backoff. + + Endpoint call is retried on exceptions (ex: connection timeout, proxy error,...) + and/or on specific status codes (ex: service unavailable). If the call failed more + than `max_retries`, the exception is thrown or `raise_for_status` is called on the + response object. + + Re-implement mechanisms from the `backoff` library to avoid adding an external + dependencies to `hugging_face_hub`. See https://github.com/litl/backoff. + + Args: + method (`Literal["GET", "OPTIONS", "HEAD", "POST", "PUT", "PATCH", "DELETE"]`): + HTTP method to perform. + url (`str`): + The URL of the resource to fetch. + max_retries (`int`, *optional*, defaults to `5`): + Maximum number of retries, defaults to 5 (no retries). + base_wait_time (`float`, *optional*, defaults to `1`): + Duration (in seconds) to wait before retrying the first time. + Wait time between retries then grows exponentially, capped by + `max_wait_time`. + max_wait_time (`float`, *optional*, defaults to `8`): + Maximum duration (in seconds) to wait before retrying. + retry_on_exceptions (`Type[Exception]` or `Tuple[Type[Exception]]`, *optional*): + Define which exceptions must be caught to retry the request. Can be a single type or a tuple of types. + By default, retry on `requests.Timeout`, `requests.ConnectionError` and `requests.exceptions.ChunkedEncodingError`. + retry_on_status_codes (`int` or `Tuple[int]`, *optional*, defaults to `(500, 502, 503, 504)`): + Define on which status codes the request must be retried. By default, 5xx errors are retried. + **kwargs (`dict`, *optional*): + kwargs to pass to `requests.request`. + + Example: + ``` + >>> from huggingface_hub.utils import http_backoff + + # Same usage as "requests.request". + >>> response = http_backoff("GET", "https://www.google.com") + >>> response.raise_for_status() + + # If you expect a Gateway Timeout from time to time + >>> http_backoff("PUT", upload_url, data=data, retry_on_status_codes=504) + >>> response.raise_for_status() + ``` + + > [!WARNING] + > When using `requests` it is possible to stream data by passing an iterator to the + > `data` argument. On http backoff this is a problem as the iterator is not reset + > after a failed call. This issue is mitigated for file objects or any IO streams + > by saving the initial position of the cursor (with `data.tell()`) and resetting the + > cursor between each call (with `data.seek()`). For arbitrary iterators, http backoff + > will fail. If this is a hard constraint for you, please let us know by opening an + > issue on [Github](https://github.com/huggingface/huggingface_hub). + """ + if isinstance(retry_on_exceptions, type): # Tuple from single exception type + retry_on_exceptions = (retry_on_exceptions,) + + if isinstance(retry_on_status_codes, int): # Tuple from single status code + retry_on_status_codes = (retry_on_status_codes,) + + nb_tries = 0 + sleep_time = base_wait_time + + # If `data` is used and is a file object (or any IO), it will be consumed on the + # first HTTP request. We need to save the initial position so that the full content + # of the file is re-sent on http backoff. See warning tip in docstring. + io_obj_initial_pos = None + if "data" in kwargs and isinstance(kwargs["data"], (io.IOBase, SliceFileObj)): + io_obj_initial_pos = kwargs["data"].tell() + + session = get_session() + while True: + nb_tries += 1 + try: + # If `data` is used and is a file object (or any IO), set back cursor to + # initial position. + if io_obj_initial_pos is not None: + kwargs["data"].seek(io_obj_initial_pos) + + # Perform request and return if status_code is not in the retry list. + response = session.request(method=method, url=url, **kwargs) + if response.status_code not in retry_on_status_codes: + return response + + # Wrong status code returned (HTTP 503 for instance) + logger.warning(f"HTTP Error {response.status_code} thrown while requesting {method} {url}") + if nb_tries > max_retries: + response.raise_for_status() # Will raise uncaught exception + # We return response to avoid infinite loop in the corner case where the + # user ask for retry on a status code that doesn't raise_for_status. + return response + + except retry_on_exceptions as err: + logger.warning(f"'{err}' thrown while requesting {method} {url}") + + if isinstance(err, requests.ConnectionError): + reset_sessions() # In case of SSLError it's best to reset the shared requests.Session objects + + if nb_tries > max_retries: + raise err + + # Sleep for X seconds + logger.warning(f"Retrying in {sleep_time}s [Retry {nb_tries}/{max_retries}].") + time.sleep(sleep_time) + + # Update sleep time for next retry + sleep_time = min(max_wait_time, sleep_time * 2) # Exponential backoff + + +def fix_hf_endpoint_in_url(url: str, endpoint: Optional[str]) -> str: + """Replace the default endpoint in a URL by a custom one. + + This is useful when using a proxy and the Hugging Face Hub returns a URL with the default endpoint. + """ + endpoint = endpoint.rstrip("/") if endpoint else constants.ENDPOINT + # check if a proxy has been set => if yes, update the returned URL to use the proxy + if endpoint not in (constants._HF_DEFAULT_ENDPOINT, constants._HF_DEFAULT_STAGING_ENDPOINT): + url = url.replace(constants._HF_DEFAULT_ENDPOINT, endpoint) + url = url.replace(constants._HF_DEFAULT_STAGING_ENDPOINT, endpoint) + return url + + +def hf_raise_for_status(response: Response, endpoint_name: Optional[str] = None) -> None: + """ + Internal version of `response.raise_for_status()` that will refine a + potential HTTPError. Raised exception will be an instance of `HfHubHTTPError`. + + This helper is meant to be the unique method to raise_for_status when making a call + to the Hugging Face Hub. + + + Example: + ```py + import requests + from huggingface_hub.utils import get_session, hf_raise_for_status, HfHubHTTPError + + response = get_session().post(...) + try: + hf_raise_for_status(response) + except HfHubHTTPError as e: + print(str(e)) # formatted message + e.request_id, e.server_message # details returned by server + + # Complete the error message with additional information once it's raised + e.append_to_message("\n`create_commit` expects the repository to exist.") + raise + ``` + + Args: + response (`Response`): + Response from the server. + endpoint_name (`str`, *optional*): + Name of the endpoint that has been called. If provided, the error message + will be more complete. + + > [!WARNING] + > Raises when the request has failed: + > + > - [`~utils.RepositoryNotFoundError`] + > If the repository to download from cannot be found. This may be because it + > doesn't exist, because `repo_type` is not set correctly, or because the repo + > is `private` and you do not have access. + > - [`~utils.GatedRepoError`] + > If the repository exists but is gated and the user is not on the authorized + > list. + > - [`~utils.RevisionNotFoundError`] + > If the repository exists but the revision couldn't be find. + > - [`~utils.EntryNotFoundError`] + > If the repository exists but the entry (e.g. the requested file) couldn't be + > find. + > - [`~utils.BadRequestError`] + > If request failed with a HTTP 400 BadRequest error. + > - [`~utils.HfHubHTTPError`] + > If request failed for a reason not listed above. + """ + try: + response.raise_for_status() + except HTTPError as e: + error_code = response.headers.get("X-Error-Code") + error_message = response.headers.get("X-Error-Message") + + if error_code == "RevisionNotFound": + message = f"{response.status_code} Client Error." + "\n\n" + f"Revision Not Found for url: {response.url}." + raise _format(RevisionNotFoundError, message, response) from e + + elif error_code == "EntryNotFound": + message = f"{response.status_code} Client Error." + "\n\n" + f"Entry Not Found for url: {response.url}." + raise _format(EntryNotFoundError, message, response) from e + + elif error_code == "GatedRepo": + message = ( + f"{response.status_code} Client Error." + "\n\n" + f"Cannot access gated repo for url {response.url}." + ) + raise _format(GatedRepoError, message, response) from e + + elif error_message == "Access to this resource is disabled.": + message = ( + f"{response.status_code} Client Error." + + "\n\n" + + f"Cannot access repository for url {response.url}." + + "\n" + + "Access to this resource is disabled." + ) + raise _format(DisabledRepoError, message, response) from e + + elif error_code == "RepoNotFound" or ( + response.status_code == 401 + and error_message != "Invalid credentials in Authorization header" + and response.request is not None + and response.request.url is not None + and REPO_API_REGEX.search(response.request.url) is not None + ): + # 401 is misleading as it is returned for: + # - private and gated repos if user is not authenticated + # - missing repos + # => for now, we process them as `RepoNotFound` anyway. + # See https://gist.github.com/Wauplin/46c27ad266b15998ce56a6603796f0b9 + message = ( + f"{response.status_code} Client Error." + + "\n\n" + + f"Repository Not Found for url: {response.url}." + + "\nPlease make sure you specified the correct `repo_id` and" + " `repo_type`.\nIf you are trying to access a private or gated repo," + " make sure you are authenticated. For more details, see" + " https://huggingface.co/docs/huggingface_hub/authentication" + ) + raise _format(RepositoryNotFoundError, message, response) from e + + elif response.status_code == 400: + message = ( + f"\n\nBad request for {endpoint_name} endpoint:" if endpoint_name is not None else "\n\nBad request:" + ) + raise _format(BadRequestError, message, response) from e + + elif response.status_code == 403: + message = ( + f"\n\n{response.status_code} Forbidden: {error_message}." + + f"\nCannot access content at: {response.url}." + + "\nMake sure your token has the correct permissions." + ) + raise _format(HfHubHTTPError, message, response) from e + + elif response.status_code == 416: + range_header = response.request.headers.get("Range") + message = f"{e}. Requested range: {range_header}. Content-Range: {response.headers.get('Content-Range')}." + raise _format(HfHubHTTPError, message, response) from e + + # Convert `HTTPError` into a `HfHubHTTPError` to display request information + # as well (request id and/or server error message) + raise _format(HfHubHTTPError, str(e), response) from e + + +def _format(error_type: Type[HfHubHTTPError], custom_message: str, response: Response) -> HfHubHTTPError: + server_errors = [] + + # Retrieve server error from header + from_headers = response.headers.get("X-Error-Message") + if from_headers is not None: + server_errors.append(from_headers) + + # Retrieve server error from body + try: + # Case errors are returned in a JSON format + data = response.json() + + error = data.get("error") + if error is not None: + if isinstance(error, list): + # Case {'error': ['my error 1', 'my error 2']} + server_errors.extend(error) + else: + # Case {'error': 'my error'} + server_errors.append(error) + + errors = data.get("errors") + if errors is not None: + # Case {'errors': [{'message': 'my error 1'}, {'message': 'my error 2'}]} + for error in errors: + if "message" in error: + server_errors.append(error["message"]) + + except JSONDecodeError: + # If content is not JSON and not HTML, append the text + content_type = response.headers.get("Content-Type", "") + if response.text and "html" not in content_type.lower(): + server_errors.append(response.text) + + # Strip all server messages + server_errors = [str(line).strip() for line in server_errors if str(line).strip()] + + # Deduplicate server messages (keep order) + # taken from https://stackoverflow.com/a/17016257 + server_errors = list(dict.fromkeys(server_errors)) + + # Format server error + server_message = "\n".join(server_errors) + + # Add server error to custom message + final_error_message = custom_message + if server_message and server_message.lower() not in custom_message.lower(): + if "\n\n" in custom_message: + final_error_message += "\n" + server_message + else: + final_error_message += "\n\n" + server_message + + # Prepare Request ID message + request_id = "" + request_id_message = "" + for header, label in ( + (X_REQUEST_ID, "Request ID"), + (X_AMZN_TRACE_ID, "Amzn Trace ID"), + (X_AMZ_CF_ID, "Amz CF ID"), + ): + value = response.headers.get(header) + if value: + request_id = str(value) + request_id_message = f" ({label}: {value})" + break + + # Add Request ID + if request_id and request_id.lower() not in final_error_message.lower(): + if "\n" in final_error_message: + newline_index = final_error_message.index("\n") + final_error_message = ( + final_error_message[:newline_index] + request_id_message + final_error_message[newline_index:] + ) + else: + final_error_message += request_id_message + + # Return + return error_type(final_error_message.strip(), response=response, server_message=server_message or None) + + +def _curlify(request: requests.PreparedRequest) -> str: + """Convert a `requests.PreparedRequest` into a curl command (str). + + Used for debug purposes only. + + Implementation vendored from https://github.com/ofw/curlify/blob/master/curlify.py. + MIT License Copyright (c) 2016 Egor. + """ + parts: List[Tuple[Any, Any]] = [ + ("curl", None), + ("-X", request.method), + ] + + for k, v in sorted(request.headers.items()): + if k.lower() == "authorization": + v = "" # Hide authorization header, no matter its value (can be Bearer, Key, etc.) + parts += [("-H", "{0}: {1}".format(k, v))] + + if request.body: + body = request.body + if isinstance(body, bytes): + body = body.decode("utf-8", errors="ignore") + elif hasattr(body, "read"): + body = "" # Don't try to read it to avoid consuming the stream + if len(body) > 1000: + body = body[:1000] + " ... [truncated]" + parts += [("-d", body.replace("\n", ""))] + + parts += [(None, request.url)] + + flat_parts = [] + for k, v in parts: + if k: + flat_parts.append(quote(k)) + if v: + flat_parts.append(quote(v)) + + return " ".join(flat_parts) + + +# Regex to parse HTTP Range header +RANGE_REGEX = re.compile(r"^\s*bytes\s*=\s*(\d*)\s*-\s*(\d*)\s*$", re.IGNORECASE) + + +def _adjust_range_header(original_range: Optional[str], resume_size: int) -> Optional[str]: + """ + Adjust HTTP Range header to account for resume position. + """ + if not original_range: + return f"bytes={resume_size}-" + + if "," in original_range: + raise ValueError(f"Multiple ranges detected - {original_range!r}, not supported yet.") + + match = RANGE_REGEX.match(original_range) + if not match: + raise RuntimeError(f"Invalid range format - {original_range!r}.") + start, end = match.groups() + + if not start: + if not end: + raise RuntimeError(f"Invalid range format - {original_range!r}.") + + new_suffix = int(end) - resume_size + new_range = f"bytes=-{new_suffix}" + if new_suffix <= 0: + raise RuntimeError(f"Empty new range - {new_range!r}.") + return new_range + + start = int(start) + new_start = start + resume_size + if end: + end = int(end) + new_range = f"bytes={new_start}-{end}" + if new_start > end: + raise RuntimeError(f"Empty new range - {new_range!r}.") + return new_range + + return f"bytes={new_start}-" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_lfs.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_lfs.py new file mode 100644 index 0000000000000000000000000000000000000000..307f371ffa79a8ae726ee03458c52e230a792898 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_lfs.py @@ -0,0 +1,110 @@ +# coding=utf-8 +# Copyright 2019-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Git LFS related utilities""" + +import io +import os +from contextlib import AbstractContextManager +from typing import BinaryIO + + +class SliceFileObj(AbstractContextManager): + """ + Utility context manager to read a *slice* of a seekable file-like object as a seekable, file-like object. + + This is NOT thread safe + + Inspired by stackoverflow.com/a/29838711/593036 + + Credits to @julien-c + + Args: + fileobj (`BinaryIO`): + A file-like object to slice. MUST implement `tell()` and `seek()` (and `read()` of course). + `fileobj` will be reset to its original position when exiting the context manager. + seek_from (`int`): + The start of the slice (offset from position 0 in bytes). + read_limit (`int`): + The maximum number of bytes to read from the slice. + + Attributes: + previous_position (`int`): + The previous position + + Examples: + + Reading 200 bytes with an offset of 128 bytes from a file (ie bytes 128 to 327): + ```python + >>> with open("path/to/file", "rb") as file: + ... with SliceFileObj(file, seek_from=128, read_limit=200) as fslice: + ... fslice.read(...) + ``` + + Reading a file in chunks of 512 bytes + ```python + >>> import os + >>> chunk_size = 512 + >>> file_size = os.getsize("path/to/file") + >>> with open("path/to/file", "rb") as file: + ... for chunk_idx in range(ceil(file_size / chunk_size)): + ... with SliceFileObj(file, seek_from=chunk_idx * chunk_size, read_limit=chunk_size) as fslice: + ... chunk = fslice.read(...) + + ``` + """ + + def __init__(self, fileobj: BinaryIO, seek_from: int, read_limit: int): + self.fileobj = fileobj + self.seek_from = seek_from + self.read_limit = read_limit + + def __enter__(self): + self._previous_position = self.fileobj.tell() + end_of_stream = self.fileobj.seek(0, os.SEEK_END) + self._len = min(self.read_limit, end_of_stream - self.seek_from) + # ^^ The actual number of bytes that can be read from the slice + self.fileobj.seek(self.seek_from, io.SEEK_SET) + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.fileobj.seek(self._previous_position, io.SEEK_SET) + + def read(self, n: int = -1): + pos = self.tell() + if pos >= self._len: + return b"" + remaining_amount = self._len - pos + data = self.fileobj.read(remaining_amount if n < 0 else min(n, remaining_amount)) + return data + + def tell(self) -> int: + return self.fileobj.tell() - self.seek_from + + def seek(self, offset: int, whence: int = os.SEEK_SET) -> int: + start = self.seek_from + end = start + self._len + if whence in (os.SEEK_SET, os.SEEK_END): + offset = start + offset if whence == os.SEEK_SET else end + offset + offset = max(start, min(offset, end)) + whence = os.SEEK_SET + elif whence == os.SEEK_CUR: + cur_pos = self.fileobj.tell() + offset = max(start - cur_pos, min(offset, end - cur_pos)) + else: + raise ValueError(f"whence value {whence} is not supported") + return self.fileobj.seek(offset, whence) - self.seek_from + + def __iter__(self): + yield self.read(n=4 * 1024 * 1024) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_paths.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_paths.py new file mode 100644 index 0000000000000000000000000000000000000000..4f2c0ebce070bbde4900e919a3aca7cfc331e747 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_paths.py @@ -0,0 +1,141 @@ +# coding=utf-8 +# Copyright 2022-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains utilities to handle paths in Huggingface Hub.""" + +from fnmatch import fnmatch +from pathlib import Path +from typing import Callable, Generator, Iterable, List, Optional, TypeVar, Union + + +T = TypeVar("T") + +# Always ignore `.git` and `.cache/huggingface` folders in commits +DEFAULT_IGNORE_PATTERNS = [ + ".git", + ".git/*", + "*/.git", + "**/.git/**", + ".cache/huggingface", + ".cache/huggingface/*", + "*/.cache/huggingface", + "**/.cache/huggingface/**", +] +# Forbidden to commit these folders +FORBIDDEN_FOLDERS = [".git", ".cache"] + + +def filter_repo_objects( + items: Iterable[T], + *, + allow_patterns: Optional[Union[List[str], str]] = None, + ignore_patterns: Optional[Union[List[str], str]] = None, + key: Optional[Callable[[T], str]] = None, +) -> Generator[T, None, None]: + """Filter repo objects based on an allowlist and a denylist. + + Input must be a list of paths (`str` or `Path`) or a list of arbitrary objects. + In the later case, `key` must be provided and specifies a function of one argument + that is used to extract a path from each element in iterable. + + Patterns are Unix shell-style wildcards which are NOT regular expressions. See + https://docs.python.org/3/library/fnmatch.html for more details. + + Args: + items (`Iterable`): + List of items to filter. + allow_patterns (`str` or `List[str]`, *optional*): + Patterns constituting the allowlist. If provided, item paths must match at + least one pattern from the allowlist. + ignore_patterns (`str` or `List[str]`, *optional*): + Patterns constituting the denylist. If provided, item paths must not match + any patterns from the denylist. + key (`Callable[[T], str]`, *optional*): + Single-argument function to extract a path from each item. If not provided, + the `items` must already be `str` or `Path`. + + Returns: + Filtered list of objects, as a generator. + + Raises: + :class:`ValueError`: + If `key` is not provided and items are not `str` or `Path`. + + Example usage with paths: + ```python + >>> # Filter only PDFs that are not hidden. + >>> list(filter_repo_objects( + ... ["aaa.PDF", "bbb.jpg", ".ccc.pdf", ".ddd.png"], + ... allow_patterns=["*.pdf"], + ... ignore_patterns=[".*"], + ... )) + ["aaa.pdf"] + ``` + + Example usage with objects: + ```python + >>> list(filter_repo_objects( + ... [ + ... CommitOperationAdd(path_or_fileobj="/tmp/aaa.pdf", path_in_repo="aaa.pdf") + ... CommitOperationAdd(path_or_fileobj="/tmp/bbb.jpg", path_in_repo="bbb.jpg") + ... CommitOperationAdd(path_or_fileobj="/tmp/.ccc.pdf", path_in_repo=".ccc.pdf") + ... CommitOperationAdd(path_or_fileobj="/tmp/.ddd.png", path_in_repo=".ddd.png") + ... ], + ... allow_patterns=["*.pdf"], + ... ignore_patterns=[".*"], + ... key=lambda x: x.repo_in_path + ... )) + [CommitOperationAdd(path_or_fileobj="/tmp/aaa.pdf", path_in_repo="aaa.pdf")] + ``` + """ + if isinstance(allow_patterns, str): + allow_patterns = [allow_patterns] + + if isinstance(ignore_patterns, str): + ignore_patterns = [ignore_patterns] + + if allow_patterns is not None: + allow_patterns = [_add_wildcard_to_directories(p) for p in allow_patterns] + if ignore_patterns is not None: + ignore_patterns = [_add_wildcard_to_directories(p) for p in ignore_patterns] + + if key is None: + + def _identity(item: T) -> str: + if isinstance(item, str): + return item + if isinstance(item, Path): + return str(item) + raise ValueError(f"Please provide `key` argument in `filter_repo_objects`: `{item}` is not a string.") + + key = _identity # Items must be `str` or `Path`, otherwise raise ValueError + + for item in items: + path = key(item) + + # Skip if there's an allowlist and path doesn't match any + if allow_patterns is not None and not any(fnmatch(path, r) for r in allow_patterns): + continue + + # Skip if there's a denylist and path matches any + if ignore_patterns is not None and any(fnmatch(path, r) for r in ignore_patterns): + continue + + yield item + + +def _add_wildcard_to_directories(pattern: str) -> str: + if pattern[-1] == "/": + return pattern + "*" + return pattern diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_runtime.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_runtime.py new file mode 100644 index 0000000000000000000000000000000000000000..9e38e6da7493074703032150b8b7d6766ed0fed6 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_runtime.py @@ -0,0 +1,395 @@ +# coding=utf-8 +# Copyright 2022-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Check presence of installed packages at runtime.""" + +import importlib.metadata +import os +import platform +import sys +import warnings +from typing import Any, Dict + +from .. import __version__, constants + + +_PY_VERSION: str = sys.version.split()[0].rstrip("+") + +_package_versions = {} + +_CANDIDATES = { + "aiohttp": {"aiohttp"}, + "fastai": {"fastai"}, + "fastapi": {"fastapi"}, + "fastcore": {"fastcore"}, + "gradio": {"gradio"}, + "graphviz": {"graphviz"}, + "hf_transfer": {"hf_transfer"}, + "hf_xet": {"hf_xet"}, + "jinja": {"Jinja2"}, + "keras": {"keras"}, + "numpy": {"numpy"}, + "pillow": {"Pillow"}, + "pydantic": {"pydantic"}, + "pydot": {"pydot"}, + "safetensors": {"safetensors"}, + "tensorboard": {"tensorboardX"}, + "tensorflow": ( + "tensorflow", + "tensorflow-cpu", + "tensorflow-gpu", + "tf-nightly", + "tf-nightly-cpu", + "tf-nightly-gpu", + "intel-tensorflow", + "intel-tensorflow-avx512", + "tensorflow-rocm", + "tensorflow-macos", + ), + "torch": {"torch"}, +} + +# Check once at runtime +for candidate_name, package_names in _CANDIDATES.items(): + _package_versions[candidate_name] = "N/A" + for name in package_names: + try: + _package_versions[candidate_name] = importlib.metadata.version(name) + break + except importlib.metadata.PackageNotFoundError: + pass + + +def _get_version(package_name: str) -> str: + return _package_versions.get(package_name, "N/A") + + +def is_package_available(package_name: str) -> bool: + return _get_version(package_name) != "N/A" + + +# Python +def get_python_version() -> str: + return _PY_VERSION + + +# Huggingface Hub +def get_hf_hub_version() -> str: + return __version__ + + +# aiohttp +def is_aiohttp_available() -> bool: + return is_package_available("aiohttp") + + +def get_aiohttp_version() -> str: + return _get_version("aiohttp") + + +# FastAI +def is_fastai_available() -> bool: + return is_package_available("fastai") + + +def get_fastai_version() -> str: + return _get_version("fastai") + + +# FastAPI +def is_fastapi_available() -> bool: + return is_package_available("fastapi") + + +def get_fastapi_version() -> str: + return _get_version("fastapi") + + +# Fastcore +def is_fastcore_available() -> bool: + return is_package_available("fastcore") + + +def get_fastcore_version() -> str: + return _get_version("fastcore") + + +# FastAI +def is_gradio_available() -> bool: + return is_package_available("gradio") + + +def get_gradio_version() -> str: + return _get_version("gradio") + + +# Graphviz +def is_graphviz_available() -> bool: + return is_package_available("graphviz") + + +def get_graphviz_version() -> str: + return _get_version("graphviz") + + +# hf_transfer +def is_hf_transfer_available() -> bool: + return is_package_available("hf_transfer") + + +def get_hf_transfer_version() -> str: + return _get_version("hf_transfer") + + +# xet +def is_xet_available() -> bool: + # since hf_xet is automatically used if available, allow explicit disabling via environment variable + if constants.HF_HUB_DISABLE_XET: + return False + + return is_package_available("hf_xet") + + +def get_xet_version() -> str: + return _get_version("hf_xet") + + +# keras +def is_keras_available() -> bool: + return is_package_available("keras") + + +def get_keras_version() -> str: + return _get_version("keras") + + +# Numpy +def is_numpy_available() -> bool: + return is_package_available("numpy") + + +def get_numpy_version() -> str: + return _get_version("numpy") + + +# Jinja +def is_jinja_available() -> bool: + return is_package_available("jinja") + + +def get_jinja_version() -> str: + return _get_version("jinja") + + +# Pillow +def is_pillow_available() -> bool: + return is_package_available("pillow") + + +def get_pillow_version() -> str: + return _get_version("pillow") + + +# Pydantic +def is_pydantic_available() -> bool: + if not is_package_available("pydantic"): + return False + # For Pydantic, we add an extra check to test whether it is correctly installed or not. If both pydantic 2.x and + # typing_extensions<=4.5.0 are installed, then pydantic will fail at import time. This should not happen when + # it is installed with `pip install huggingface_hub[inference]` but it can happen when it is installed manually + # by the user in an environment that we don't control. + # + # Usually we won't need to do this kind of check on optional dependencies. However, pydantic is a special case + # as it is automatically imported when doing `from huggingface_hub import ...` even if the user doesn't use it. + # + # See https://github.com/huggingface/huggingface_hub/pull/1829 for more details. + try: + from pydantic import validator # noqa: F401 + except ImportError: + # Example: "ImportError: cannot import name 'TypeAliasType' from 'typing_extensions'" + warnings.warn( + "Pydantic is installed but cannot be imported. Please check your installation. `huggingface_hub` will " + "default to not using Pydantic. Error message: '{e}'" + ) + return False + return True + + +def get_pydantic_version() -> str: + return _get_version("pydantic") + + +# Pydot +def is_pydot_available() -> bool: + return is_package_available("pydot") + + +def get_pydot_version() -> str: + return _get_version("pydot") + + +# Tensorboard +def is_tensorboard_available() -> bool: + return is_package_available("tensorboard") + + +def get_tensorboard_version() -> str: + return _get_version("tensorboard") + + +# Tensorflow +def is_tf_available() -> bool: + return is_package_available("tensorflow") + + +def get_tf_version() -> str: + return _get_version("tensorflow") + + +# Torch +def is_torch_available() -> bool: + return is_package_available("torch") + + +def get_torch_version() -> str: + return _get_version("torch") + + +# Safetensors +def is_safetensors_available() -> bool: + return is_package_available("safetensors") + + +# Shell-related helpers +try: + # Set to `True` if script is running in a Google Colab notebook. + # If running in Google Colab, git credential store is set globally which makes the + # warning disappear. See https://github.com/huggingface/huggingface_hub/issues/1043 + # + # Taken from https://stackoverflow.com/a/63519730. + _is_google_colab = "google.colab" in str(get_ipython()) # type: ignore # noqa: F821 +except NameError: + _is_google_colab = False + + +def is_notebook() -> bool: + """Return `True` if code is executed in a notebook (Jupyter, Colab, QTconsole). + + Taken from https://stackoverflow.com/a/39662359. + Adapted to make it work with Google colab as well. + """ + try: + shell_class = get_ipython().__class__ # type: ignore # noqa: F821 + for parent_class in shell_class.__mro__: # e.g. "is subclass of" + if parent_class.__name__ == "ZMQInteractiveShell": + return True # Jupyter notebook, Google colab or qtconsole + return False + except NameError: + return False # Probably standard Python interpreter + + +def is_google_colab() -> bool: + """Return `True` if code is executed in a Google colab. + + Taken from https://stackoverflow.com/a/63519730. + """ + return _is_google_colab + + +def is_colab_enterprise() -> bool: + """Return `True` if code is executed in a Google Colab Enterprise environment.""" + return os.environ.get("VERTEX_PRODUCT") == "COLAB_ENTERPRISE" + + +def dump_environment_info() -> Dict[str, Any]: + """Dump information about the machine to help debugging issues. + + Similar helper exist in: + - `datasets` (https://github.com/huggingface/datasets/blob/main/src/datasets/commands/env.py) + - `diffusers` (https://github.com/huggingface/diffusers/blob/main/src/diffusers/commands/env.py) + - `transformers` (https://github.com/huggingface/transformers/blob/main/src/transformers/commands/env.py) + """ + from huggingface_hub import get_token, whoami + from huggingface_hub.utils import list_credential_helpers + + token = get_token() + + # Generic machine info + info: Dict[str, Any] = { + "huggingface_hub version": get_hf_hub_version(), + "Platform": platform.platform(), + "Python version": get_python_version(), + } + + # Interpreter info + try: + shell_class = get_ipython().__class__ # type: ignore # noqa: F821 + info["Running in iPython ?"] = "Yes" + info["iPython shell"] = shell_class.__name__ + except NameError: + info["Running in iPython ?"] = "No" + info["Running in notebook ?"] = "Yes" if is_notebook() else "No" + info["Running in Google Colab ?"] = "Yes" if is_google_colab() else "No" + info["Running in Google Colab Enterprise ?"] = "Yes" if is_colab_enterprise() else "No" + # Login info + info["Token path ?"] = constants.HF_TOKEN_PATH + info["Has saved token ?"] = token is not None + if token is not None: + try: + info["Who am I ?"] = whoami()["name"] + except Exception: + pass + + try: + info["Configured git credential helpers"] = ", ".join(list_credential_helpers()) + except Exception: + pass + + # Installed dependencies + info["FastAI"] = get_fastai_version() + info["Tensorflow"] = get_tf_version() + info["Torch"] = get_torch_version() + info["Jinja2"] = get_jinja_version() + info["Graphviz"] = get_graphviz_version() + info["keras"] = get_keras_version() + info["Pydot"] = get_pydot_version() + info["Pillow"] = get_pillow_version() + info["hf_transfer"] = get_hf_transfer_version() + info["gradio"] = get_gradio_version() + info["tensorboard"] = get_tensorboard_version() + info["numpy"] = get_numpy_version() + info["pydantic"] = get_pydantic_version() + info["aiohttp"] = get_aiohttp_version() + info["hf_xet"] = get_xet_version() + + # Environment variables + info["ENDPOINT"] = constants.ENDPOINT + info["HF_HUB_CACHE"] = constants.HF_HUB_CACHE + info["HF_ASSETS_CACHE"] = constants.HF_ASSETS_CACHE + info["HF_TOKEN_PATH"] = constants.HF_TOKEN_PATH + info["HF_STORED_TOKENS_PATH"] = constants.HF_STORED_TOKENS_PATH + info["HF_HUB_OFFLINE"] = constants.HF_HUB_OFFLINE + info["HF_HUB_DISABLE_TELEMETRY"] = constants.HF_HUB_DISABLE_TELEMETRY + info["HF_HUB_DISABLE_PROGRESS_BARS"] = constants.HF_HUB_DISABLE_PROGRESS_BARS + info["HF_HUB_DISABLE_SYMLINKS_WARNING"] = constants.HF_HUB_DISABLE_SYMLINKS_WARNING + info["HF_HUB_DISABLE_EXPERIMENTAL_WARNING"] = constants.HF_HUB_DISABLE_EXPERIMENTAL_WARNING + info["HF_HUB_DISABLE_IMPLICIT_TOKEN"] = constants.HF_HUB_DISABLE_IMPLICIT_TOKEN + info["HF_HUB_DISABLE_XET"] = constants.HF_HUB_DISABLE_XET + info["HF_HUB_ENABLE_HF_TRANSFER"] = constants.HF_HUB_ENABLE_HF_TRANSFER + info["HF_HUB_ETAG_TIMEOUT"] = constants.HF_HUB_ETAG_TIMEOUT + info["HF_HUB_DOWNLOAD_TIMEOUT"] = constants.HF_HUB_DOWNLOAD_TIMEOUT + + print("\nCopy-and-paste the text below in your GitHub issue.\n") + print("\n".join([f"- {prop}: {val}" for prop, val in info.items()]) + "\n") + return info diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_xet.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_xet.py new file mode 100644 index 0000000000000000000000000000000000000000..3dcf99068f87eebdf3c684edf6026a576bd34eaf --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_xet.py @@ -0,0 +1,192 @@ +from dataclasses import dataclass +from enum import Enum +from typing import Dict, Optional + +import requests + +from .. import constants +from . import get_session, hf_raise_for_status, validate_hf_hub_args + + +class XetTokenType(str, Enum): + READ = "read" + WRITE = "write" + + +@dataclass(frozen=True) +class XetFileData: + file_hash: str + refresh_route: str + + +@dataclass(frozen=True) +class XetConnectionInfo: + access_token: str + expiration_unix_epoch: int + endpoint: str + + +def parse_xet_file_data_from_response( + response: requests.Response, endpoint: Optional[str] = None +) -> Optional[XetFileData]: + """ + Parse XET file metadata from an HTTP response. + + This function extracts XET file metadata from the HTTP headers or HTTP links + of a given response object. If the required metadata is not found, it returns `None`. + + Args: + response (`requests.Response`): + The HTTP response object containing headers dict and links dict to extract the XET metadata from. + Returns: + `Optional[XetFileData]`: + An instance of `XetFileData` containing the file hash and refresh route if the metadata + is found. Returns `None` if the required metadata is missing. + """ + if response is None: + return None + try: + file_hash = response.headers[constants.HUGGINGFACE_HEADER_X_XET_HASH] + + if constants.HUGGINGFACE_HEADER_LINK_XET_AUTH_KEY in response.links: + refresh_route = response.links[constants.HUGGINGFACE_HEADER_LINK_XET_AUTH_KEY]["url"] + else: + refresh_route = response.headers[constants.HUGGINGFACE_HEADER_X_XET_REFRESH_ROUTE] + except KeyError: + return None + endpoint = endpoint if endpoint is not None else constants.ENDPOINT + if refresh_route.startswith(constants.HUGGINGFACE_CO_URL_HOME): + refresh_route = refresh_route.replace(constants.HUGGINGFACE_CO_URL_HOME.rstrip("/"), endpoint.rstrip("/")) + return XetFileData( + file_hash=file_hash, + refresh_route=refresh_route, + ) + + +def parse_xet_connection_info_from_headers(headers: Dict[str, str]) -> Optional[XetConnectionInfo]: + """ + Parse XET connection info from the HTTP headers or return None if not found. + Args: + headers (`Dict`): + HTTP headers to extract the XET metadata from. + Returns: + `XetConnectionInfo` or `None`: + The information needed to connect to the XET storage service. + Returns `None` if the headers do not contain the XET connection info. + """ + try: + endpoint = headers[constants.HUGGINGFACE_HEADER_X_XET_ENDPOINT] + access_token = headers[constants.HUGGINGFACE_HEADER_X_XET_ACCESS_TOKEN] + expiration_unix_epoch = int(headers[constants.HUGGINGFACE_HEADER_X_XET_EXPIRATION]) + except (KeyError, ValueError, TypeError): + return None + + return XetConnectionInfo( + endpoint=endpoint, + access_token=access_token, + expiration_unix_epoch=expiration_unix_epoch, + ) + + +@validate_hf_hub_args +def refresh_xet_connection_info( + *, + file_data: XetFileData, + headers: Dict[str, str], +) -> XetConnectionInfo: + """ + Utilizes the information in the parsed metadata to request the Hub xet connection information. + This includes the access token, expiration, and XET service URL. + Args: + file_data: (`XetFileData`): + The file data needed to refresh the xet connection information. + headers (`Dict[str, str]`): + Headers to use for the request, including authorization headers and user agent. + Returns: + `XetConnectionInfo`: + The connection information needed to make the request to the xet storage service. + Raises: + [`~utils.HfHubHTTPError`] + If the Hub API returned an error. + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If the Hub API response is improperly formatted. + """ + if file_data.refresh_route is None: + raise ValueError("The provided xet metadata does not contain a refresh endpoint.") + return _fetch_xet_connection_info_with_url(file_data.refresh_route, headers) + + +@validate_hf_hub_args +def fetch_xet_connection_info_from_repo_info( + *, + token_type: XetTokenType, + repo_id: str, + repo_type: str, + revision: Optional[str] = None, + headers: Dict[str, str], + endpoint: Optional[str] = None, + params: Optional[Dict[str, str]] = None, +) -> XetConnectionInfo: + """ + Uses the repo info to request a xet access token from Hub. + Args: + token_type (`XetTokenType`): + Type of the token to request: `"read"` or `"write"`. + repo_id (`str`): + A namespace (user or an organization) and a repo name separated by a `/`. + repo_type (`str`): + Type of the repo to upload to: `"model"`, `"dataset"` or `"space"`. + revision (`str`, `optional`): + The revision of the repo to get the token for. + headers (`Dict[str, str]`): + Headers to use for the request, including authorization headers and user agent. + endpoint (`str`, `optional`): + The endpoint to use for the request. Defaults to the Hub endpoint. + params (`Dict[str, str]`, `optional`): + Additional parameters to pass with the request. + Returns: + `XetConnectionInfo`: + The connection information needed to make the request to the xet storage service. + Raises: + [`~utils.HfHubHTTPError`] + If the Hub API returned an error. + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If the Hub API response is improperly formatted. + """ + endpoint = endpoint if endpoint is not None else constants.ENDPOINT + url = f"{endpoint}/api/{repo_type}s/{repo_id}/xet-{token_type.value}-token/{revision}" + return _fetch_xet_connection_info_with_url(url, headers, params) + + +@validate_hf_hub_args +def _fetch_xet_connection_info_with_url( + url: str, + headers: Dict[str, str], + params: Optional[Dict[str, str]] = None, +) -> XetConnectionInfo: + """ + Requests the xet connection info from the supplied URL. This includes the + access token, expiration time, and endpoint to use for the xet storage service. + Args: + url: (`str`): + The access token endpoint URL. + headers (`Dict[str, str]`): + Headers to use for the request, including authorization headers and user agent. + params (`Dict[str, str]`, `optional`): + Additional parameters to pass with the request. + Returns: + `XetConnectionInfo`: + The connection information needed to make the request to the xet storage service. + Raises: + [`~utils.HfHubHTTPError`] + If the Hub API returned an error. + [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) + If the Hub API response is improperly formatted. + """ + resp = get_session().get(headers=headers, url=url, params=params) + hf_raise_for_status(resp) + + metadata = parse_xet_connection_info_from_headers(resp.headers) # type: ignore + if metadata is None: + raise ValueError("Xet headers have not been correctly set by the server.") + return metadata diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_xet_progress_reporting.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_xet_progress_reporting.py new file mode 100644 index 0000000000000000000000000000000000000000..e47740d5c5ea27253debc9e29c1eae9d10a6034f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/_xet_progress_reporting.py @@ -0,0 +1,162 @@ +from collections import OrderedDict +from typing import List + +from hf_xet import PyItemProgressUpdate, PyTotalProgressUpdate + +from . import is_google_colab, is_notebook +from .tqdm import tqdm + + +class XetProgressReporter: + """ + Reports on progress for Xet uploads. + + Shows summary progress bars when running in notebooks or GUIs, and detailed per-file progress in console environments. + """ + + def __init__(self, n_lines: int = 10, description_width: int = 30): + self.n_lines = n_lines + self.description_width = description_width + + self.per_file_progress = is_google_colab() or not is_notebook() + + self.tqdm_settings = { + "unit": "B", + "unit_scale": True, + "leave": True, + "unit_divisor": 1000, + "nrows": n_lines + 3 if self.per_file_progress else 3, + "miniters": 1, + "bar_format": "{l_bar}{bar}| {n_fmt:>5}B / {total_fmt:>5}B{postfix:>12}", + } + + # Overall progress bars + self.data_processing_bar = tqdm( + total=0, desc=self.format_desc("Processing Files (0 / 0)", False), position=0, **self.tqdm_settings + ) + + self.upload_bar = tqdm( + total=0, desc=self.format_desc("New Data Upload", False), position=1, **self.tqdm_settings + ) + + self.known_items: set[str] = set() + self.completed_items: set[str] = set() + + # Item bars (scrolling view) + self.item_state: OrderedDict[str, PyItemProgressUpdate] = OrderedDict() + self.current_bars: List = [None] * self.n_lines + + def format_desc(self, name: str, indent: bool) -> str: + """ + if name is longer than width characters, prints ... at the start and then the last width-3 characters of the name, otherwise + the whole name right justified into description_width characters. Also adds some padding. + """ + + if not self.per_file_progress: + # Here we just use the defaults. + return name + + padding = " " if indent else "" + width = self.description_width - len(padding) + + if len(name) > width: + name = f"...{name[-(width - 3) :]}" + + return f"{padding}{name.ljust(width)}" + + def update_progress(self, total_update: PyTotalProgressUpdate, item_updates: List[PyItemProgressUpdate]): + # Update all the per-item values. + for item in item_updates: + item_name = item.item_name + + self.known_items.add(item_name) + + # Only care about items where the processing has already started. + if item.bytes_completed == 0: + continue + + # Overwrite the existing value in there. + self.item_state[item_name] = item + + bar_idx = 0 + new_completed = [] + + # Now, go through and update all the bars + for name, item in self.item_state.items(): + # Is this ready to be removed on the next update? + if item.bytes_completed == item.total_bytes: + self.completed_items.add(name) + new_completed.append(name) + + # If we're only showing summary information, then don't update the individual bars + if not self.per_file_progress: + continue + + # If we've run out of bars to use, then collapse the last ones together. + if bar_idx >= len(self.current_bars): + bar = self.current_bars[-1] + in_final_bar_mode = True + final_bar_aggregation_count = bar_idx + 1 - len(self.current_bars) + else: + bar = self.current_bars[bar_idx] + in_final_bar_mode = False + + if bar is None: + self.current_bars[bar_idx] = tqdm( + desc=self.format_desc(name, True), + position=2 + bar_idx, # Set to the position past the initial bars. + total=item.total_bytes, + initial=item.bytes_completed, + **self.tqdm_settings, + ) + + elif in_final_bar_mode: + bar.n += item.bytes_completed + bar.total += item.total_bytes + bar.set_description(self.format_desc(f"[+ {final_bar_aggregation_count} files]", True), refresh=False) + else: + bar.set_description(self.format_desc(name, True), refresh=False) + bar.n = item.bytes_completed + bar.total = item.total_bytes + + bar_idx += 1 + + # Remove all the completed ones from the ordered dictionary + for name in new_completed: + # Only remove ones from consideration to make room for more items coming in. + if len(self.item_state) <= self.n_lines: + break + + del self.item_state[name] + + if self.per_file_progress: + # Now manually refresh each of the bars + for bar in self.current_bars: + if bar: + bar.refresh() + + # Update overall bars + def postfix(speed): + s = tqdm.format_sizeof(speed) if speed is not None else "???" + return f"{s}B/s ".rjust(10, " ") + + self.data_processing_bar.total = total_update.total_bytes + self.data_processing_bar.set_description( + self.format_desc(f"Processing Files ({len(self.completed_items)} / {len(self.known_items)})", False), + refresh=False, + ) + self.data_processing_bar.set_postfix_str(postfix(total_update.total_bytes_completion_rate), refresh=False) + self.data_processing_bar.update(total_update.total_bytes_completion_increment) + + self.upload_bar.total = total_update.total_transfer_bytes + self.upload_bar.set_postfix_str(postfix(total_update.total_transfer_bytes_completion_rate), refresh=False) + self.upload_bar.update(total_update.total_transfer_bytes_completion_increment) + + def close(self, _success): + self.data_processing_bar.close() + self.upload_bar.close() + + if self.per_file_progress: + for bar in self.current_bars: + if bar: + bar.close() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/insecure_hashlib.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/insecure_hashlib.py new file mode 100644 index 0000000000000000000000000000000000000000..6901b6d647cc706b85333a66f3bcb7d8c5e2ee9e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/insecure_hashlib.py @@ -0,0 +1,38 @@ +# Taken from https://github.com/mlflow/mlflow/pull/10119 +# +# DO NOT use this function for security purposes (e.g., password hashing). +# +# In Python >= 3.9, insecure hashing algorithms such as MD5 fail in FIPS-compliant +# environments unless `usedforsecurity=False` is explicitly passed. +# +# References: +# - https://github.com/mlflow/mlflow/issues/9905 +# - https://github.com/mlflow/mlflow/pull/10119 +# - https://docs.python.org/3/library/hashlib.html +# - https://github.com/huggingface/transformers/pull/27038 +# +# Usage: +# ```python +# # Use +# from huggingface_hub.utils.insecure_hashlib import sha256 +# # instead of +# from hashlib import sha256 +# +# # Use +# from huggingface_hub.utils import insecure_hashlib +# # instead of +# import hashlib +# ``` +import functools +import hashlib +import sys + + +if sys.version_info >= (3, 9): + md5 = functools.partial(hashlib.md5, usedforsecurity=False) + sha1 = functools.partial(hashlib.sha1, usedforsecurity=False) + sha256 = functools.partial(hashlib.sha256, usedforsecurity=False) +else: + md5 = hashlib.md5 + sha1 = hashlib.sha1 + sha256 = hashlib.sha256 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/tqdm.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/tqdm.py new file mode 100644 index 0000000000000000000000000000000000000000..4c1fcef4beb73bae13c57b3f66c5828e775b7cd9 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/huggingface_hub/utils/tqdm.py @@ -0,0 +1,307 @@ +# coding=utf-8 +# Copyright 2021 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License +"""Utility helpers to handle progress bars in `huggingface_hub`. + +Example: + 1. Use `huggingface_hub.utils.tqdm` as you would use `tqdm.tqdm` or `tqdm.auto.tqdm`. + 2. To disable progress bars, either use `disable_progress_bars()` helper or set the + environment variable `HF_HUB_DISABLE_PROGRESS_BARS` to 1. + 3. To re-enable progress bars, use `enable_progress_bars()`. + 4. To check whether progress bars are disabled, use `are_progress_bars_disabled()`. + +NOTE: Environment variable `HF_HUB_DISABLE_PROGRESS_BARS` has the priority. + +Example: + ```py + >>> from huggingface_hub.utils import are_progress_bars_disabled, disable_progress_bars, enable_progress_bars, tqdm + + # Disable progress bars globally + >>> disable_progress_bars() + + # Use as normal `tqdm` + >>> for _ in tqdm(range(5)): + ... pass + + # Still not showing progress bars, as `disable=False` is overwritten to `True`. + >>> for _ in tqdm(range(5), disable=False): + ... pass + + >>> are_progress_bars_disabled() + True + + # Re-enable progress bars globally + >>> enable_progress_bars() + + # Progress bar will be shown ! + >>> for _ in tqdm(range(5)): + ... pass + 100%|███████████████████████████████████████| 5/5 [00:00<00:00, 117817.53it/s] + ``` + +Group-based control: + ```python + # Disable progress bars for a specific group + >>> disable_progress_bars("peft.foo") + + # Check state of different groups + >>> assert not are_progress_bars_disabled("peft")) + >>> assert not are_progress_bars_disabled("peft.something") + >>> assert are_progress_bars_disabled("peft.foo")) + >>> assert are_progress_bars_disabled("peft.foo.bar")) + + # Enable progress bars for a subgroup + >>> enable_progress_bars("peft.foo.bar") + + # Check if enabling a subgroup affects the parent group + >>> assert are_progress_bars_disabled("peft.foo")) + >>> assert not are_progress_bars_disabled("peft.foo.bar")) + + # No progress bar for `name="peft.foo"` + >>> for _ in tqdm(range(5), name="peft.foo"): + ... pass + + # Progress bar will be shown for `name="peft.foo.bar"` + >>> for _ in tqdm(range(5), name="peft.foo.bar"): + ... pass + 100%|███████████████████████████████████████| 5/5 [00:00<00:00, 117817.53it/s] + + ``` +""" + +import io +import logging +import os +import warnings +from contextlib import contextmanager, nullcontext +from pathlib import Path +from typing import ContextManager, Dict, Iterator, Optional, Union + +from tqdm.auto import tqdm as old_tqdm + +from ..constants import HF_HUB_DISABLE_PROGRESS_BARS + + +# The `HF_HUB_DISABLE_PROGRESS_BARS` environment variable can be True, False, or not set (None), +# allowing for control over progress bar visibility. When set, this variable takes precedence +# over programmatic settings, dictating whether progress bars should be shown or hidden globally. +# Essentially, the environment variable's setting overrides any code-based configurations. +# +# If `HF_HUB_DISABLE_PROGRESS_BARS` is not defined (None), it implies that users can manage +# progress bar visibility through code. By default, progress bars are turned on. + + +progress_bar_states: Dict[str, bool] = {} + + +def disable_progress_bars(name: Optional[str] = None) -> None: + """ + Disable progress bars either globally or for a specified group. + + This function updates the state of progress bars based on a group name. + If no group name is provided, all progress bars are disabled. The operation + respects the `HF_HUB_DISABLE_PROGRESS_BARS` environment variable's setting. + + Args: + name (`str`, *optional*): + The name of the group for which to disable the progress bars. If None, + progress bars are disabled globally. + + Raises: + Warning: If the environment variable precludes changes. + """ + if HF_HUB_DISABLE_PROGRESS_BARS is False: + warnings.warn( + "Cannot disable progress bars: environment variable `HF_HUB_DISABLE_PROGRESS_BARS=0` is set and has priority." + ) + return + + if name is None: + progress_bar_states.clear() + progress_bar_states["_global"] = False + else: + keys_to_remove = [key for key in progress_bar_states if key.startswith(f"{name}.")] + for key in keys_to_remove: + del progress_bar_states[key] + progress_bar_states[name] = False + + +def enable_progress_bars(name: Optional[str] = None) -> None: + """ + Enable progress bars either globally or for a specified group. + + This function sets the progress bars to enabled for the specified group or globally + if no group is specified. The operation is subject to the `HF_HUB_DISABLE_PROGRESS_BARS` + environment setting. + + Args: + name (`str`, *optional*): + The name of the group for which to enable the progress bars. If None, + progress bars are enabled globally. + + Raises: + Warning: If the environment variable precludes changes. + """ + if HF_HUB_DISABLE_PROGRESS_BARS is True: + warnings.warn( + "Cannot enable progress bars: environment variable `HF_HUB_DISABLE_PROGRESS_BARS=1` is set and has priority." + ) + return + + if name is None: + progress_bar_states.clear() + progress_bar_states["_global"] = True + else: + keys_to_remove = [key for key in progress_bar_states if key.startswith(f"{name}.")] + for key in keys_to_remove: + del progress_bar_states[key] + progress_bar_states[name] = True + + +def are_progress_bars_disabled(name: Optional[str] = None) -> bool: + """ + Check if progress bars are disabled globally or for a specific group. + + This function returns whether progress bars are disabled for a given group or globally. + It checks the `HF_HUB_DISABLE_PROGRESS_BARS` environment variable first, then the programmatic + settings. + + Args: + name (`str`, *optional*): + The group name to check; if None, checks the global setting. + + Returns: + `bool`: True if progress bars are disabled, False otherwise. + """ + if HF_HUB_DISABLE_PROGRESS_BARS is True: + return True + + if name is None: + return not progress_bar_states.get("_global", True) + + while name: + if name in progress_bar_states: + return not progress_bar_states[name] + name = ".".join(name.split(".")[:-1]) + + return not progress_bar_states.get("_global", True) + + +def is_tqdm_disabled(log_level: int) -> Optional[bool]: + """ + Determine if tqdm progress bars should be disabled based on logging level and environment settings. + + see https://github.com/huggingface/huggingface_hub/pull/2000 and https://github.com/huggingface/huggingface_hub/pull/2698. + """ + if log_level == logging.NOTSET: + return True + if os.getenv("TQDM_POSITION") == "-1": + return False + return None + + +class tqdm(old_tqdm): + """ + Class to override `disable` argument in case progress bars are globally disabled. + + Taken from https://github.com/tqdm/tqdm/issues/619#issuecomment-619639324. + """ + + def __init__(self, *args, **kwargs): + name = kwargs.pop("name", None) # do not pass `name` to `tqdm` + if are_progress_bars_disabled(name): + kwargs["disable"] = True + super().__init__(*args, **kwargs) + + def __delattr__(self, attr: str) -> None: + """Fix for https://github.com/huggingface/huggingface_hub/issues/1603""" + try: + super().__delattr__(attr) + except AttributeError: + if attr != "_lock": + raise + + +@contextmanager +def tqdm_stream_file(path: Union[Path, str]) -> Iterator[io.BufferedReader]: + """ + Open a file as binary and wrap the `read` method to display a progress bar when it's streamed. + + First implemented in `transformers` in 2019 but removed when switched to git-lfs. Used in `huggingface_hub` to show + progress bar when uploading an LFS file to the Hub. See github.com/huggingface/transformers/pull/2078#discussion_r354739608 + for implementation details. + + Note: currently implementation handles only files stored on disk as it is the most common use case. Could be + extended to stream any `BinaryIO` object but we might have to debug some corner cases. + + Example: + ```py + >>> with tqdm_stream_file("config.json") as f: + >>> requests.put(url, data=f) + config.json: 100%|█████████████████████████| 8.19k/8.19k [00:02<00:00, 3.72kB/s] + ``` + """ + if isinstance(path, str): + path = Path(path) + + with path.open("rb") as f: + total_size = path.stat().st_size + pbar = tqdm( + unit="B", + unit_scale=True, + total=total_size, + initial=0, + desc=path.name, + ) + + f_read = f.read + + def _inner_read(size: Optional[int] = -1) -> bytes: + data = f_read(size) + pbar.update(len(data)) + return data + + f.read = _inner_read # type: ignore + + yield f + + pbar.close() + + +def _get_progress_bar_context( + *, + desc: str, + log_level: int, + total: Optional[int] = None, + initial: int = 0, + unit: str = "B", + unit_scale: bool = True, + name: Optional[str] = None, + _tqdm_bar: Optional[tqdm] = None, +) -> ContextManager[tqdm]: + if _tqdm_bar is not None: + return nullcontext(_tqdm_bar) + # ^ `contextlib.nullcontext` mimics a context manager that does nothing + # Makes it easier to use the same code path for both cases but in the later + # case, the progress bar is not closed when exiting the context manager. + + return tqdm( + unit=unit, + unit_scale=unit_scale, + total=total, + initial=initial, + desc=desc, + disable=is_tqdm_disabled(log_level=log_level), + name=name, + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/calculus/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/calculus/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c4730aa4c6f0a40d620418742875293df1ee1ebb Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/calculus/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/calculus/__pycache__/approximation.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/calculus/__pycache__/approximation.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f22cd39c1a58b4a901d13940b38ef6fc0244eb6e Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/calculus/__pycache__/approximation.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/calculus/__pycache__/calculus.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/calculus/__pycache__/calculus.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8c76d286718cdf80effe3295d45a1560e3d35f48 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/calculus/__pycache__/calculus.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/calculus/__pycache__/differentiation.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/calculus/__pycache__/differentiation.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..02efc7ec6db7f5c72040efdff68ba0d542a0f2aa Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/calculus/__pycache__/differentiation.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/calculus/__pycache__/extrapolation.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/calculus/__pycache__/extrapolation.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0608a97010082168781bf14fdc823838b437ff80 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/calculus/__pycache__/extrapolation.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/calculus/__pycache__/inverselaplace.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/calculus/__pycache__/inverselaplace.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..217bad3b13950ccc1861ed36bb28eee125c66a5f Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/calculus/__pycache__/inverselaplace.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/calculus/__pycache__/odes.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/calculus/__pycache__/odes.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8e778263309c20f50218d541a6e96cd350528864 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/calculus/__pycache__/odes.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/calculus/__pycache__/optimization.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/calculus/__pycache__/optimization.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..872daaf2e4da110786a9ef1bccc5396c6ffbc397 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/calculus/__pycache__/optimization.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/calculus/__pycache__/polynomials.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/calculus/__pycache__/polynomials.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..275e7d30ab64c75de56a073800e877cd715a7896 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/calculus/__pycache__/polynomials.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/calculus/__pycache__/quadrature.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/calculus/__pycache__/quadrature.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..012dbafb256b5644831b7fc01d879ae56c4f502f Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/calculus/__pycache__/quadrature.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3c0f8a63560943afeaaa9739bf5555d2a883de86 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/bessel.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/bessel.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1a2503864cccec0bdf5717c9a5f4ad388768b53a Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/bessel.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/elliptic.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/elliptic.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f369c669cae88efd3ccd9e35d59dc3f973cee081 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/elliptic.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/expintegrals.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/expintegrals.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..48f9362c404561d1383bae3f07a13d1a6d554233 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/expintegrals.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/factorials.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/factorials.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..12112a59184a7dafa2acc7d451a66ae4b777294e Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/factorials.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/functions.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/functions.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..52a618a11e17e94122acfe9fbf84f12b68f2a97f Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/functions.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/hypergeometric.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/hypergeometric.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..20f66c97a73fae2510d02da3287823ee01d5a424 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/hypergeometric.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/orthogonal.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/orthogonal.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..24696226eddd9706add26a8be07f44545e85fe14 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/orthogonal.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/qfunctions.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/qfunctions.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a44b2f7ec1882a92f556786d00c29775e14cf112 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/qfunctions.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/rszeta.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/rszeta.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ba0dde78841ad46f4295f88bf18337d0be2fbffc Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/rszeta.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/signals.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/signals.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7678ca423b2a24c3db39ef043e9fe181495110bc Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/signals.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/theta.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/theta.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4647bf32357880bcc5160eee5b239fc19a26a1c3 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/theta.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/zeta.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/zeta.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..883b7ab4dc134975b2bedb28fc511c9e7752c81e Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/zeta.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/zetazeros.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/zetazeros.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0632c42d69c9f32256d8ab6b8de2be57a3c982ea Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/functions/__pycache__/zetazeros.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/libmp/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/libmp/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6f7336ff493513d0a142e1dbfe7b7ea258128637 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/libmp/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/libmp/__pycache__/backend.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/libmp/__pycache__/backend.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f84488e04fdbe90db85f9f317899f43a216f34e3 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/libmp/__pycache__/backend.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/libmp/__pycache__/gammazeta.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/libmp/__pycache__/gammazeta.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f46d294b17f5264786d12fdb696f5ed29d2e384e Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/libmp/__pycache__/gammazeta.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/libmp/__pycache__/libelefun.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/libmp/__pycache__/libelefun.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..22551b9960dc8c6155e31977bd503d000abe1416 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/libmp/__pycache__/libelefun.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/libmp/__pycache__/libhyper.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/libmp/__pycache__/libhyper.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..80c39412fb305562a17df183acd63c279b822802 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/libmp/__pycache__/libhyper.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/libmp/__pycache__/libintmath.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/libmp/__pycache__/libintmath.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4fa6af8ea560d1068bf5fcca5d5fde094a1bab56 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/libmp/__pycache__/libintmath.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/libmp/__pycache__/libmpc.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/libmp/__pycache__/libmpc.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b4e8f38cd4a41a93042778ed5ed1fbe14eb00f85 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/libmp/__pycache__/libmpc.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/libmp/__pycache__/libmpf.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/libmp/__pycache__/libmpf.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fb4b1abef5909d2f21e456d26f6acfbd95c71734 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/libmp/__pycache__/libmpf.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/libmp/__pycache__/libmpi.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/libmp/__pycache__/libmpi.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e67b32d589908190a929cadaa02bfb8787c44b78 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/libmp/__pycache__/libmpi.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/matrices/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/matrices/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5cdc96fdb0807444329b450cf7ec41f63413e63e Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/matrices/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/matrices/__pycache__/calculus.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/matrices/__pycache__/calculus.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9dd243d66a3b8b16d5cd9391fcaea8a0a8b90a65 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/matrices/__pycache__/calculus.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/matrices/__pycache__/eigen.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/matrices/__pycache__/eigen.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b482f5ac7b1f9ce4a9f05a179cda9bcb2b564088 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/matrices/__pycache__/eigen.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/matrices/__pycache__/eigen_symmetric.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/matrices/__pycache__/eigen_symmetric.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3a04f9b05edd8ecc01dc64479ec249708554b31f Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/matrices/__pycache__/eigen_symmetric.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/matrices/__pycache__/linalg.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/matrices/__pycache__/linalg.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..aed77151074c6ad1beb156ca7b990f8509322b60 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/matrices/__pycache__/linalg.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/matrices/__pycache__/matrices.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/matrices/__pycache__/matrices.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f7c1213492af6065ddb792ab336384b2167712f0 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/matrices/__pycache__/matrices.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/__pycache__/test_basic_ops.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/__pycache__/test_basic_ops.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a423e8a629ac001cd8e6be2631d851cb7f036482 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/__pycache__/test_basic_ops.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/__pycache__/test_bitwise.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/__pycache__/test_bitwise.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c8ee45aa4f591ca51ff721db99132d4449e7329e Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/__pycache__/test_bitwise.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/__pycache__/test_diff.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/__pycache__/test_diff.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3d2420e4a7ae039085c7df8d7f4966ac3b553bd8 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/__pycache__/test_diff.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/__pycache__/test_quad.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/__pycache__/test_quad.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..842fa67964bd224cf8d554105f97466c330a4576 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/__pycache__/test_quad.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/extratest_gamma.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/extratest_gamma.py new file mode 100644 index 0000000000000000000000000000000000000000..5a27b61b19aba0abf6bdb8adc16fc1ec7689b67a --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/extratest_gamma.py @@ -0,0 +1,215 @@ +from mpmath import * +from mpmath.libmp import ifac + +import sys +if "-dps" in sys.argv: + maxdps = int(sys.argv[sys.argv.index("-dps")+1]) +else: + maxdps = 1000 + +raise_ = "-raise" in sys.argv + +errcount = 0 + +def check(name, func, z, y): + global errcount + try: + x = func(z) + except: + errcount += 1 + if raise_: + raise + print() + print(name) + print("EXCEPTION") + import traceback + traceback.print_tb(sys.exc_info()[2]) + print() + return + xre = x.real + xim = x.imag + yre = y.real + yim = y.imag + tol = eps*8 + err = 0 + if abs(xre-yre) > abs(yre)*tol: + err = 1 + print() + print("Error! %s (re = %s, wanted %s, err=%s)" % (name, nstr(xre,10), nstr(yre,10), nstr(abs(xre-yre)))) + errcount += 1 + if raise_: + raise SystemExit + if abs(xim-yim) > abs(yim)*tol: + err = 1 + print() + print("Error! %s (im = %s, wanted %s, err=%s)" % (name, nstr(xim,10), nstr(yim,10), nstr(abs(xim-yim)))) + errcount += 1 + if raise_: + raise SystemExit + if not err: + sys.stdout.write("%s ok; " % name) + +def testcase(case): + z, result = case + print("Testing z =", z) + mp.dps = 1010 + z = eval(z) + mp.dps = maxdps + 50 + if result is None: + gamma_val = gamma(z) + loggamma_val = loggamma(z) + factorial_val = factorial(z) + rgamma_val = rgamma(z) + else: + loggamma_val = eval(result) + gamma_val = exp(loggamma_val) + factorial_val = z * gamma_val + rgamma_val = 1/gamma_val + for dps in [5, 10, 15, 25, 40, 60, 90, 120, 250, 600, 1000, 1800, 3600]: + if dps > maxdps: + break + mp.dps = dps + print("dps = %s" % dps) + check("gamma", gamma, z, gamma_val) + check("rgamma", rgamma, z, rgamma_val) + check("loggamma", loggamma, z, loggamma_val) + check("factorial", factorial, z, factorial_val) + print() + mp.dps = 15 + +testcases = [] + +# Basic values +for n in list(range(1,200)) + list(range(201,2000,17)): + testcases.append(["%s" % n, None]) +for n in range(-200,200): + testcases.append(["%s+0.5" % n, None]) + testcases.append(["%s+0.37" % n, None]) + +testcases += [\ +["(0.1+1j)", None], +["(-0.1+1j)", None], +["(0.1-1j)", None], +["(-0.1-1j)", None], +["10j", None], +["-10j", None], +["100j", None], +["10000j", None], +["-10000000j", None], +["(10**100)*j", None], +["125+(10**100)*j", None], +["-125+(10**100)*j", None], +["(10**10)*(1+j)", None], +["(10**10)*(-1+j)", None], +["(10**100)*(1+j)", None], +["(10**100)*(-1+j)", None], +["(1.5-1j)", None], +["(6+4j)", None], +["(4+1j)", None], +["(3.5+2j)", None], +["(1.5-1j)", None], +["(-6-4j)", None], +["(-2-3j)", None], +["(-2.5-2j)", None], +["(4+1j)", None], +["(3+3j)", None], +["(2-2j)", None], +["1", "0"], +["2", "0"], +["3", "log(2)"], +["4", "log(6)"], +["5", "log(24)"], +["0.5", "log(pi)/2"], +["1.5", "log(sqrt(pi)/2)"], +["2.5", "log(3*sqrt(pi)/4)"], +["mpf('0.37')", None], +["0.25", "log(sqrt(2*sqrt(2*pi**3)/agm(1,sqrt(2))))"], +["-0.4", None], +["mpf('-1.9')", None], +["mpf('12.8')", None], +["mpf('33.7')", None], +["mpf('95.2')", None], +["mpf('160.3')", None], +["mpf('2057.8')", None], +["25", "log(ifac(24))"], +["80", "log(ifac(79))"], +["500", "log(ifac(500-1))"], +["8000", "log(ifac(8000-1))"], +["8000.5", None], +["mpf('8000.1')", None], +["mpf('1.37e10')", None], +["mpf('1.37e10')*(1+j)", None], +["mpf('1.37e10')*(-1+j)", None], +["mpf('1.37e10')*(-1-j)", None], +["mpf('1.37e10')*(-1+j)", None], +["mpf('1.37e100')", None], +["mpf('1.37e100')*(1+j)", None], +["mpf('1.37e100')*(-1+j)", None], +["mpf('1.37e100')*(-1-j)", None], +["mpf('1.37e100')*(-1+j)", None], +["3+4j", +"mpc('" +"-1.7566267846037841105306041816232757851567066070613445016197619371316057169" +"4723618263960834804618463052988607348289672535780644470689771115236512106002" +"5970873471563240537307638968509556191696167970488390423963867031934333890838" +"8009531786948197210025029725361069435208930363494971027388382086721660805397" +"9163230643216054580167976201709951509519218635460317367338612500626714783631" +"7498317478048447525674016344322545858832610325861086336204591943822302971823" +"5161814175530618223688296232894588415495615809337292518431903058265147109853" +"1710568942184987827643886816200452860853873815413367529829631430146227470517" +"6579967222200868632179482214312673161276976117132204633283806161971389519137" +"1243359764435612951384238091232760634271570950240717650166551484551654327989" +"9360285030081716934130446150245110557038117075172576825490035434069388648124" +"6678152254554001586736120762641422590778766100376515737713938521275749049949" +"1284143906816424244705094759339932733567910991920631339597278805393743140853" +"391550313363278558195609260225928','" +"4.74266443803465792819488940755002274088830335171164611359052405215840070271" +"5906813009373171139767051863542508136875688550817670379002790304870822775498" +"2809996675877564504192565392367259119610438951593128982646945990372179860613" +"4294436498090428077839141927485901735557543641049637962003652638924845391650" +"9546290137755550107224907606529385248390667634297183361902055842228798984200" +"9591180450211798341715874477629099687609819466457990642030707080894518168924" +"6805549314043258530272479246115112769957368212585759640878745385160943755234" +"9398036774908108204370323896757543121853650025529763655312360354244898913463" +"7115955702828838923393113618205074162812089732064414530813087483533203244056" +"0546577484241423134079056537777170351934430586103623577814746004431994179990" +"5318522939077992613855205801498201930221975721246498720895122345420698451980" +"0051215797310305885845964334761831751370672996984756815410977750799748813563" +"8784405288158432214886648743541773208808731479748217023665577802702269468013" +"673719173759245720489020315779001')"], +] + +for z in [4, 14, 34, 64]: + testcases.append(["(2+j)*%s/3" % z, None]) + testcases.append(["(-2+j)*%s/3" % z, None]) + testcases.append(["(1+2*j)*%s/3" % z, None]) + testcases.append(["(2-j)*%s/3" % z, None]) + testcases.append(["(20+j)*%s/3" % z, None]) + testcases.append(["(-20+j)*%s/3" % z, None]) + testcases.append(["(1+20*j)*%s/3" % z, None]) + testcases.append(["(20-j)*%s/3" % z, None]) + testcases.append(["(200+j)*%s/3" % z, None]) + testcases.append(["(-200+j)*%s/3" % z, None]) + testcases.append(["(1+200*j)*%s/3" % z, None]) + testcases.append(["(200-j)*%s/3" % z, None]) + +# Poles +for n in [0,1,2,3,4,25,-1,-2,-3,-4,-20,-21,-50,-51,-200,-201,-20000,-20001]: + for t in ['1e-5', '1e-20', '1e-100', '1e-10000']: + testcases.append(["fadd(%s,'%s',exact=True)" % (n, t), None]) + testcases.append(["fsub(%s,'%s',exact=True)" % (n, t), None]) + testcases.append(["fadd(%s,'%sj',exact=True)" % (n, t), None]) + testcases.append(["fsub(%s,'%sj',exact=True)" % (n, t), None]) + +if __name__ == "__main__": + from timeit import default_timer as clock + tot_time = 0.0 + for case in testcases: + t1 = clock() + testcase(case) + t2 = clock() + print("Test time:", t2-t1) + print() + tot_time += (t2-t1) + print("Total time:", tot_time) + print("Errors:", errcount) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/extratest_zeta.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/extratest_zeta.py new file mode 100644 index 0000000000000000000000000000000000000000..582b3d9cbd956b9cdf94309e0e718371fe716101 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/extratest_zeta.py @@ -0,0 +1,30 @@ +from mpmath import zetazero +from timeit import default_timer as clock + +def test_zetazero(): + cases = [\ + (399999999, 156762524.6750591511), + (241389216, 97490234.2276711795), + (526196239, 202950727.691229534), + (542964976, 209039046.578535272), + (1048449112, 388858885.231056486), + (1048449113, 388858885.384337406), + (1048449114, 388858886.002285122), + (1048449115, 388858886.00239369), + (1048449116, 388858886.690745053) + ] + for n, v in cases: + print(n, v) + t1 = clock() + ok = zetazero(n).ae(complex(0.5,v)) + t2 = clock() + print("ok =", ok, ("(time = %s)" % round(t2-t1,3))) + print("Now computing two huge zeros (this may take hours)") + print("Computing zetazero(8637740722917)") + ok = zetazero(8637740722917).ae(complex(0.5,2124447368584.39296466152)) + print("ok =", ok) + ok = zetazero(8637740722918).ae(complex(0.5,2124447368584.39298170604)) + print("ok =", ok) + +if __name__ == "__main__": + test_zetazero() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/runtests.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/runtests.py new file mode 100644 index 0000000000000000000000000000000000000000..70fde272fdc0e05e3d8951edddca380bd36139ab --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/runtests.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python + +""" +python runtests.py -py + Use py.test to run tests (more useful for debugging) + +python runtests.py -coverage + Generate test coverage report. Statistics are written to /tmp + +python runtests.py -profile + Generate profile stats (this is much slower) + +python runtests.py -nogmpy + Run tests without using GMPY even if it exists + +python runtests.py -strict + Enforce extra tests in normalize() + +python runtests.py -local + Insert '../..' at the beginning of sys.path to use local mpmath + +python runtests.py -skip ... + Skip tests from the listed modules + +Additional arguments are used to filter the tests to run. Only files that have +one of the arguments in their name are executed. + +""" + +import sys, os, traceback + +profile = False +if "-profile" in sys.argv: + sys.argv.remove('-profile') + profile = True + +coverage = False +if "-coverage" in sys.argv: + sys.argv.remove('-coverage') + coverage = True + +if "-nogmpy" in sys.argv: + sys.argv.remove('-nogmpy') + os.environ['MPMATH_NOGMPY'] = 'Y' + +if "-strict" in sys.argv: + sys.argv.remove('-strict') + os.environ['MPMATH_STRICT'] = 'Y' + +if "-local" in sys.argv: + sys.argv.remove('-local') + importdir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), + '../..')) +else: + importdir = '' + +# TODO: add a flag for this +testdir = '' + +def testit(importdir='', testdir='', exit_on_fail=False): + """Run all tests in testdir while importing from importdir.""" + if importdir: + sys.path.insert(1, importdir) + if testdir: + sys.path.insert(1, testdir) + import os.path + import mpmath + print("mpmath imported from %s" % os.path.dirname(mpmath.__file__)) + print("mpmath backend: %s" % mpmath.libmp.backend.BACKEND) + print("mpmath mp class: %s" % repr(mpmath.mp)) + print("mpmath version: %s" % mpmath.__version__) + print("Python version: %s" % sys.version) + print("") + if "-py" in sys.argv: + sys.argv.remove('-py') + import py + py.test.cmdline.main() + else: + import glob + from timeit import default_timer as clock + modules = [] + args = sys.argv[1:] + excluded = [] + if '-skip' in args: + excluded = args[args.index('-skip')+1:] + args = args[:args.index('-skip')] + # search for tests in directory of this file if not otherwise specified + if not testdir: + pattern = os.path.dirname(sys.argv[0]) + else: + pattern = testdir + if pattern: + pattern += '/' + pattern += 'test*.py' + # look for tests (respecting specified filter) + for f in glob.glob(pattern): + name = os.path.splitext(os.path.basename(f))[0] + # If run as a script, only run tests given as args, if any are given + if args and __name__ == "__main__": + ok = False + for arg in args: + if arg in name: + ok = True + break + if not ok: + continue + elif name in excluded: + continue + module = __import__(name) + priority = module.__dict__.get('priority', 100) + if priority == 666: + modules = [[priority, name, module]] + break + modules.append([priority, name, module]) + # execute tests + modules.sort() + tstart = clock() + for priority, name, module in modules: + print(name) + for f in sorted(module.__dict__.keys()): + if f.startswith('test_'): + if coverage and ('numpy' in f): + continue + sys.stdout.write(" " + f[5:].ljust(25) + " ") + t1 = clock() + try: + module.__dict__[f]() + except: + etype, evalue, trb = sys.exc_info() + if etype in (KeyboardInterrupt, SystemExit): + raise + print("") + print("TEST FAILED!") + print("") + traceback.print_exc() + if exit_on_fail: + return + t2 = clock() + print("ok " + " " + ("%.7f" % (t2-t1)) + " s") + tend = clock() + print("") + print("finished tests in " + ("%.2f" % (tend-tstart)) + " seconds") + # clean sys.path + if importdir: + sys.path.remove(importdir) + if testdir: + sys.path.remove(testdir) + +if __name__ == '__main__': + if profile: + import cProfile + cProfile.run("testit('%s', '%s')" % (importdir, testdir), sort=1) + elif coverage: + import trace + tracer = trace.Trace(ignoredirs=[sys.prefix, sys.exec_prefix], + trace=0, count=1) + tracer.run('testit(importdir, testdir)') + r = tracer.results() + r.write_results(show_missing=True, summary=True, coverdir="/tmp") + else: + testit(importdir, testdir) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_basic_ops.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_basic_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..f577c7fa9f9734876b6767f6cc21144df305d82f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_basic_ops.py @@ -0,0 +1,451 @@ +import mpmath +from mpmath import * +from mpmath.libmp import * +import random +import sys + +try: + long = long +except NameError: + long = int + +def test_type_compare(): + assert mpf(2) == mpc(2,0) + assert mpf(0) == mpc(0) + assert mpf(2) != mpc(2, 0.00001) + assert mpf(2) == 2.0 + assert mpf(2) != 3.0 + assert mpf(2) == 2 + assert mpf(2) != '2.0' + assert mpc(2) != '2.0' + +def test_add(): + assert mpf(2.5) + mpf(3) == 5.5 + assert mpf(2.5) + 3 == 5.5 + assert mpf(2.5) + 3.0 == 5.5 + assert 3 + mpf(2.5) == 5.5 + assert 3.0 + mpf(2.5) == 5.5 + assert (3+0j) + mpf(2.5) == 5.5 + assert mpc(2.5) + mpf(3) == 5.5 + assert mpc(2.5) + 3 == 5.5 + assert mpc(2.5) + 3.0 == 5.5 + assert mpc(2.5) + (3+0j) == 5.5 + assert 3 + mpc(2.5) == 5.5 + assert 3.0 + mpc(2.5) == 5.5 + assert (3+0j) + mpc(2.5) == 5.5 + +def test_sub(): + assert mpf(2.5) - mpf(3) == -0.5 + assert mpf(2.5) - 3 == -0.5 + assert mpf(2.5) - 3.0 == -0.5 + assert 3 - mpf(2.5) == 0.5 + assert 3.0 - mpf(2.5) == 0.5 + assert (3+0j) - mpf(2.5) == 0.5 + assert mpc(2.5) - mpf(3) == -0.5 + assert mpc(2.5) - 3 == -0.5 + assert mpc(2.5) - 3.0 == -0.5 + assert mpc(2.5) - (3+0j) == -0.5 + assert 3 - mpc(2.5) == 0.5 + assert 3.0 - mpc(2.5) == 0.5 + assert (3+0j) - mpc(2.5) == 0.5 + +def test_mul(): + assert mpf(2.5) * mpf(3) == 7.5 + assert mpf(2.5) * 3 == 7.5 + assert mpf(2.5) * 3.0 == 7.5 + assert 3 * mpf(2.5) == 7.5 + assert 3.0 * mpf(2.5) == 7.5 + assert (3+0j) * mpf(2.5) == 7.5 + assert mpc(2.5) * mpf(3) == 7.5 + assert mpc(2.5) * 3 == 7.5 + assert mpc(2.5) * 3.0 == 7.5 + assert mpc(2.5) * (3+0j) == 7.5 + assert 3 * mpc(2.5) == 7.5 + assert 3.0 * mpc(2.5) == 7.5 + assert (3+0j) * mpc(2.5) == 7.5 + +def test_div(): + assert mpf(6) / mpf(3) == 2.0 + assert mpf(6) / 3 == 2.0 + assert mpf(6) / 3.0 == 2.0 + assert 6 / mpf(3) == 2.0 + assert 6.0 / mpf(3) == 2.0 + assert (6+0j) / mpf(3.0) == 2.0 + assert mpc(6) / mpf(3) == 2.0 + assert mpc(6) / 3 == 2.0 + assert mpc(6) / 3.0 == 2.0 + assert mpc(6) / (3+0j) == 2.0 + assert 6 / mpc(3) == 2.0 + assert 6.0 / mpc(3) == 2.0 + assert (6+0j) / mpc(3) == 2.0 + +def test_pow(): + assert mpf(6) ** mpf(3) == 216.0 + assert mpf(6) ** 3 == 216.0 + assert mpf(6) ** 3.0 == 216.0 + assert 6 ** mpf(3) == 216.0 + assert 6.0 ** mpf(3) == 216.0 + assert (6+0j) ** mpf(3.0) == 216.0 + assert mpc(6) ** mpf(3) == 216.0 + assert mpc(6) ** 3 == 216.0 + assert mpc(6) ** 3.0 == 216.0 + assert mpc(6) ** (3+0j) == 216.0 + assert 6 ** mpc(3) == 216.0 + assert 6.0 ** mpc(3) == 216.0 + assert (6+0j) ** mpc(3) == 216.0 + +def test_mixed_misc(): + assert 1 + mpf(3) == mpf(3) + 1 == 4 + assert 1 - mpf(3) == -(mpf(3) - 1) == -2 + assert 3 * mpf(2) == mpf(2) * 3 == 6 + assert 6 / mpf(2) == mpf(6) / 2 == 3 + assert 1.0 + mpf(3) == mpf(3) + 1.0 == 4 + assert 1.0 - mpf(3) == -(mpf(3) - 1.0) == -2 + assert 3.0 * mpf(2) == mpf(2) * 3.0 == 6 + assert 6.0 / mpf(2) == mpf(6) / 2.0 == 3 + +def test_add_misc(): + mp.dps = 15 + assert mpf(4) + mpf(-70) == -66 + assert mpf(1) + mpf(1.1)/80 == 1 + 1.1/80 + assert mpf((1, 10000000000)) + mpf(3) == mpf((1, 10000000000)) + assert mpf(3) + mpf((1, 10000000000)) == mpf((1, 10000000000)) + assert mpf((1, -10000000000)) + mpf(3) == mpf(3) + assert mpf(3) + mpf((1, -10000000000)) == mpf(3) + assert mpf(1) + 1e-15 != 1 + assert mpf(1) + 1e-20 == 1 + assert mpf(1.07e-22) + 0 == mpf(1.07e-22) + assert mpf(0) + mpf(1.07e-22) == mpf(1.07e-22) + +def test_complex_misc(): + # many more tests needed + assert 1 + mpc(2) == 3 + assert not mpc(2).ae(2 + 1e-13) + assert mpc(2+1e-15j).ae(2) + +def test_complex_zeros(): + for a in [0,2]: + for b in [0,3]: + for c in [0,4]: + for d in [0,5]: + assert mpc(a,b)*mpc(c,d) == complex(a,b)*complex(c,d) + +def test_hash(): + for i in range(-256, 256): + assert hash(mpf(i)) == hash(i) + assert hash(mpf(0.5)) == hash(0.5) + assert hash(mpc(2,3)) == hash(2+3j) + # Check that this doesn't fail + assert hash(inf) + # Check that overflow doesn't assign equal hashes to large numbers + assert hash(mpf('1e1000')) != hash('1e10000') + assert hash(mpc(100,'1e1000')) != hash(mpc(200,'1e1000')) + from mpmath.rational import mpq + assert hash(mp.mpq(1,3)) + assert hash(mp.mpq(0,1)) == 0 + assert hash(mp.mpq(-1,1)) == hash(-1) + assert hash(mp.mpq(1,1)) == hash(1) + assert hash(mp.mpq(5,1)) == hash(5) + assert hash(mp.mpq(1,2)) == hash(0.5) + if sys.version_info >= (3, 2): + assert hash(mpf(1)*2**2000) == hash(2**2000) + assert hash(mpf(1)/2**2000) == hash(mpq(1,2**2000)) + +# Advanced rounding test +def test_add_rounding(): + mp.dps = 15 + a = from_float(1e-50) + assert mpf_sub(mpf_add(fone, a, 53, round_up), fone, 53, round_up) == from_float(2.2204460492503131e-16) + assert mpf_sub(fone, a, 53, round_up) == fone + assert mpf_sub(fone, mpf_sub(fone, a, 53, round_down), 53, round_down) == from_float(1.1102230246251565e-16) + assert mpf_add(fone, a, 53, round_down) == fone + +def test_almost_equal(): + assert mpf(1.2).ae(mpf(1.20000001), 1e-7) + assert not mpf(1.2).ae(mpf(1.20000001), 1e-9) + assert not mpf(-0.7818314824680298).ae(mpf(-0.774695868667929)) + +def test_arithmetic_functions(): + import operator + ops = [(operator.add, fadd), (operator.sub, fsub), (operator.mul, fmul), + (operator.truediv, fdiv)] + a = mpf(0.27) + b = mpf(1.13) + c = mpc(0.51+2.16j) + d = mpc(1.08-0.99j) + for x in [a,b,c,d]: + for y in [a,b,c,d]: + for op, fop in ops: + if fop is not fdiv: + mp.prec = 200 + z0 = op(x,y) + mp.prec = 60 + z1 = op(x,y) + mp.prec = 53 + z2 = op(x,y) + assert fop(x, y, prec=60) == z1 + assert fop(x, y) == z2 + if fop is not fdiv: + assert fop(x, y, prec=inf) == z0 + assert fop(x, y, dps=inf) == z0 + assert fop(x, y, exact=True) == z0 + assert fneg(fneg(z1, exact=True), prec=inf) == z1 + assert fneg(z1) == -(+z1) + mp.dps = 15 + +def test_exact_integer_arithmetic(): + # XXX: re-fix this so that all operations are tested with all rounding modes + random.seed(0) + for prec in [6, 10, 25, 40, 100, 250, 725]: + for rounding in ['d', 'u', 'f', 'c', 'n']: + mp.dps = prec + M = 10**(prec-2) + M2 = 10**(prec//2-2) + for i in range(10): + a = random.randint(-M, M) + b = random.randint(-M, M) + assert mpf(a, rounding=rounding) == a + assert int(mpf(a, rounding=rounding)) == a + assert int(mpf(str(a), rounding=rounding)) == a + assert mpf(a) + mpf(b) == a + b + assert mpf(a) - mpf(b) == a - b + assert -mpf(a) == -a + a = random.randint(-M2, M2) + b = random.randint(-M2, M2) + assert mpf(a) * mpf(b) == a*b + assert mpf_mul(from_int(a), from_int(b), mp.prec, rounding) == from_int(a*b) + mp.dps = 15 + +def test_odd_int_bug(): + assert to_int(from_int(3), round_nearest) == 3 + +def test_str_1000_digits(): + mp.dps = 1001 + # last digit may be wrong + assert str(mpf(2)**0.5)[-10:-1] == '9518488472'[:9] + assert str(pi)[-10:-1] == '2164201989'[:9] + mp.dps = 15 + +def test_str_10000_digits(): + mp.dps = 10001 + # last digit may be wrong + assert str(mpf(2)**0.5)[-10:-1] == '5873258351'[:9] + assert str(pi)[-10:-1] == '5256375678'[:9] + mp.dps = 15 + +def test_monitor(): + f = lambda x: x**2 + a = [] + b = [] + g = monitor(f, a.append, b.append) + assert g(3) == 9 + assert g(4) == 16 + assert a[0] == ((3,), {}) + assert b[0] == 9 + +def test_nint_distance(): + assert nint_distance(mpf(-3)) == (-3, -inf) + assert nint_distance(mpc(-3)) == (-3, -inf) + assert nint_distance(mpf(-3.1)) == (-3, -3) + assert nint_distance(mpf(-3.01)) == (-3, -6) + assert nint_distance(mpf(-3.001)) == (-3, -9) + assert nint_distance(mpf(-3.0001)) == (-3, -13) + assert nint_distance(mpf(-2.9)) == (-3, -3) + assert nint_distance(mpf(-2.99)) == (-3, -6) + assert nint_distance(mpf(-2.999)) == (-3, -9) + assert nint_distance(mpf(-2.9999)) == (-3, -13) + assert nint_distance(mpc(-3+0.1j)) == (-3, -3) + assert nint_distance(mpc(-3+0.01j)) == (-3, -6) + assert nint_distance(mpc(-3.1+0.1j)) == (-3, -3) + assert nint_distance(mpc(-3.01+0.01j)) == (-3, -6) + assert nint_distance(mpc(-3.001+0.001j)) == (-3, -9) + assert nint_distance(mpf(0)) == (0, -inf) + assert nint_distance(mpf(0.01)) == (0, -6) + assert nint_distance(mpf('1e-100')) == (0, -332) + +def test_floor_ceil_nint_frac(): + mp.dps = 15 + for n in range(-10,10): + assert floor(n) == n + assert floor(n+0.5) == n + assert ceil(n) == n + assert ceil(n+0.5) == n+1 + assert nint(n) == n + # nint rounds to even + if n % 2 == 1: + assert nint(n+0.5) == n+1 + else: + assert nint(n+0.5) == n + assert floor(inf) == inf + assert floor(ninf) == ninf + assert isnan(floor(nan)) + assert ceil(inf) == inf + assert ceil(ninf) == ninf + assert isnan(ceil(nan)) + assert nint(inf) == inf + assert nint(ninf) == ninf + assert isnan(nint(nan)) + assert floor(0.1) == 0 + assert floor(0.9) == 0 + assert floor(-0.1) == -1 + assert floor(-0.9) == -1 + assert floor(10000000000.1) == 10000000000 + assert floor(10000000000.9) == 10000000000 + assert floor(-10000000000.1) == -10000000000-1 + assert floor(-10000000000.9) == -10000000000-1 + assert floor(1e-100) == 0 + assert floor(-1e-100) == -1 + assert floor(1e100) == 1e100 + assert floor(-1e100) == -1e100 + assert ceil(0.1) == 1 + assert ceil(0.9) == 1 + assert ceil(-0.1) == 0 + assert ceil(-0.9) == 0 + assert ceil(10000000000.1) == 10000000000+1 + assert ceil(10000000000.9) == 10000000000+1 + assert ceil(-10000000000.1) == -10000000000 + assert ceil(-10000000000.9) == -10000000000 + assert ceil(1e-100) == 1 + assert ceil(-1e-100) == 0 + assert ceil(1e100) == 1e100 + assert ceil(-1e100) == -1e100 + assert nint(0.1) == 0 + assert nint(0.9) == 1 + assert nint(-0.1) == 0 + assert nint(-0.9) == -1 + assert nint(10000000000.1) == 10000000000 + assert nint(10000000000.9) == 10000000000+1 + assert nint(-10000000000.1) == -10000000000 + assert nint(-10000000000.9) == -10000000000-1 + assert nint(1e-100) == 0 + assert nint(-1e-100) == 0 + assert nint(1e100) == 1e100 + assert nint(-1e100) == -1e100 + assert floor(3.2+4.6j) == 3+4j + assert ceil(3.2+4.6j) == 4+5j + assert nint(3.2+4.6j) == 3+5j + for n in range(-10,10): + assert frac(n) == 0 + assert frac(0.25) == 0.25 + assert frac(1.25) == 0.25 + assert frac(2.25) == 0.25 + assert frac(-0.25) == 0.75 + assert frac(-1.25) == 0.75 + assert frac(-2.25) == 0.75 + assert frac('1e100000000000000') == 0 + u = mpf('1e-100000000000000') + assert frac(u) == u + assert frac(-u) == 1 # rounding! + u = mpf('1e-400') + assert frac(-u, prec=0) == fsub(1, u, exact=True) + assert frac(3.25+4.75j) == 0.25+0.75j + +def test_isnan_etc(): + from mpmath.rational import mpq + assert isnan(nan) == True + assert isnan(3) == False + assert isnan(mpf(3)) == False + assert isnan(inf) == False + assert isnan(mpc(2,nan)) == True + assert isnan(mpc(2,nan)) == True + assert isnan(mpc(nan,nan)) == True + assert isnan(mpc(2,2)) == False + assert isnan(mpc(nan,inf)) == True + assert isnan(mpc(inf,inf)) == False + assert isnan(mpq((3,2))) == False + assert isnan(mpq((0,1))) == False + assert isinf(inf) == True + assert isinf(-inf) == True + assert isinf(3) == False + assert isinf(nan) == False + assert isinf(3+4j) == False + assert isinf(mpc(inf)) == True + assert isinf(mpc(3,inf)) == True + assert isinf(mpc(inf,3)) == True + assert isinf(mpc(inf,inf)) == True + assert isinf(mpc(nan,inf)) == True + assert isinf(mpc(inf,nan)) == True + assert isinf(mpc(nan,nan)) == False + assert isinf(mpq((3,2))) == False + assert isinf(mpq((0,1))) == False + assert isnormal(3) == True + assert isnormal(3.5) == True + assert isnormal(mpf(3.5)) == True + assert isnormal(0) == False + assert isnormal(mpf(0)) == False + assert isnormal(0.0) == False + assert isnormal(inf) == False + assert isnormal(-inf) == False + assert isnormal(nan) == False + assert isnormal(float(inf)) == False + assert isnormal(mpc(0,0)) == False + assert isnormal(mpc(3,0)) == True + assert isnormal(mpc(0,3)) == True + assert isnormal(mpc(3,3)) == True + assert isnormal(mpc(0,nan)) == False + assert isnormal(mpc(0,inf)) == False + assert isnormal(mpc(3,nan)) == False + assert isnormal(mpc(3,inf)) == False + assert isnormal(mpc(3,-inf)) == False + assert isnormal(mpc(nan,0)) == False + assert isnormal(mpc(inf,0)) == False + assert isnormal(mpc(nan,3)) == False + assert isnormal(mpc(inf,3)) == False + assert isnormal(mpc(inf,nan)) == False + assert isnormal(mpc(nan,inf)) == False + assert isnormal(mpc(nan,nan)) == False + assert isnormal(mpc(inf,inf)) == False + assert isnormal(mpq((3,2))) == True + assert isnormal(mpq((0,1))) == False + assert isint(3) == True + assert isint(0) == True + assert isint(long(3)) == True + assert isint(long(0)) == True + assert isint(mpf(3)) == True + assert isint(mpf(0)) == True + assert isint(mpf(-3)) == True + assert isint(mpf(3.2)) == False + assert isint(3.2) == False + assert isint(nan) == False + assert isint(inf) == False + assert isint(-inf) == False + assert isint(mpc(0)) == True + assert isint(mpc(3)) == True + assert isint(mpc(3.2)) == False + assert isint(mpc(3,inf)) == False + assert isint(mpc(inf)) == False + assert isint(mpc(3,2)) == False + assert isint(mpc(0,2)) == False + assert isint(mpc(3,2),gaussian=True) == True + assert isint(mpc(3,0),gaussian=True) == True + assert isint(mpc(0,3),gaussian=True) == True + assert isint(3+4j) == False + assert isint(3+4j, gaussian=True) == True + assert isint(3+0j) == True + assert isint(mpq((3,2))) == False + assert isint(mpq((3,9))) == False + assert isint(mpq((9,3))) == True + assert isint(mpq((0,4))) == True + assert isint(mpq((1,1))) == True + assert isint(mpq((-1,1))) == True + assert mp.isnpint(0) == True + assert mp.isnpint(1) == False + assert mp.isnpint(-1) == True + assert mp.isnpint(-1.1) == False + assert mp.isnpint(-1.0) == True + assert mp.isnpint(mp.mpq(1,2)) == False + assert mp.isnpint(mp.mpq(-1,2)) == False + assert mp.isnpint(mp.mpq(-3,1)) == True + assert mp.isnpint(mp.mpq(0,1)) == True + assert mp.isnpint(mp.mpq(1,1)) == False + assert mp.isnpint(0+0j) == True + assert mp.isnpint(-1+0j) == True + assert mp.isnpint(-1.1+0j) == False + assert mp.isnpint(-1+0.1j) == False + assert mp.isnpint(0+0.1j) == False + + +def test_issue_438(): + assert mpf(finf) == mpf('inf') + assert mpf(fninf) == mpf('-inf') + assert mpf(fnan)._mpf_ == mpf('nan')._mpf_ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_bitwise.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_bitwise.py new file mode 100644 index 0000000000000000000000000000000000000000..4f61b69fc8819cf275abaedd98847c58c3b5924a --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_bitwise.py @@ -0,0 +1,188 @@ +""" +Test bit-level integer and mpf operations +""" + +from mpmath import * +from mpmath.libmp import * + +def test_bitcount(): + assert bitcount(0) == 0 + assert bitcount(1) == 1 + assert bitcount(7) == 3 + assert bitcount(8) == 4 + assert bitcount(2**100) == 101 + assert bitcount(2**100-1) == 100 + +def test_trailing(): + assert trailing(0) == 0 + assert trailing(1) == 0 + assert trailing(2) == 1 + assert trailing(7) == 0 + assert trailing(8) == 3 + assert trailing(2**100) == 100 + assert trailing(2**100-1) == 0 + +def test_round_down(): + assert from_man_exp(0, -4, 4, round_down)[:3] == (0, 0, 0) + assert from_man_exp(0xf0, -4, 4, round_down)[:3] == (0, 15, 0) + assert from_man_exp(0xf1, -4, 4, round_down)[:3] == (0, 15, 0) + assert from_man_exp(0xff, -4, 4, round_down)[:3] == (0, 15, 0) + assert from_man_exp(-0xf0, -4, 4, round_down)[:3] == (1, 15, 0) + assert from_man_exp(-0xf1, -4, 4, round_down)[:3] == (1, 15, 0) + assert from_man_exp(-0xff, -4, 4, round_down)[:3] == (1, 15, 0) + +def test_round_up(): + assert from_man_exp(0, -4, 4, round_up)[:3] == (0, 0, 0) + assert from_man_exp(0xf0, -4, 4, round_up)[:3] == (0, 15, 0) + assert from_man_exp(0xf1, -4, 4, round_up)[:3] == (0, 1, 4) + assert from_man_exp(0xff, -4, 4, round_up)[:3] == (0, 1, 4) + assert from_man_exp(-0xf0, -4, 4, round_up)[:3] == (1, 15, 0) + assert from_man_exp(-0xf1, -4, 4, round_up)[:3] == (1, 1, 4) + assert from_man_exp(-0xff, -4, 4, round_up)[:3] == (1, 1, 4) + +def test_round_floor(): + assert from_man_exp(0, -4, 4, round_floor)[:3] == (0, 0, 0) + assert from_man_exp(0xf0, -4, 4, round_floor)[:3] == (0, 15, 0) + assert from_man_exp(0xf1, -4, 4, round_floor)[:3] == (0, 15, 0) + assert from_man_exp(0xff, -4, 4, round_floor)[:3] == (0, 15, 0) + assert from_man_exp(-0xf0, -4, 4, round_floor)[:3] == (1, 15, 0) + assert from_man_exp(-0xf1, -4, 4, round_floor)[:3] == (1, 1, 4) + assert from_man_exp(-0xff, -4, 4, round_floor)[:3] == (1, 1, 4) + +def test_round_ceiling(): + assert from_man_exp(0, -4, 4, round_ceiling)[:3] == (0, 0, 0) + assert from_man_exp(0xf0, -4, 4, round_ceiling)[:3] == (0, 15, 0) + assert from_man_exp(0xf1, -4, 4, round_ceiling)[:3] == (0, 1, 4) + assert from_man_exp(0xff, -4, 4, round_ceiling)[:3] == (0, 1, 4) + assert from_man_exp(-0xf0, -4, 4, round_ceiling)[:3] == (1, 15, 0) + assert from_man_exp(-0xf1, -4, 4, round_ceiling)[:3] == (1, 15, 0) + assert from_man_exp(-0xff, -4, 4, round_ceiling)[:3] == (1, 15, 0) + +def test_round_nearest(): + assert from_man_exp(0, -4, 4, round_nearest)[:3] == (0, 0, 0) + assert from_man_exp(0xf0, -4, 4, round_nearest)[:3] == (0, 15, 0) + assert from_man_exp(0xf7, -4, 4, round_nearest)[:3] == (0, 15, 0) + assert from_man_exp(0xf8, -4, 4, round_nearest)[:3] == (0, 1, 4) # 1111.1000 -> 10000.0 + assert from_man_exp(0xf9, -4, 4, round_nearest)[:3] == (0, 1, 4) # 1111.1001 -> 10000.0 + assert from_man_exp(0xe8, -4, 4, round_nearest)[:3] == (0, 7, 1) # 1110.1000 -> 1110.0 + assert from_man_exp(0xe9, -4, 4, round_nearest)[:3] == (0, 15, 0) # 1110.1001 -> 1111.0 + assert from_man_exp(-0xf0, -4, 4, round_nearest)[:3] == (1, 15, 0) + assert from_man_exp(-0xf7, -4, 4, round_nearest)[:3] == (1, 15, 0) + assert from_man_exp(-0xf8, -4, 4, round_nearest)[:3] == (1, 1, 4) + assert from_man_exp(-0xf9, -4, 4, round_nearest)[:3] == (1, 1, 4) + assert from_man_exp(-0xe8, -4, 4, round_nearest)[:3] == (1, 7, 1) + assert from_man_exp(-0xe9, -4, 4, round_nearest)[:3] == (1, 15, 0) + +def test_rounding_bugs(): + # 1 less than power-of-two cases + assert from_man_exp(72057594037927935, -56, 53, round_up) == (0, 1, 0, 1) + assert from_man_exp(73786976294838205979, -65, 53, round_nearest) == (0, 1, 1, 1) + assert from_man_exp(31, 0, 4, round_up) == (0, 1, 5, 1) + assert from_man_exp(-31, 0, 4, round_floor) == (1, 1, 5, 1) + assert from_man_exp(255, 0, 7, round_up) == (0, 1, 8, 1) + assert from_man_exp(-255, 0, 7, round_floor) == (1, 1, 8, 1) + +def test_rounding_issue_200(): + a = from_man_exp(9867,-100) + b = from_man_exp(9867,-200) + c = from_man_exp(-1,0) + z = (1, 1023, -10, 10) + assert mpf_add(a, c, 10, 'd') == z + assert mpf_add(b, c, 10, 'd') == z + assert mpf_add(c, a, 10, 'd') == z + assert mpf_add(c, b, 10, 'd') == z + +def test_perturb(): + a = fone + b = from_float(0.99999999999999989) + c = from_float(1.0000000000000002) + assert mpf_perturb(a, 0, 53, round_nearest) == a + assert mpf_perturb(a, 1, 53, round_nearest) == a + assert mpf_perturb(a, 0, 53, round_up) == c + assert mpf_perturb(a, 0, 53, round_ceiling) == c + assert mpf_perturb(a, 0, 53, round_down) == a + assert mpf_perturb(a, 0, 53, round_floor) == a + assert mpf_perturb(a, 1, 53, round_up) == a + assert mpf_perturb(a, 1, 53, round_ceiling) == a + assert mpf_perturb(a, 1, 53, round_down) == b + assert mpf_perturb(a, 1, 53, round_floor) == b + a = mpf_neg(a) + b = mpf_neg(b) + c = mpf_neg(c) + assert mpf_perturb(a, 0, 53, round_nearest) == a + assert mpf_perturb(a, 1, 53, round_nearest) == a + assert mpf_perturb(a, 0, 53, round_up) == a + assert mpf_perturb(a, 0, 53, round_floor) == a + assert mpf_perturb(a, 0, 53, round_down) == b + assert mpf_perturb(a, 0, 53, round_ceiling) == b + assert mpf_perturb(a, 1, 53, round_up) == c + assert mpf_perturb(a, 1, 53, round_floor) == c + assert mpf_perturb(a, 1, 53, round_down) == a + assert mpf_perturb(a, 1, 53, round_ceiling) == a + +def test_add_exact(): + ff = from_float + assert mpf_add(ff(3.0), ff(2.5)) == ff(5.5) + assert mpf_add(ff(3.0), ff(-2.5)) == ff(0.5) + assert mpf_add(ff(-3.0), ff(2.5)) == ff(-0.5) + assert mpf_add(ff(-3.0), ff(-2.5)) == ff(-5.5) + assert mpf_sub(mpf_add(fone, ff(1e-100)), fone) == ff(1e-100) + assert mpf_sub(mpf_add(ff(1e-100), fone), fone) == ff(1e-100) + assert mpf_sub(mpf_add(fone, ff(-1e-100)), fone) == ff(-1e-100) + assert mpf_sub(mpf_add(ff(-1e-100), fone), fone) == ff(-1e-100) + assert mpf_add(fone, fzero) == fone + assert mpf_add(fzero, fone) == fone + assert mpf_add(fzero, fzero) == fzero + +def test_long_exponent_shifts(): + mp.dps = 15 + # Check for possible bugs due to exponent arithmetic overflow + # in a C implementation + x = mpf(1) + for p in [32, 64]: + a = ldexp(1,2**(p-1)) + b = ldexp(1,2**p) + c = ldexp(1,2**(p+1)) + d = ldexp(1,-2**(p-1)) + e = ldexp(1,-2**p) + f = ldexp(1,-2**(p+1)) + assert (x+a) == a + assert (x+b) == b + assert (x+c) == c + assert (x+d) == x + assert (x+e) == x + assert (x+f) == x + assert (a+x) == a + assert (b+x) == b + assert (c+x) == c + assert (d+x) == x + assert (e+x) == x + assert (f+x) == x + assert (x-a) == -a + assert (x-b) == -b + assert (x-c) == -c + assert (x-d) == x + assert (x-e) == x + assert (x-f) == x + assert (a-x) == a + assert (b-x) == b + assert (c-x) == c + assert (d-x) == -x + assert (e-x) == -x + assert (f-x) == -x + +def test_float_rounding(): + mp.prec = 64 + for x in [mpf(1), mpf(1)+eps, mpf(1)-eps, -mpf(1)+eps, -mpf(1)-eps]: + fa = float(x) + fb = float(fadd(x,0,prec=53,rounding='n')) + assert fa == fb + z = mpc(x,x) + ca = complex(z) + cb = complex(fadd(z,0,prec=53,rounding='n')) + assert ca == cb + for rnd in ['n', 'd', 'u', 'f', 'c']: + fa = to_float(x._mpf_, rnd=rnd) + fb = to_float(fadd(x,0,prec=53,rounding=rnd)._mpf_, rnd=rnd) + assert fa == fb + mp.prec = 53 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_compatibility.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_compatibility.py new file mode 100644 index 0000000000000000000000000000000000000000..f26d6044b521306b6d1eaeadc5c7839be226dc54 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_compatibility.py @@ -0,0 +1,77 @@ +from mpmath import * +from random import seed, randint, random +import math + +# Test compatibility with Python floats, which are +# IEEE doubles (53-bit) + +N = 5000 +seed(1) + +# Choosing exponents between roughly -140, 140 ensures that +# the Python floats don't overflow or underflow +xs = [(random()-1) * 10**randint(-140, 140) for x in range(N)] +ys = [(random()-1) * 10**randint(-140, 140) for x in range(N)] + +# include some equal values +ys[int(N*0.8):] = xs[int(N*0.8):] + +# Detect whether Python is compiled to use 80-bit floating-point +# instructions, in which case the double compatibility test breaks +uses_x87 = -4.1974624032366689e+117 / -8.4657370748010221e-47 \ + == 4.9581771393902231e+163 + +def test_double_compatibility(): + mp.prec = 53 + for x, y in zip(xs, ys): + mpx = mpf(x) + mpy = mpf(y) + assert mpf(x) == x + assert (mpx < mpy) == (x < y) + assert (mpx > mpy) == (x > y) + assert (mpx == mpy) == (x == y) + assert (mpx != mpy) == (x != y) + assert (mpx <= mpy) == (x <= y) + assert (mpx >= mpy) == (x >= y) + assert mpx == mpx + if uses_x87: + mp.prec = 64 + a = mpx + mpy + b = mpx * mpy + c = mpx / mpy + d = mpx % mpy + mp.prec = 53 + assert +a == x + y + assert +b == x * y + assert +c == x / y + assert +d == x % y + else: + assert mpx + mpy == x + y + assert mpx * mpy == x * y + assert mpx / mpy == x / y + assert mpx % mpy == x % y + assert abs(mpx) == abs(x) + assert mpf(repr(x)) == x + assert ceil(mpx) == math.ceil(x) + assert floor(mpx) == math.floor(x) + +def test_sqrt(): + # this fails quite often. it appers to be float + # that rounds the wrong way, not mpf + fail = 0 + mp.prec = 53 + for x in xs: + x = abs(x) + mp.prec = 100 + mp_high = mpf(x)**0.5 + mp.prec = 53 + mp_low = mpf(x)**0.5 + fp = x**0.5 + assert abs(mp_low-mp_high) <= abs(fp-mp_high) + fail += mp_low != fp + assert fail < N/10 + +def test_bugs(): + # particular bugs + assert mpf(4.4408920985006262E-16) < mpf(1.7763568394002505E-15) + assert mpf(-4.4408920985006262E-16) > mpf(-1.7763568394002505E-15) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_convert.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_convert.py new file mode 100644 index 0000000000000000000000000000000000000000..cb1db5b55c89e980e08fc3fa43cc9715ad68cac9 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_convert.py @@ -0,0 +1,233 @@ +import random +from mpmath import * +from mpmath.libmp import * + + +def test_basic_string(): + """ + Test basic string conversion + """ + mp.dps = 15 + assert mpf('3') == mpf('3.0') == mpf('0003.') == mpf('0.03e2') == mpf(3.0) + assert mpf('30') == mpf('30.0') == mpf('00030.') == mpf(30.0) + for i in range(10): + for j in range(10): + assert mpf('%ie%i' % (i,j)) == i * 10**j + assert str(mpf('25000.0')) == '25000.0' + assert str(mpf('2500.0')) == '2500.0' + assert str(mpf('250.0')) == '250.0' + assert str(mpf('25.0')) == '25.0' + assert str(mpf('2.5')) == '2.5' + assert str(mpf('0.25')) == '0.25' + assert str(mpf('0.025')) == '0.025' + assert str(mpf('0.0025')) == '0.0025' + assert str(mpf('0.00025')) == '0.00025' + assert str(mpf('0.000025')) == '2.5e-5' + assert str(mpf(0)) == '0.0' + assert str(mpf('2.5e1000000000000000000000')) == '2.5e+1000000000000000000000' + assert str(mpf('2.6e-1000000000000000000000')) == '2.6e-1000000000000000000000' + assert str(mpf(1.23402834e-15)) == '1.23402834e-15' + assert str(mpf(-1.23402834e-15)) == '-1.23402834e-15' + assert str(mpf(-1.2344e-15)) == '-1.2344e-15' + assert repr(mpf(-1.2344e-15)) == "mpf('-1.2343999999999999e-15')" + assert str(mpf("2163048125L")) == '2163048125.0' + assert str(mpf("-2163048125l")) == '-2163048125.0' + assert str(mpf("-2163048125L/1088391168")) == '-1.98738118113799' + assert str(mpf("2163048125/1088391168l")) == '1.98738118113799' + +def test_pretty(): + mp.pretty = True + assert repr(mpf(2.5)) == '2.5' + assert repr(mpc(2.5,3.5)) == '(2.5 + 3.5j)' + mp.pretty = False + iv.pretty = True + assert repr(mpi(2.5,3.5)) == '[2.5, 3.5]' + iv.pretty = False + +def test_str_whitespace(): + assert mpf('1.26 ') == 1.26 + +def test_unicode(): + mp.dps = 15 + try: + unicode = unicode + except NameError: + unicode = str + assert mpf(unicode('2.76')) == 2.76 + assert mpf(unicode('inf')) == inf + +def test_str_format(): + assert to_str(from_float(0.1),15,strip_zeros=False) == '0.100000000000000' + assert to_str(from_float(0.0),15,show_zero_exponent=True) == '0.0e+0' + assert to_str(from_float(0.0),0,show_zero_exponent=True) == '.0e+0' + assert to_str(from_float(0.0),0,show_zero_exponent=False) == '.0' + assert to_str(from_float(0.0),1,show_zero_exponent=True) == '0.0e+0' + assert to_str(from_float(0.0),1,show_zero_exponent=False) == '0.0' + assert to_str(from_float(1.23),3,show_zero_exponent=True) == '1.23e+0' + assert to_str(from_float(1.23456789000000e-2),15,strip_zeros=False,min_fixed=0,max_fixed=0) == '1.23456789000000e-2' + assert to_str(from_float(1.23456789000000e+2),15,strip_zeros=False,min_fixed=0,max_fixed=0) == '1.23456789000000e+2' + assert to_str(from_float(2.1287e14), 15, max_fixed=1000) == '212870000000000.0' + assert to_str(from_float(2.1287e15), 15, max_fixed=1000) == '2128700000000000.0' + assert to_str(from_float(2.1287e16), 15, max_fixed=1000) == '21287000000000000.0' + assert to_str(from_float(2.1287e30), 15, max_fixed=1000) == '2128700000000000000000000000000.0' + +def test_tight_string_conversion(): + mp.dps = 15 + # In an old version, '0.5' wasn't recognized as representing + # an exact binary number and was erroneously rounded up or down + assert from_str('0.5', 10, round_floor) == fhalf + assert from_str('0.5', 10, round_ceiling) == fhalf + +def test_eval_repr_invariant(): + """Test that eval(repr(x)) == x""" + random.seed(123) + for dps in [10, 15, 20, 50, 100]: + mp.dps = dps + for i in range(1000): + a = mpf(random.random())**0.5 * 10**random.randint(-100, 100) + assert eval(repr(a)) == a + mp.dps = 15 + +def test_str_bugs(): + mp.dps = 15 + # Decimal rounding used to give the wrong exponent in some cases + assert str(mpf('1e600')) == '1.0e+600' + assert str(mpf('1e10000')) == '1.0e+10000' + +def test_str_prec0(): + assert to_str(from_float(1.234), 0) == '.0e+0' + assert to_str(from_float(1e-15), 0) == '.0e-15' + assert to_str(from_float(1e+15), 0) == '.0e+15' + assert to_str(from_float(-1e-15), 0) == '-.0e-15' + assert to_str(from_float(-1e+15), 0) == '-.0e+15' + +def test_convert_rational(): + mp.dps = 15 + assert from_rational(30, 5, 53, round_nearest) == (0, 3, 1, 2) + assert from_rational(-7, 4, 53, round_nearest) == (1, 7, -2, 3) + assert to_rational((0, 1, -1, 1)) == (1, 2) + +def test_custom_class(): + class mympf: + @property + def _mpf_(self): + return mpf(3.5)._mpf_ + class mympc: + @property + def _mpc_(self): + return mpf(3.5)._mpf_, mpf(2.5)._mpf_ + assert mpf(2) + mympf() == 5.5 + assert mympf() + mpf(2) == 5.5 + assert mpf(mympf()) == 3.5 + assert mympc() + mpc(2) == mpc(5.5, 2.5) + assert mpc(2) + mympc() == mpc(5.5, 2.5) + assert mpc(mympc()) == (3.5+2.5j) + +def test_conversion_methods(): + class SomethingRandom: + pass + class SomethingReal: + def _mpmath_(self, prec, rounding): + return mp.make_mpf(from_str('1.3', prec, rounding)) + class SomethingComplex: + def _mpmath_(self, prec, rounding): + return mp.make_mpc((from_str('1.3', prec, rounding), \ + from_str('1.7', prec, rounding))) + x = mpf(3) + z = mpc(3) + a = SomethingRandom() + y = SomethingReal() + w = SomethingComplex() + for d in [15, 45]: + mp.dps = d + assert (x+y).ae(mpf('4.3')) + assert (y+x).ae(mpf('4.3')) + assert (x+w).ae(mpc('4.3', '1.7')) + assert (w+x).ae(mpc('4.3', '1.7')) + assert (z+y).ae(mpc('4.3')) + assert (y+z).ae(mpc('4.3')) + assert (z+w).ae(mpc('4.3', '1.7')) + assert (w+z).ae(mpc('4.3', '1.7')) + x-y; y-x; x-w; w-x; z-y; y-z; z-w; w-z + x*y; y*x; x*w; w*x; z*y; y*z; z*w; w*z + x/y; y/x; x/w; w/x; z/y; y/z; z/w; w/z + x**y; y**x; x**w; w**x; z**y; y**z; z**w; w**z + x==y; y==x; x==w; w==x; z==y; y==z; z==w; w==z + mp.dps = 15 + assert x.__add__(a) is NotImplemented + assert x.__radd__(a) is NotImplemented + assert x.__lt__(a) is NotImplemented + assert x.__gt__(a) is NotImplemented + assert x.__le__(a) is NotImplemented + assert x.__ge__(a) is NotImplemented + assert x.__eq__(a) is NotImplemented + assert x.__ne__(a) is NotImplemented + # implementation detail + if hasattr(x, "__cmp__"): + assert x.__cmp__(a) is NotImplemented + assert x.__sub__(a) is NotImplemented + assert x.__rsub__(a) is NotImplemented + assert x.__mul__(a) is NotImplemented + assert x.__rmul__(a) is NotImplemented + assert x.__div__(a) is NotImplemented + assert x.__rdiv__(a) is NotImplemented + assert x.__mod__(a) is NotImplemented + assert x.__rmod__(a) is NotImplemented + assert x.__pow__(a) is NotImplemented + assert x.__rpow__(a) is NotImplemented + assert z.__add__(a) is NotImplemented + assert z.__radd__(a) is NotImplemented + assert z.__eq__(a) is NotImplemented + assert z.__ne__(a) is NotImplemented + assert z.__sub__(a) is NotImplemented + assert z.__rsub__(a) is NotImplemented + assert z.__mul__(a) is NotImplemented + assert z.__rmul__(a) is NotImplemented + assert z.__div__(a) is NotImplemented + assert z.__rdiv__(a) is NotImplemented + assert z.__pow__(a) is NotImplemented + assert z.__rpow__(a) is NotImplemented + +def test_mpmathify(): + assert mpmathify('1/2') == 0.5 + assert mpmathify('(1.0+1.0j)') == mpc(1, 1) + assert mpmathify('(1.2e-10 - 3.4e5j)') == mpc('1.2e-10', '-3.4e5') + assert mpmathify('1j') == mpc(1j) + +def test_issue548(): + try: + # This expression is invalid, but may trigger the ReDOS vulnerability + # in the regular expression for parsing complex numbers. + mpmathify('(' + '1' * 5000 + '!j') + except: + return + # The expression is invalid and should raise an exception. + assert False + +def test_compatibility(): + try: + import numpy as np + from fractions import Fraction + from decimal import Decimal + import decimal + except ImportError: + return + # numpy types + for nptype in np.core.numerictypes.typeDict.values(): + if issubclass(nptype, np.complexfloating): + x = nptype(complex(0.5, -0.5)) + elif issubclass(nptype, np.floating): + x = nptype(0.5) + elif issubclass(nptype, np.integer): + x = nptype(2) + # Handle the weird types + try: diff = np.abs(type(np.sqrt(x))(sqrt(x)) - np.sqrt(x)) + except: continue + assert diff < 2.0**-53 + #Fraction and Decimal + oldprec = mp.prec + mp.prec = 1000 + decimal.getcontext().prec = mp.dps + assert sqrt(Fraction(2, 3)).ae(sqrt(mpf('2/3'))) + assert sqrt(Decimal(2)/Decimal(3)).ae(sqrt(mpf('2/3'))) + mp.prec = oldprec diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_division.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_division.py new file mode 100644 index 0000000000000000000000000000000000000000..c704cadeb953793ac0a887aa09c4278cf68a2824 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_division.py @@ -0,0 +1,143 @@ +from mpmath.libmp import * +from mpmath import mpf, mp + +from random import randint, choice, seed + +all_modes = [round_floor, round_ceiling, round_down, round_up, round_nearest] + +fb = from_bstr +fi = from_int +ff = from_float + + +def test_div_1_3(): + a = fi(1) + b = fi(3) + c = fi(-1) + + # floor rounds down, ceiling rounds up + assert mpf_div(a, b, 7, round_floor) == fb('0.01010101') + assert mpf_div(a, b, 7, round_ceiling) == fb('0.01010110') + assert mpf_div(a, b, 7, round_down) == fb('0.01010101') + assert mpf_div(a, b, 7, round_up) == fb('0.01010110') + assert mpf_div(a, b, 7, round_nearest) == fb('0.01010101') + + # floor rounds up, ceiling rounds down + assert mpf_div(c, b, 7, round_floor) == fb('-0.01010110') + assert mpf_div(c, b, 7, round_ceiling) == fb('-0.01010101') + assert mpf_div(c, b, 7, round_down) == fb('-0.01010101') + assert mpf_div(c, b, 7, round_up) == fb('-0.01010110') + assert mpf_div(c, b, 7, round_nearest) == fb('-0.01010101') + +def test_mpf_divi_1_3(): + a = 1 + b = fi(3) + c = -1 + assert mpf_rdiv_int(a, b, 7, round_floor) == fb('0.01010101') + assert mpf_rdiv_int(a, b, 7, round_ceiling) == fb('0.01010110') + assert mpf_rdiv_int(a, b, 7, round_down) == fb('0.01010101') + assert mpf_rdiv_int(a, b, 7, round_up) == fb('0.01010110') + assert mpf_rdiv_int(a, b, 7, round_nearest) == fb('0.01010101') + assert mpf_rdiv_int(c, b, 7, round_floor) == fb('-0.01010110') + assert mpf_rdiv_int(c, b, 7, round_ceiling) == fb('-0.01010101') + assert mpf_rdiv_int(c, b, 7, round_down) == fb('-0.01010101') + assert mpf_rdiv_int(c, b, 7, round_up) == fb('-0.01010110') + assert mpf_rdiv_int(c, b, 7, round_nearest) == fb('-0.01010101') + + +def test_div_300(): + + q = fi(1000000) + a = fi(300499999) # a/q is a little less than a half-integer + b = fi(300500000) # b/q exactly a half-integer + c = fi(300500001) # c/q is a little more than a half-integer + + # Check nearest integer rounding (prec=9 as 2**8 < 300 < 2**9) + + assert mpf_div(a, q, 9, round_down) == fi(300) + assert mpf_div(b, q, 9, round_down) == fi(300) + assert mpf_div(c, q, 9, round_down) == fi(300) + assert mpf_div(a, q, 9, round_up) == fi(301) + assert mpf_div(b, q, 9, round_up) == fi(301) + assert mpf_div(c, q, 9, round_up) == fi(301) + + # Nearest even integer is down + assert mpf_div(a, q, 9, round_nearest) == fi(300) + assert mpf_div(b, q, 9, round_nearest) == fi(300) + assert mpf_div(c, q, 9, round_nearest) == fi(301) + + # Nearest even integer is up + a = fi(301499999) + b = fi(301500000) + c = fi(301500001) + assert mpf_div(a, q, 9, round_nearest) == fi(301) + assert mpf_div(b, q, 9, round_nearest) == fi(302) + assert mpf_div(c, q, 9, round_nearest) == fi(302) + + +def test_tight_integer_division(): + # Test that integer division at tightest possible precision is exact + N = 100 + seed(1) + for i in range(N): + a = choice([1, -1]) * randint(1, 1< 1: + print("original matrix (hessenberg):\n", A) + + n = A.rows + + Q, H = mp.hessenberg(A) + + if verbose > 1: + print("Q:\n",Q) + print("H:\n",H) + + B = Q * H * Q.transpose_conj() + + eps = mp.exp(0.8 * mp.log(mp.eps)) + + err0 = 0 + for x in xrange(n): + for y in xrange(n): + err0 += abs(A[y,x] - B[y,x]) + err0 /= n * n + + err1 = 0 + for x in xrange(n): + for y in xrange(x + 2, n): + err1 += abs(H[y,x]) + + if verbose > 0: + print("difference (H):", err0, err1) + + if verbose > 1: + print("B:\n", B) + + assert err0 < eps + assert err1 == 0 + + +def run_schur(A, verbose = 0): + if verbose > 1: + print("original matrix (schur):\n", A) + + n = A.rows + + Q, R = mp.schur(A) + + if verbose > 1: + print("Q:\n", Q) + print("R:\n", R) + + B = Q * R * Q.transpose_conj() + C = Q * Q.transpose_conj() + + eps = mp.exp(0.8 * mp.log(mp.eps)) + + err0 = 0 + for x in xrange(n): + for y in xrange(n): + err0 += abs(A[y,x] - B[y,x]) + err0 /= n * n + + err1 = 0 + for x in xrange(n): + for y in xrange(n): + if x == y: + C[y,x] -= 1 + err1 += abs(C[y,x]) + err1 /= n * n + + err2 = 0 + for x in xrange(n): + for y in xrange(x + 1, n): + err2 += abs(R[y,x]) + + if verbose > 0: + print("difference (S):", err0, err1, err2) + + if verbose > 1: + print("B:\n", B) + + assert err0 < eps + assert err1 < eps + assert err2 == 0 + +def run_eig(A, verbose = 0): + if verbose > 1: + print("original matrix (eig):\n", A) + + n = A.rows + + E, EL, ER = mp.eig(A, left = True, right = True) + + if verbose > 1: + print("E:\n", E) + print("EL:\n", EL) + print("ER:\n", ER) + + eps = mp.exp(0.8 * mp.log(mp.eps)) + + err0 = 0 + for i in xrange(n): + B = A * ER[:,i] - E[i] * ER[:,i] + err0 = max(err0, mp.mnorm(B)) + + B = EL[i,:] * A - EL[i,:] * E[i] + err0 = max(err0, mp.mnorm(B)) + + err0 /= n * n + + if verbose > 0: + print("difference (E):", err0) + + assert err0 < eps + +##################### + +def test_eig_dyn(): + v = 0 + for i in xrange(5): + n = 1 + int(mp.rand() * 5) + if mp.rand() > 0.5: + # real + A = 2 * mp.randmatrix(n, n) - 1 + if mp.rand() > 0.5: + A *= 10 + for x in xrange(n): + for y in xrange(n): + A[x,y] = int(A[x,y]) + else: + A = (2 * mp.randmatrix(n, n) - 1) + 1j * (2 * mp.randmatrix(n, n) - 1) + if mp.rand() > 0.5: + A *= 10 + for x in xrange(n): + for y in xrange(n): + A[x,y] = int(mp.re(A[x,y])) + 1j * int(mp.im(A[x,y])) + + run_hessenberg(A, verbose = v) + run_schur(A, verbose = v) + run_eig(A, verbose = v) + +def test_eig(): + v = 0 + AS = [] + + A = mp.matrix([[2, 1, 0], # jordan block of size 3 + [0, 2, 1], + [0, 0, 2]]) + AS.append(A) + AS.append(A.transpose()) + + A = mp.matrix([[2, 0, 0], # jordan block of size 2 + [0, 2, 1], + [0, 0, 2]]) + AS.append(A) + AS.append(A.transpose()) + + A = mp.matrix([[2, 0, 1], # jordan block of size 2 + [0, 2, 0], + [0, 0, 2]]) + AS.append(A) + AS.append(A.transpose()) + + A= mp.matrix([[0, 0, 1], # cyclic + [1, 0, 0], + [0, 1, 0]]) + AS.append(A) + AS.append(A.transpose()) + + for A in AS: + run_hessenberg(A, verbose = v) + run_schur(A, verbose = v) + run_eig(A, verbose = v) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_eigen_symmetric.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_eigen_symmetric.py new file mode 100644 index 0000000000000000000000000000000000000000..aab3d8ea3142aada6e14ad6d3ea25a7e8293554d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_eigen_symmetric.py @@ -0,0 +1,357 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +from mpmath import mp +from mpmath import libmp + +xrange = libmp.backend.xrange + +def run_eigsy(A, verbose = False): + if verbose: + print("original matrix:\n", str(A)) + + D, Q = mp.eigsy(A) + B = Q * mp.diag(D) * Q.transpose() + C = A - B + E = Q * Q.transpose() - mp.eye(A.rows) + + if verbose: + print("eigenvalues:\n", D) + print("eigenvectors:\n", Q) + + NC = mp.mnorm(C) + NE = mp.mnorm(E) + + if verbose: + print("difference:", NC, "\n", C, "\n") + print("difference:", NE, "\n", E, "\n") + + eps = mp.exp( 0.8 * mp.log(mp.eps)) + + assert NC < eps + assert NE < eps + + return NC + +def run_eighe(A, verbose = False): + if verbose: + print("original matrix:\n", str(A)) + + D, Q = mp.eighe(A) + B = Q * mp.diag(D) * Q.transpose_conj() + C = A - B + E = Q * Q.transpose_conj() - mp.eye(A.rows) + + if verbose: + print("eigenvalues:\n", D) + print("eigenvectors:\n", Q) + + NC = mp.mnorm(C) + NE = mp.mnorm(E) + + if verbose: + print("difference:", NC, "\n", C, "\n") + print("difference:", NE, "\n", E, "\n") + + eps = mp.exp( 0.8 * mp.log(mp.eps)) + + assert NC < eps + assert NE < eps + + return NC + +def run_svd_r(A, full_matrices = False, verbose = True): + + m, n = A.rows, A.cols + + eps = mp.exp(0.8 * mp.log(mp.eps)) + + if verbose: + print("original matrix:\n", str(A)) + print("full", full_matrices) + + U, S0, V = mp.svd_r(A, full_matrices = full_matrices) + + S = mp.zeros(U.cols, V.rows) + for j in xrange(min(m, n)): + S[j,j] = S0[j] + + if verbose: + print("U:\n", str(U)) + print("S:\n", str(S0)) + print("V:\n", str(V)) + + C = U * S * V - A + err = mp.mnorm(C) + if verbose: + print("C\n", str(C), "\n", err) + assert err < eps + + D = V * V.transpose() - mp.eye(V.rows) + err = mp.mnorm(D) + if verbose: + print("D:\n", str(D), "\n", err) + assert err < eps + + E = U.transpose() * U - mp.eye(U.cols) + err = mp.mnorm(E) + if verbose: + print("E:\n", str(E), "\n", err) + assert err < eps + +def run_svd_c(A, full_matrices = False, verbose = True): + + m, n = A.rows, A.cols + + eps = mp.exp(0.8 * mp.log(mp.eps)) + + if verbose: + print("original matrix:\n", str(A)) + print("full", full_matrices) + + U, S0, V = mp.svd_c(A, full_matrices = full_matrices) + + S = mp.zeros(U.cols, V.rows) + for j in xrange(min(m, n)): + S[j,j] = S0[j] + + if verbose: + print("U:\n", str(U)) + print("S:\n", str(S0)) + print("V:\n", str(V)) + + C = U * S * V - A + err = mp.mnorm(C) + if verbose: + print("C\n", str(C), "\n", err) + assert err < eps + + D = V * V.transpose_conj() - mp.eye(V.rows) + err = mp.mnorm(D) + if verbose: + print("D:\n", str(D), "\n", err) + assert err < eps + + E = U.transpose_conj() * U - mp.eye(U.cols) + err = mp.mnorm(E) + if verbose: + print("E:\n", str(E), "\n", err) + assert err < eps + +def run_gauss(qtype, a, b): + eps = 1e-5 + + d, e = mp.gauss_quadrature(len(a), qtype) + d -= mp.matrix(a) + e -= mp.matrix(b) + + assert mp.mnorm(d) < eps + assert mp.mnorm(e) < eps + +def irandmatrix(n, range = 10): + """ + random matrix with integer entries + """ + A = mp.matrix(n, n) + for i in xrange(n): + for j in xrange(n): + A[i,j]=int( (2 * mp.rand() - 1) * range) + return A + +####################### + +def test_eighe_fixed_matrix(): + A = mp.matrix([[2, 3], [3, 5]]) + run_eigsy(A) + run_eighe(A) + + A = mp.matrix([[7, -11], [-11, 13]]) + run_eigsy(A) + run_eighe(A) + + A = mp.matrix([[2, 11, 7], [11, 3, 13], [7, 13, 5]]) + run_eigsy(A) + run_eighe(A) + + A = mp.matrix([[2, 0, 7], [0, 3, 1], [7, 1, 5]]) + run_eigsy(A) + run_eighe(A) + + # + + A = mp.matrix([[2, 3+7j], [3-7j, 5]]) + run_eighe(A) + + A = mp.matrix([[2, -11j, 0], [+11j, 3, 29j], [0, -29j, 5]]) + run_eighe(A) + + A = mp.matrix([[2, 11 + 17j, 7 + 19j], [11 - 17j, 3, -13 + 23j], [7 - 19j, -13 - 23j, 5]]) + run_eighe(A) + +def test_eigsy_randmatrix(): + N = 5 + + for a in xrange(10): + A = 2 * mp.randmatrix(N, N) - 1 + + for i in xrange(0, N): + for j in xrange(i + 1, N): + A[j,i] = A[i,j] + + run_eigsy(A) + +def test_eighe_randmatrix(): + N = 5 + + for a in xrange(10): + A = (2 * mp.randmatrix(N, N) - 1) + 1j * (2 * mp.randmatrix(N, N) - 1) + + for i in xrange(0, N): + A[i,i] = mp.re(A[i,i]) + for j in xrange(i + 1, N): + A[j,i] = mp.conj(A[i,j]) + + run_eighe(A) + +def test_eigsy_irandmatrix(): + N = 4 + R = 4 + + for a in xrange(10): + A=irandmatrix(N, R) + + for i in xrange(0, N): + for j in xrange(i + 1, N): + A[j,i] = A[i,j] + + run_eigsy(A) + +def test_eighe_irandmatrix(): + N = 4 + R = 4 + + for a in xrange(10): + A=irandmatrix(N, R) + 1j * irandmatrix(N, R) + + for i in xrange(0, N): + A[i,i] = mp.re(A[i,i]) + for j in xrange(i + 1, N): + A[j,i] = mp.conj(A[i,j]) + + run_eighe(A) + +def test_svd_r_rand(): + for i in xrange(5): + full = mp.rand() > 0.5 + m = 1 + int(mp.rand() * 10) + n = 1 + int(mp.rand() * 10) + A = 2 * mp.randmatrix(m, n) - 1 + if mp.rand() > 0.5: + A *= 10 + for x in xrange(m): + for y in xrange(n): + A[x,y]=int(A[x,y]) + + run_svd_r(A, full_matrices = full, verbose = False) + +def test_svd_c_rand(): + for i in xrange(5): + full = mp.rand() > 0.5 + m = 1 + int(mp.rand() * 10) + n = 1 + int(mp.rand() * 10) + A = (2 * mp.randmatrix(m, n) - 1) + 1j * (2 * mp.randmatrix(m, n) - 1) + if mp.rand() > 0.5: + A *= 10 + for x in xrange(m): + for y in xrange(n): + A[x,y]=int(mp.re(A[x,y])) + 1j * int(mp.im(A[x,y])) + + run_svd_c(A, full_matrices=full, verbose=False) + +def test_svd_test_case(): + # a test case from Golub and Reinsch + # (see wilkinson/reinsch: handbook for auto. comp., vol ii-linear algebra, 134-151(1971).) + + eps = mp.exp(0.8 * mp.log(mp.eps)) + + a = [[22, 10, 2, 3, 7], + [14, 7, 10, 0, 8], + [-1, 13, -1, -11, 3], + [-3, -2, 13, -2, 4], + [ 9, 8, 1, -2, 4], + [ 9, 1, -7, 5, -1], + [ 2, -6, 6, 5, 1], + [ 4, 5, 0, -2, 2]] + + a = mp.matrix(a) + b = mp.matrix([mp.sqrt(1248), 20, mp.sqrt(384), 0, 0]) + + S = mp.svd_r(a, compute_uv = False) + S -= b + assert mp.mnorm(S) < eps + + S = mp.svd_c(a, compute_uv = False) + S -= b + assert mp.mnorm(S) < eps + + +def test_gauss_quadrature_static(): + a = [-0.57735027, 0.57735027] + b = [ 1, 1] + run_gauss("legendre", a , b) + + a = [ -0.906179846, -0.538469310, 0, 0.538469310, 0.906179846] + b = [ 0.23692689, 0.47862867, 0.56888889, 0.47862867, 0.23692689] + run_gauss("legendre", a , b) + + a = [ 0.06943184, 0.33000948, 0.66999052, 0.93056816] + b = [ 0.17392742, 0.32607258, 0.32607258, 0.17392742] + run_gauss("legendre01", a , b) + + a = [-0.70710678, 0.70710678] + b = [ 0.88622693, 0.88622693] + run_gauss("hermite", a , b) + + a = [ -2.02018287, -0.958572465, 0, 0.958572465, 2.02018287] + b = [ 0.01995324, 0.39361932, 0.94530872, 0.39361932, 0.01995324] + run_gauss("hermite", a , b) + + a = [ 0.41577456, 2.29428036, 6.28994508] + b = [ 0.71109301, 0.27851773, 0.01038926] + run_gauss("laguerre", a , b) + +def test_gauss_quadrature_dynamic(verbose = False): + n = 5 + + A = mp.randmatrix(2 * n, 1) + + def F(x): + r = 0 + for i in xrange(len(A) - 1, -1, -1): + r = r * x + A[i] + return r + + def run(qtype, FW, R, alpha = 0, beta = 0): + X, W = mp.gauss_quadrature(n, qtype, alpha = alpha, beta = beta) + + a = 0 + for i in xrange(len(X)): + a += W[i] * F(X[i]) + + b = mp.quad(lambda x: FW(x) * F(x), R) + + c = mp.fabs(a - b) + + if verbose: + print(qtype, c, a, b) + + assert c < 1e-5 + + run("legendre", lambda x: 1, [-1, 1]) + run("legendre01", lambda x: 1, [0, 1]) + run("hermite", lambda x: mp.exp(-x*x), [-mp.inf, mp.inf]) + run("laguerre", lambda x: mp.exp(-x), [0, mp.inf]) + run("glaguerre", lambda x: mp.sqrt(x)*mp.exp(-x), [0, mp.inf], alpha = 1 / mp.mpf(2)) + run("chebyshev1", lambda x: 1/mp.sqrt(1-x*x), [-1, 1]) + run("chebyshev2", lambda x: mp.sqrt(1-x*x), [-1, 1]) + run("jacobi", lambda x: (1-x)**(1/mp.mpf(3)) * (1+x)**(1/mp.mpf(5)), [-1, 1], alpha = 1 / mp.mpf(3), beta = 1 / mp.mpf(5) ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_functions.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_functions.py new file mode 100644 index 0000000000000000000000000000000000000000..3bfe852f008173eb636c147abc83d71dbdd4d23a --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_functions.py @@ -0,0 +1,920 @@ +from mpmath.libmp import * +from mpmath import * +import random +import time +import math +import cmath + +def mpc_ae(a, b, eps=eps): + res = True + res = res and a.real.ae(b.real, eps) + res = res and a.imag.ae(b.imag, eps) + return res + +#---------------------------------------------------------------------------- +# Constants and functions +# + +tpi = "3.1415926535897932384626433832795028841971693993751058209749445923078\ +1640628620899862803482534211706798" +te = "2.71828182845904523536028747135266249775724709369995957496696762772407\ +663035354759457138217852516642743" +tdegree = "0.017453292519943295769236907684886127134428718885417254560971914\ +4017100911460344944368224156963450948221" +teuler = "0.5772156649015328606065120900824024310421593359399235988057672348\ +84867726777664670936947063291746749516" +tln2 = "0.693147180559945309417232121458176568075500134360255254120680009493\ +393621969694715605863326996418687542" +tln10 = "2.30258509299404568401799145468436420760110148862877297603332790096\ +757260967735248023599720508959829834" +tcatalan = "0.91596559417721901505460351493238411077414937428167213426649811\ +9621763019776254769479356512926115106249" +tkhinchin = "2.6854520010653064453097148354817956938203822939944629530511523\ +4555721885953715200280114117493184769800" +tglaisher = "1.2824271291006226368753425688697917277676889273250011920637400\ +2174040630885882646112973649195820237439420646" +tapery = "1.2020569031595942853997381615114499907649862923404988817922715553\ +4183820578631309018645587360933525815" +tphi = "1.618033988749894848204586834365638117720309179805762862135448622705\ +26046281890244970720720418939113748475" +tmertens = "0.26149721284764278375542683860869585905156664826119920619206421\ +3924924510897368209714142631434246651052" +ttwinprime = "0.660161815846869573927812110014555778432623360284733413319448\ +423335405642304495277143760031413839867912" + +def test_constants(): + for prec in [3, 7, 10, 15, 20, 37, 80, 100, 29]: + mp.dps = prec + assert pi == mpf(tpi) + assert e == mpf(te) + assert degree == mpf(tdegree) + assert euler == mpf(teuler) + assert ln2 == mpf(tln2) + assert ln10 == mpf(tln10) + assert catalan == mpf(tcatalan) + assert khinchin == mpf(tkhinchin) + assert glaisher == mpf(tglaisher) + assert phi == mpf(tphi) + if prec < 50: + assert mertens == mpf(tmertens) + assert twinprime == mpf(ttwinprime) + mp.dps = 15 + assert pi >= -1 + assert pi > 2 + assert pi > 3 + assert pi < 4 + +def test_exact_sqrts(): + for i in range(20000): + assert sqrt(mpf(i*i)) == i + random.seed(1) + for prec in [100, 300, 1000, 10000]: + mp.dps = prec + for i in range(20): + A = random.randint(10**(prec//2-2), 10**(prec//2-1)) + assert sqrt(mpf(A*A)) == A + mp.dps = 15 + for i in range(100): + for a in [1, 8, 25, 112307]: + assert sqrt(mpf((a*a, 2*i))) == mpf((a, i)) + assert sqrt(mpf((a*a, -2*i))) == mpf((a, -i)) + +def test_sqrt_rounding(): + for i in [2, 3, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15]: + i = from_int(i) + for dps in [7, 15, 83, 106, 2000]: + mp.dps = dps + a = mpf_pow_int(mpf_sqrt(i, mp.prec, round_down), 2, mp.prec, round_down) + b = mpf_pow_int(mpf_sqrt(i, mp.prec, round_up), 2, mp.prec, round_up) + assert mpf_lt(a, i) + assert mpf_gt(b, i) + random.seed(1234) + prec = 100 + for rnd in [round_down, round_nearest, round_ceiling]: + for i in range(100): + a = mpf_rand(prec) + b = mpf_mul(a, a) + assert mpf_sqrt(b, prec, rnd) == a + # Test some extreme cases + mp.dps = 100 + a = mpf(9) + 1e-90 + b = mpf(9) - 1e-90 + mp.dps = 15 + assert sqrt(a, rounding='d') == 3 + assert sqrt(a, rounding='n') == 3 + assert sqrt(a, rounding='u') > 3 + assert sqrt(b, rounding='d') < 3 + assert sqrt(b, rounding='n') == 3 + assert sqrt(b, rounding='u') == 3 + # A worst case, from the MPFR test suite + assert sqrt(mpf('7.0503726185518891')) == mpf('2.655253776675949') + +def test_float_sqrt(): + mp.dps = 15 + # These should round identically + for x in [0, 1e-7, 0.1, 0.5, 1, 2, 3, 4, 5, 0.333, 76.19]: + assert sqrt(mpf(x)) == float(x)**0.5 + assert sqrt(-1) == 1j + assert sqrt(-2).ae(cmath.sqrt(-2)) + assert sqrt(-3).ae(cmath.sqrt(-3)) + assert sqrt(-100).ae(cmath.sqrt(-100)) + assert sqrt(1j).ae(cmath.sqrt(1j)) + assert sqrt(-1j).ae(cmath.sqrt(-1j)) + assert sqrt(math.pi + math.e*1j).ae(cmath.sqrt(math.pi + math.e*1j)) + assert sqrt(math.pi - math.e*1j).ae(cmath.sqrt(math.pi - math.e*1j)) + +def test_hypot(): + assert hypot(0, 0) == 0 + assert hypot(0, 0.33) == mpf(0.33) + assert hypot(0.33, 0) == mpf(0.33) + assert hypot(-0.33, 0) == mpf(0.33) + assert hypot(3, 4) == mpf(5) + +def test_exact_cbrt(): + for i in range(0, 20000, 200): + assert cbrt(mpf(i*i*i)) == i + random.seed(1) + for prec in [100, 300, 1000, 10000]: + mp.dps = prec + A = random.randint(10**(prec//2-2), 10**(prec//2-1)) + assert cbrt(mpf(A*A*A)) == A + mp.dps = 15 + +def test_exp(): + assert exp(0) == 1 + assert exp(10000).ae(mpf('8.8068182256629215873e4342')) + assert exp(-10000).ae(mpf('1.1354838653147360985e-4343')) + a = exp(mpf((1, 8198646019315405, -53, 53))) + assert(a.bc == bitcount(a.man)) + mp.prec = 67 + a = exp(mpf((1, 1781864658064754565, -60, 61))) + assert(a.bc == bitcount(a.man)) + mp.prec = 53 + assert exp(ln2 * 10).ae(1024) + assert exp(2+2j).ae(cmath.exp(2+2j)) + +def test_issue_73(): + mp.dps = 512 + a = exp(-1) + b = exp(1) + mp.dps = 15 + assert (+a).ae(0.36787944117144233) + assert (+b).ae(2.7182818284590451) + +def test_log(): + mp.dps = 15 + assert log(1) == 0 + for x in [0.5, 1.5, 2.0, 3.0, 100, 10**50, 1e-50]: + assert log(x).ae(math.log(x)) + assert log(x, x) == 1 + assert log(1024, 2) == 10 + assert log(10**1234, 10) == 1234 + assert log(2+2j).ae(cmath.log(2+2j)) + # Accuracy near 1 + assert (log(0.6+0.8j).real*10**17).ae(2.2204460492503131) + assert (log(0.6-0.8j).real*10**17).ae(2.2204460492503131) + assert (log(0.8-0.6j).real*10**17).ae(2.2204460492503131) + assert (log(1+1e-8j).real*10**16).ae(0.5) + assert (log(1-1e-8j).real*10**16).ae(0.5) + assert (log(-1+1e-8j).real*10**16).ae(0.5) + assert (log(-1-1e-8j).real*10**16).ae(0.5) + assert (log(1j+1e-8).real*10**16).ae(0.5) + assert (log(1j-1e-8).real*10**16).ae(0.5) + assert (log(-1j+1e-8).real*10**16).ae(0.5) + assert (log(-1j-1e-8).real*10**16).ae(0.5) + assert (log(1+1e-40j).real*10**80).ae(0.5) + assert (log(1j+1e-40).real*10**80).ae(0.5) + # Huge + assert log(ldexp(1.234,10**20)).ae(log(2)*1e20) + assert log(ldexp(1.234,10**200)).ae(log(2)*1e200) + # Some special values + assert log(mpc(0,0)) == mpc(-inf,0) + assert isnan(log(mpc(nan,0)).real) + assert isnan(log(mpc(nan,0)).imag) + assert isnan(log(mpc(0,nan)).real) + assert isnan(log(mpc(0,nan)).imag) + assert isnan(log(mpc(nan,1)).real) + assert isnan(log(mpc(nan,1)).imag) + assert isnan(log(mpc(1,nan)).real) + assert isnan(log(mpc(1,nan)).imag) + +def test_trig_hyperb_basic(): + for x in (list(range(100)) + list(range(-100,0))): + t = x / 4.1 + assert cos(mpf(t)).ae(math.cos(t)) + assert sin(mpf(t)).ae(math.sin(t)) + assert tan(mpf(t)).ae(math.tan(t)) + assert cosh(mpf(t)).ae(math.cosh(t)) + assert sinh(mpf(t)).ae(math.sinh(t)) + assert tanh(mpf(t)).ae(math.tanh(t)) + assert sin(1+1j).ae(cmath.sin(1+1j)) + assert sin(-4-3.6j).ae(cmath.sin(-4-3.6j)) + assert cos(1+1j).ae(cmath.cos(1+1j)) + assert cos(-4-3.6j).ae(cmath.cos(-4-3.6j)) + +def test_degrees(): + assert cos(0*degree) == 1 + assert cos(90*degree).ae(0) + assert cos(180*degree).ae(-1) + assert cos(270*degree).ae(0) + assert cos(360*degree).ae(1) + assert sin(0*degree) == 0 + assert sin(90*degree).ae(1) + assert sin(180*degree).ae(0) + assert sin(270*degree).ae(-1) + assert sin(360*degree).ae(0) + +def random_complexes(N): + random.seed(1) + a = [] + for i in range(N): + x1 = random.uniform(-10, 10) + y1 = random.uniform(-10, 10) + x2 = random.uniform(-10, 10) + y2 = random.uniform(-10, 10) + z1 = complex(x1, y1) + z2 = complex(x2, y2) + a.append((z1, z2)) + return a + +def test_complex_powers(): + for dps in [15, 30, 100]: + # Check accuracy for complex square root + mp.dps = dps + a = mpc(1j)**0.5 + assert a.real == a.imag == mpf(2)**0.5 / 2 + mp.dps = 15 + random.seed(1) + for (z1, z2) in random_complexes(100): + assert (mpc(z1)**mpc(z2)).ae(z1**z2, 1e-12) + assert (e**(-pi*1j)).ae(-1) + mp.dps = 50 + assert (e**(-pi*1j)).ae(-1) + mp.dps = 15 + +def test_complex_sqrt_accuracy(): + def test_mpc_sqrt(lst): + for a, b in lst: + z = mpc(a + j*b) + assert mpc_ae(sqrt(z*z), z) + z = mpc(-a + j*b) + assert mpc_ae(sqrt(z*z), -z) + z = mpc(a - j*b) + assert mpc_ae(sqrt(z*z), z) + z = mpc(-a - j*b) + assert mpc_ae(sqrt(z*z), -z) + random.seed(2) + N = 10 + mp.dps = 30 + dps = mp.dps + test_mpc_sqrt([(random.uniform(0, 10),random.uniform(0, 10)) for i in range(N)]) + test_mpc_sqrt([(i + 0.1, (i + 0.2)*10**i) for i in range(N)]) + mp.dps = 15 + +def test_atan(): + mp.dps = 15 + assert atan(-2.3).ae(math.atan(-2.3)) + assert atan(1e-50) == 1e-50 + assert atan(1e50).ae(pi/2) + assert atan(-1e-50) == -1e-50 + assert atan(-1e50).ae(-pi/2) + assert atan(10**1000).ae(pi/2) + for dps in [25, 70, 100, 300, 1000]: + mp.dps = dps + assert (4*atan(1)).ae(pi) + mp.dps = 15 + pi2 = pi/2 + assert atan(mpc(inf,-1)).ae(pi2) + assert atan(mpc(inf,0)).ae(pi2) + assert atan(mpc(inf,1)).ae(pi2) + assert atan(mpc(1,inf)).ae(pi2) + assert atan(mpc(0,inf)).ae(pi2) + assert atan(mpc(-1,inf)).ae(-pi2) + assert atan(mpc(-inf,1)).ae(-pi2) + assert atan(mpc(-inf,0)).ae(-pi2) + assert atan(mpc(-inf,-1)).ae(-pi2) + assert atan(mpc(-1,-inf)).ae(-pi2) + assert atan(mpc(0,-inf)).ae(-pi2) + assert atan(mpc(1,-inf)).ae(pi2) + +def test_atan2(): + mp.dps = 15 + assert atan2(1,1).ae(pi/4) + assert atan2(1,-1).ae(3*pi/4) + assert atan2(-1,-1).ae(-3*pi/4) + assert atan2(-1,1).ae(-pi/4) + assert atan2(-1,0).ae(-pi/2) + assert atan2(1,0).ae(pi/2) + assert atan2(0,0) == 0 + assert atan2(inf,0).ae(pi/2) + assert atan2(-inf,0).ae(-pi/2) + assert isnan(atan2(inf,inf)) + assert isnan(atan2(-inf,inf)) + assert isnan(atan2(inf,-inf)) + assert isnan(atan2(3,nan)) + assert isnan(atan2(nan,3)) + assert isnan(atan2(0,nan)) + assert isnan(atan2(nan,0)) + assert atan2(0,inf) == 0 + assert atan2(0,-inf).ae(pi) + assert atan2(10,inf) == 0 + assert atan2(-10,inf) == 0 + assert atan2(-10,-inf).ae(-pi) + assert atan2(10,-inf).ae(pi) + assert atan2(inf,10).ae(pi/2) + assert atan2(inf,-10).ae(pi/2) + assert atan2(-inf,10).ae(-pi/2) + assert atan2(-inf,-10).ae(-pi/2) + +def test_areal_inverses(): + assert asin(mpf(0)) == 0 + assert asinh(mpf(0)) == 0 + assert acosh(mpf(1)) == 0 + assert isinstance(asin(mpf(0.5)), mpf) + assert isinstance(asin(mpf(2.0)), mpc) + assert isinstance(acos(mpf(0.5)), mpf) + assert isinstance(acos(mpf(2.0)), mpc) + assert isinstance(atanh(mpf(0.1)), mpf) + assert isinstance(atanh(mpf(1.1)), mpc) + + random.seed(1) + for i in range(50): + x = random.uniform(0, 1) + assert asin(mpf(x)).ae(math.asin(x)) + assert acos(mpf(x)).ae(math.acos(x)) + + x = random.uniform(-10, 10) + assert asinh(mpf(x)).ae(cmath.asinh(x).real) + assert isinstance(asinh(mpf(x)), mpf) + x = random.uniform(1, 10) + assert acosh(mpf(x)).ae(cmath.acosh(x).real) + assert isinstance(acosh(mpf(x)), mpf) + x = random.uniform(-10, 0.999) + assert isinstance(acosh(mpf(x)), mpc) + + x = random.uniform(-1, 1) + assert atanh(mpf(x)).ae(cmath.atanh(x).real) + assert isinstance(atanh(mpf(x)), mpf) + + dps = mp.dps + mp.dps = 300 + assert isinstance(asin(0.5), mpf) + mp.dps = 1000 + assert asin(1).ae(pi/2) + assert asin(-1).ae(-pi/2) + mp.dps = dps + +def test_invhyperb_inaccuracy(): + mp.dps = 15 + assert (asinh(1e-5)*10**5).ae(0.99999999998333333) + assert (asinh(1e-10)*10**10).ae(1) + assert (asinh(1e-50)*10**50).ae(1) + assert (asinh(-1e-5)*10**5).ae(-0.99999999998333333) + assert (asinh(-1e-10)*10**10).ae(-1) + assert (asinh(-1e-50)*10**50).ae(-1) + assert asinh(10**20).ae(46.744849040440862) + assert asinh(-10**20).ae(-46.744849040440862) + assert (tanh(1e-10)*10**10).ae(1) + assert (tanh(-1e-10)*10**10).ae(-1) + assert (atanh(1e-10)*10**10).ae(1) + assert (atanh(-1e-10)*10**10).ae(-1) + +def test_complex_functions(): + for x in (list(range(10)) + list(range(-10,0))): + for y in (list(range(10)) + list(range(-10,0))): + z = complex(x, y)/4.3 + 0.01j + assert exp(mpc(z)).ae(cmath.exp(z)) + assert log(mpc(z)).ae(cmath.log(z)) + assert cos(mpc(z)).ae(cmath.cos(z)) + assert sin(mpc(z)).ae(cmath.sin(z)) + assert tan(mpc(z)).ae(cmath.tan(z)) + assert sinh(mpc(z)).ae(cmath.sinh(z)) + assert cosh(mpc(z)).ae(cmath.cosh(z)) + assert tanh(mpc(z)).ae(cmath.tanh(z)) + +def test_complex_inverse_functions(): + mp.dps = 15 + iv.dps = 15 + for (z1, z2) in random_complexes(30): + # apparently cmath uses a different branch, so we + # can't use it for comparison + assert sinh(asinh(z1)).ae(z1) + # + assert acosh(z1).ae(cmath.acosh(z1)) + assert atanh(z1).ae(cmath.atanh(z1)) + assert atan(z1).ae(cmath.atan(z1)) + # the reason we set a big eps here is that the cmath + # functions are inaccurate + assert asin(z1).ae(cmath.asin(z1), rel_eps=1e-12) + assert acos(z1).ae(cmath.acos(z1), rel_eps=1e-12) + one = mpf(1) + for i in range(-9, 10, 3): + for k in range(-9, 10, 3): + a = 0.9*j*10**k + 0.8*one*10**i + b = cos(acos(a)) + assert b.ae(a) + b = sin(asin(a)) + assert b.ae(a) + one = mpf(1) + err = 2*10**-15 + for i in range(-9, 9, 3): + for k in range(-9, 9, 3): + a = -0.9*10**k + j*0.8*one*10**i + b = cosh(acosh(a)) + assert b.ae(a, err) + b = sinh(asinh(a)) + assert b.ae(a, err) + +def test_reciprocal_functions(): + assert sec(3).ae(-1.01010866590799375) + assert csc(3).ae(7.08616739573718592) + assert cot(3).ae(-7.01525255143453347) + assert sech(3).ae(0.0993279274194332078) + assert csch(3).ae(0.0998215696688227329) + assert coth(3).ae(1.00496982331368917) + assert asec(3).ae(1.23095941734077468) + assert acsc(3).ae(0.339836909454121937) + assert acot(3).ae(0.321750554396642193) + assert asech(0.5).ae(1.31695789692481671) + assert acsch(3).ae(0.327450150237258443) + assert acoth(3).ae(0.346573590279972655) + assert acot(0).ae(1.5707963267948966192) + assert acoth(0).ae(1.5707963267948966192j) + +def test_ldexp(): + mp.dps = 15 + assert ldexp(mpf(2.5), 0) == 2.5 + assert ldexp(mpf(2.5), -1) == 1.25 + assert ldexp(mpf(2.5), 2) == 10 + assert ldexp(mpf('inf'), 3) == mpf('inf') + +def test_frexp(): + mp.dps = 15 + assert frexp(0) == (0.0, 0) + assert frexp(9) == (0.5625, 4) + assert frexp(1) == (0.5, 1) + assert frexp(0.2) == (0.8, -2) + assert frexp(1000) == (0.9765625, 10) + +def test_aliases(): + assert ln(7) == log(7) + assert log10(3.75) == log(3.75,10) + assert degrees(5.6) == 5.6 / degree + assert radians(5.6) == 5.6 * degree + assert power(-1,0.5) == j + assert fmod(25,7) == 4.0 and isinstance(fmod(25,7), mpf) + +def test_arg_sign(): + assert arg(3) == 0 + assert arg(-3).ae(pi) + assert arg(j).ae(pi/2) + assert arg(-j).ae(-pi/2) + assert arg(0) == 0 + assert isnan(atan2(3,nan)) + assert isnan(atan2(nan,3)) + assert isnan(atan2(0,nan)) + assert isnan(atan2(nan,0)) + assert isnan(atan2(nan,nan)) + assert arg(inf) == 0 + assert arg(-inf).ae(pi) + assert isnan(arg(nan)) + #assert arg(inf*j).ae(pi/2) + assert sign(0) == 0 + assert sign(3) == 1 + assert sign(-3) == -1 + assert sign(inf) == 1 + assert sign(-inf) == -1 + assert isnan(sign(nan)) + assert sign(j) == j + assert sign(-3*j) == -j + assert sign(1+j).ae((1+j)/sqrt(2)) + +def test_misc_bugs(): + # test that this doesn't raise an exception + mp.dps = 1000 + log(1302) + mp.dps = 15 + +def test_arange(): + assert arange(10) == [mpf('0.0'), mpf('1.0'), mpf('2.0'), mpf('3.0'), + mpf('4.0'), mpf('5.0'), mpf('6.0'), mpf('7.0'), + mpf('8.0'), mpf('9.0')] + assert arange(-5, 5) == [mpf('-5.0'), mpf('-4.0'), mpf('-3.0'), + mpf('-2.0'), mpf('-1.0'), mpf('0.0'), + mpf('1.0'), mpf('2.0'), mpf('3.0'), mpf('4.0')] + assert arange(0, 1, 0.1) == [mpf('0.0'), mpf('0.10000000000000001'), + mpf('0.20000000000000001'), + mpf('0.30000000000000004'), + mpf('0.40000000000000002'), + mpf('0.5'), mpf('0.60000000000000009'), + mpf('0.70000000000000007'), + mpf('0.80000000000000004'), + mpf('0.90000000000000002')] + assert arange(17, -9, -3) == [mpf('17.0'), mpf('14.0'), mpf('11.0'), + mpf('8.0'), mpf('5.0'), mpf('2.0'), + mpf('-1.0'), mpf('-4.0'), mpf('-7.0')] + assert arange(0.2, 0.1, -0.1) == [mpf('0.20000000000000001')] + assert arange(0) == [] + assert arange(1000, -1) == [] + assert arange(-1.23, 3.21, -0.0000001) == [] + +def test_linspace(): + assert linspace(2, 9, 7) == [mpf('2.0'), mpf('3.166666666666667'), + mpf('4.3333333333333339'), mpf('5.5'), mpf('6.666666666666667'), + mpf('7.8333333333333339'), mpf('9.0')] + assert linspace(2, 9, 7, endpoint=0) == [mpf('2.0'), mpf('3.0'), mpf('4.0'), + mpf('5.0'), mpf('6.0'), mpf('7.0'), mpf('8.0')] + assert linspace(2, 7, 1) == [mpf(2)] + +def test_float_cbrt(): + mp.dps = 30 + for a in arange(0,10,0.1): + assert cbrt(a*a*a).ae(a, eps) + assert cbrt(-1).ae(0.5 + j*sqrt(3)/2) + one_third = mpf(1)/3 + for a in arange(0,10,2.7) + [0.1 + 10**5]: + a = mpc(a + 1.1j) + r1 = cbrt(a) + mp.dps += 10 + r2 = pow(a, one_third) + mp.dps -= 10 + assert r1.ae(r2, eps) + mp.dps = 100 + for n in range(100, 301, 100): + w = 10**n + j*10**-3 + z = w*w*w + r = cbrt(z) + assert mpc_ae(r, w, eps) + mp.dps = 15 + +def test_root(): + mp.dps = 30 + random.seed(1) + a = random.randint(0, 10000) + p = a*a*a + r = nthroot(mpf(p), 3) + assert r == a + for n in range(4, 10): + p = p*a + assert nthroot(mpf(p), n) == a + mp.dps = 40 + for n in range(10, 5000, 100): + for a in [random.random()*10000, random.random()*10**100]: + r = nthroot(a, n) + r1 = pow(a, mpf(1)/n) + assert r.ae(r1) + r = nthroot(a, -n) + r1 = pow(a, -mpf(1)/n) + assert r.ae(r1) + # XXX: this is broken right now + # tests for nthroot rounding + for rnd in ['nearest', 'up', 'down']: + mp.rounding = rnd + for n in [-5, -3, 3, 5]: + prec = 50 + for i in range(10): + mp.prec = prec + a = rand() + mp.prec = 2*prec + b = a**n + mp.prec = prec + r = nthroot(b, n) + assert r == a + mp.dps = 30 + for n in range(3, 21): + a = (random.random() + j*random.random()) + assert nthroot(a, n).ae(pow(a, mpf(1)/n)) + assert mpc_ae(nthroot(a, n), pow(a, mpf(1)/n)) + a = (random.random()*10**100 + j*random.random()) + r = nthroot(a, n) + mp.dps += 4 + r1 = pow(a, mpf(1)/n) + mp.dps -= 4 + assert r.ae(r1) + assert mpc_ae(r, r1, eps) + r = nthroot(a, -n) + mp.dps += 4 + r1 = pow(a, -mpf(1)/n) + mp.dps -= 4 + assert r.ae(r1) + assert mpc_ae(r, r1, eps) + mp.dps = 15 + assert nthroot(4, 1) == 4 + assert nthroot(4, 0) == 1 + assert nthroot(4, -1) == 0.25 + assert nthroot(inf, 1) == inf + assert nthroot(inf, 2) == inf + assert nthroot(inf, 3) == inf + assert nthroot(inf, -1) == 0 + assert nthroot(inf, -2) == 0 + assert nthroot(inf, -3) == 0 + assert nthroot(j, 1) == j + assert nthroot(j, 0) == 1 + assert nthroot(j, -1) == -j + assert isnan(nthroot(nan, 1)) + assert isnan(nthroot(nan, 0)) + assert isnan(nthroot(nan, -1)) + assert isnan(nthroot(inf, 0)) + assert root(2,3) == nthroot(2,3) + assert root(16,4,0) == 2 + assert root(16,4,1) == 2j + assert root(16,4,2) == -2 + assert root(16,4,3) == -2j + assert root(16,4,4) == 2 + assert root(-125,3,1) == -5 + +def test_issue_136(): + for dps in [20, 80]: + mp.dps = dps + r = nthroot(mpf('-1e-20'), 4) + assert r.ae(mpf(10)**(-5) * (1 + j) * mpf(2)**(-0.5)) + mp.dps = 80 + assert nthroot('-1e-3', 4).ae(mpf(10)**(-3./4) * (1 + j)/sqrt(2)) + assert nthroot('-1e-6', 4).ae((1 + j)/(10 * sqrt(20))) + # Check that this doesn't take eternity to compute + mp.dps = 20 + assert nthroot('-1e100000000', 4).ae((1+j)*mpf('1e25000000')/sqrt(2)) + mp.dps = 15 + +def test_mpcfun_real_imag(): + mp.dps = 15 + x = mpf(0.3) + y = mpf(0.4) + assert exp(mpc(x,0)) == exp(x) + assert exp(mpc(0,y)) == mpc(cos(y),sin(y)) + assert cos(mpc(x,0)) == cos(x) + assert sin(mpc(x,0)) == sin(x) + assert cos(mpc(0,y)) == cosh(y) + assert sin(mpc(0,y)) == mpc(0,sinh(y)) + assert cospi(mpc(x,0)) == cospi(x) + assert sinpi(mpc(x,0)) == sinpi(x) + assert cospi(mpc(0,y)).ae(cosh(pi*y)) + assert sinpi(mpc(0,y)).ae(mpc(0,sinh(pi*y))) + c, s = cospi_sinpi(mpc(x,0)) + assert c == cospi(x) + assert s == sinpi(x) + c, s = cospi_sinpi(mpc(0,y)) + assert c.ae(cosh(pi*y)) + assert s.ae(mpc(0,sinh(pi*y))) + c, s = cos_sin(mpc(x,0)) + assert c == cos(x) + assert s == sin(x) + c, s = cos_sin(mpc(0,y)) + assert c == cosh(y) + assert s == mpc(0,sinh(y)) + +def test_perturbation_rounding(): + mp.dps = 100 + a = pi/10**50 + b = -pi/10**50 + c = 1 + a + d = 1 + b + mp.dps = 15 + assert exp(a) == 1 + assert exp(a, rounding='c') > 1 + assert exp(b, rounding='c') == 1 + assert exp(a, rounding='f') == 1 + assert exp(b, rounding='f') < 1 + assert cos(a) == 1 + assert cos(a, rounding='c') == 1 + assert cos(b, rounding='c') == 1 + assert cos(a, rounding='f') < 1 + assert cos(b, rounding='f') < 1 + for f in [sin, atan, asinh, tanh]: + assert f(a) == +a + assert f(a, rounding='c') > a + assert f(a, rounding='f') < a + assert f(b) == +b + assert f(b, rounding='c') > b + assert f(b, rounding='f') < b + for f in [asin, tan, sinh, atanh]: + assert f(a) == +a + assert f(b) == +b + assert f(a, rounding='c') > a + assert f(b, rounding='c') > b + assert f(a, rounding='f') < a + assert f(b, rounding='f') < b + assert ln(c) == +a + assert ln(d) == +b + assert ln(c, rounding='c') > a + assert ln(c, rounding='f') < a + assert ln(d, rounding='c') > b + assert ln(d, rounding='f') < b + assert cosh(a) == 1 + assert cosh(b) == 1 + assert cosh(a, rounding='c') > 1 + assert cosh(b, rounding='c') > 1 + assert cosh(a, rounding='f') == 1 + assert cosh(b, rounding='f') == 1 + +def test_integer_parts(): + assert floor(3.2) == 3 + assert ceil(3.2) == 4 + assert floor(3.2+5j) == 3+5j + assert ceil(3.2+5j) == 4+5j + +def test_complex_parts(): + assert fabs('3') == 3 + assert fabs(3+4j) == 5 + assert re(3) == 3 + assert re(1+4j) == 1 + assert im(3) == 0 + assert im(1+4j) == 4 + assert conj(3) == 3 + assert conj(3+4j) == 3-4j + assert mpf(3).conjugate() == 3 + +def test_cospi_sinpi(): + assert sinpi(0) == 0 + assert sinpi(0.5) == 1 + assert sinpi(1) == 0 + assert sinpi(1.5) == -1 + assert sinpi(2) == 0 + assert sinpi(2.5) == 1 + assert sinpi(-0.5) == -1 + assert cospi(0) == 1 + assert cospi(0.5) == 0 + assert cospi(1) == -1 + assert cospi(1.5) == 0 + assert cospi(2) == 1 + assert cospi(2.5) == 0 + assert cospi(-0.5) == 0 + assert cospi(100000000000.25).ae(sqrt(2)/2) + a = cospi(2+3j) + assert a.real.ae(cos((2+3j)*pi).real) + assert a.imag == 0 + b = sinpi(2+3j) + assert b.imag.ae(sin((2+3j)*pi).imag) + assert b.real == 0 + mp.dps = 35 + x1 = mpf(10000) - mpf('1e-15') + x2 = mpf(10000) + mpf('1e-15') + x3 = mpf(10000.5) - mpf('1e-15') + x4 = mpf(10000.5) + mpf('1e-15') + x5 = mpf(10001) - mpf('1e-15') + x6 = mpf(10001) + mpf('1e-15') + x7 = mpf(10001.5) - mpf('1e-15') + x8 = mpf(10001.5) + mpf('1e-15') + mp.dps = 15 + M = 10**15 + assert (sinpi(x1)*M).ae(-pi) + assert (sinpi(x2)*M).ae(pi) + assert (cospi(x3)*M).ae(pi) + assert (cospi(x4)*M).ae(-pi) + assert (sinpi(x5)*M).ae(pi) + assert (sinpi(x6)*M).ae(-pi) + assert (cospi(x7)*M).ae(-pi) + assert (cospi(x8)*M).ae(pi) + assert 0.999 < cospi(x1, rounding='d') < 1 + assert 0.999 < cospi(x2, rounding='d') < 1 + assert 0.999 < sinpi(x3, rounding='d') < 1 + assert 0.999 < sinpi(x4, rounding='d') < 1 + assert -1 < cospi(x5, rounding='d') < -0.999 + assert -1 < cospi(x6, rounding='d') < -0.999 + assert -1 < sinpi(x7, rounding='d') < -0.999 + assert -1 < sinpi(x8, rounding='d') < -0.999 + assert (sinpi(1e-15)*M).ae(pi) + assert (sinpi(-1e-15)*M).ae(-pi) + assert cospi(1e-15) == 1 + assert cospi(1e-15, rounding='d') < 1 + +def test_expj(): + assert expj(0) == 1 + assert expj(1).ae(exp(j)) + assert expj(j).ae(exp(-1)) + assert expj(1+j).ae(exp(j*(1+j))) + assert expjpi(0) == 1 + assert expjpi(1).ae(exp(j*pi)) + assert expjpi(j).ae(exp(-pi)) + assert expjpi(1+j).ae(exp(j*pi*(1+j))) + assert expjpi(-10**15 * j).ae('2.22579818340535731e+1364376353841841') + +def test_sinc(): + assert sinc(0) == sincpi(0) == 1 + assert sinc(inf) == sincpi(inf) == 0 + assert sinc(-inf) == sincpi(-inf) == 0 + assert sinc(2).ae(0.45464871341284084770) + assert sinc(2+3j).ae(0.4463290318402435457-2.7539470277436474940j) + assert sincpi(2) == 0 + assert sincpi(1.5).ae(-0.212206590789193781) + +def test_fibonacci(): + mp.dps = 15 + assert [fibonacci(n) for n in range(-5, 10)] == \ + [5, -3, 2, -1, 1, 0, 1, 1, 2, 3, 5, 8, 13, 21, 34] + assert fib(2.5).ae(1.4893065462657091) + assert fib(3+4j).ae(-5248.51130728372 - 14195.962288353j) + assert fib(1000).ae(4.3466557686937455e+208) + assert str(fib(10**100)) == '6.24499112864607e+2089876402499787337692720892375554168224592399182109535392875613974104853496745963277658556235103534' + mp.dps = 2100 + a = fib(10000) + assert a % 10**10 == 9947366875 + mp.dps = 15 + assert fibonacci(inf) == inf + assert fib(3+0j) == 2 + +def test_call_with_dps(): + mp.dps = 15 + assert abs(exp(1, dps=30)-e(dps=35)) < 1e-29 + +def test_tanh(): + mp.dps = 15 + assert tanh(0) == 0 + assert tanh(inf) == 1 + assert tanh(-inf) == -1 + assert isnan(tanh(nan)) + assert tanh(mpc('inf', '0')) == 1 + +def test_atanh(): + mp.dps = 15 + assert atanh(0) == 0 + assert atanh(0.5).ae(0.54930614433405484570) + assert atanh(-0.5).ae(-0.54930614433405484570) + assert atanh(1) == inf + assert atanh(-1) == -inf + assert isnan(atanh(nan)) + assert isinstance(atanh(1), mpf) + assert isinstance(atanh(-1), mpf) + # Limits at infinity + jpi2 = j*pi/2 + assert atanh(inf).ae(-jpi2) + assert atanh(-inf).ae(jpi2) + assert atanh(mpc(inf,-1)).ae(-jpi2) + assert atanh(mpc(inf,0)).ae(-jpi2) + assert atanh(mpc(inf,1)).ae(jpi2) + assert atanh(mpc(1,inf)).ae(jpi2) + assert atanh(mpc(0,inf)).ae(jpi2) + assert atanh(mpc(-1,inf)).ae(jpi2) + assert atanh(mpc(-inf,1)).ae(jpi2) + assert atanh(mpc(-inf,0)).ae(jpi2) + assert atanh(mpc(-inf,-1)).ae(-jpi2) + assert atanh(mpc(-1,-inf)).ae(-jpi2) + assert atanh(mpc(0,-inf)).ae(-jpi2) + assert atanh(mpc(1,-inf)).ae(-jpi2) + +def test_expm1(): + mp.dps = 15 + assert expm1(0) == 0 + assert expm1(3).ae(exp(3)-1) + assert expm1(inf) == inf + assert expm1(1e-50).ae(1e-50) + assert (expm1(1e-10)*1e10).ae(1.00000000005) + +def test_log1p(): + mp.dps = 15 + assert log1p(0) == 0 + assert log1p(3).ae(log(1+3)) + assert log1p(inf) == inf + assert log1p(1e-50).ae(1e-50) + assert (log1p(1e-10)*1e10).ae(0.99999999995) + +def test_powm1(): + mp.dps = 15 + assert powm1(2,3) == 7 + assert powm1(-1,2) == 0 + assert powm1(-1,0) == 0 + assert powm1(-2,0) == 0 + assert powm1(3+4j,0) == 0 + assert powm1(0,1) == -1 + assert powm1(0,0) == 0 + assert powm1(1,0) == 0 + assert powm1(1,2) == 0 + assert powm1(1,3+4j) == 0 + assert powm1(1,5) == 0 + assert powm1(j,4) == 0 + assert powm1(-j,4) == 0 + assert (powm1(2,1e-100)*1e100).ae(ln2) + assert powm1(2,'1e-100000000000') != 0 + assert (powm1(fadd(1,1e-100,exact=True), 5)*1e100).ae(5) + +def test_unitroots(): + assert unitroots(1) == [1] + assert unitroots(2) == [1, -1] + a, b, c = unitroots(3) + assert a == 1 + assert b.ae(-0.5 + 0.86602540378443864676j) + assert c.ae(-0.5 - 0.86602540378443864676j) + assert unitroots(1, primitive=True) == [1] + assert unitroots(2, primitive=True) == [-1] + assert unitroots(3, primitive=True) == unitroots(3)[1:] + assert unitroots(4, primitive=True) == [j, -j] + assert len(unitroots(17, primitive=True)) == 16 + assert len(unitroots(16, primitive=True)) == 8 + +def test_cyclotomic(): + mp.dps = 15 + assert [cyclotomic(n,1) for n in range(31)] == [1,0,2,3,2,5,1,7,2,3,1,11,1,13,1,1,2,17,1,19,1,1,1,23,1,5,1,3,1,29,1] + assert [cyclotomic(n,-1) for n in range(31)] == [1,-2,0,1,2,1,3,1,2,1,5,1,1,1,7,1,2,1,3,1,1,1,11,1,1,1,13,1,1,1,1] + assert [cyclotomic(n,j) for n in range(21)] == [1,-1+j,1+j,j,0,1,-j,j,2,-j,1,j,3,1,-j,1,2,1,j,j,5] + assert [cyclotomic(n,-j) for n in range(21)] == [1,-1-j,1-j,-j,0,1,j,-j,2,j,1,-j,3,1,j,1,2,1,-j,-j,5] + assert cyclotomic(1624,j) == 1 + assert cyclotomic(33600,j) == 1 + u = sqrt(j, prec=500) + assert cyclotomic(8, u).ae(0) + assert cyclotomic(30, u).ae(5.8284271247461900976) + assert cyclotomic(2040, u).ae(1) + assert cyclotomic(0,2.5) == 1 + assert cyclotomic(1,2.5) == 2.5-1 + assert cyclotomic(2,2.5) == 2.5+1 + assert cyclotomic(3,2.5) == 2.5**2 + 2.5 + 1 + assert cyclotomic(7,2.5) == 406.234375 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_functions2.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_functions2.py new file mode 100644 index 0000000000000000000000000000000000000000..2b2d57fcec9be0db4d921b013f24fd6a5e0e9930 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_functions2.py @@ -0,0 +1,2384 @@ +import math +import pytest +from mpmath import * + +def test_bessel(): + mp.dps = 15 + assert j0(1).ae(0.765197686557966551) + assert j0(pi).ae(-0.304242177644093864) + assert j0(1000).ae(0.0247866861524201746) + assert j0(-25).ae(0.0962667832759581162) + assert j1(1).ae(0.440050585744933516) + assert j1(pi).ae(0.284615343179752757) + assert j1(1000).ae(0.00472831190708952392) + assert j1(-25).ae(0.125350249580289905) + assert besselj(5,1).ae(0.000249757730211234431) + assert besselj(5+0j,1).ae(0.000249757730211234431) + assert besselj(5,pi).ae(0.0521411843671184747) + assert besselj(5,1000).ae(0.00502540694523318607) + assert besselj(5,-25).ae(0.0660079953984229934) + assert besselj(-3,2).ae(-0.128943249474402051) + assert besselj(-4,2).ae(0.0339957198075684341) + assert besselj(3,3+2j).ae(0.424718794929639595942 + 0.625665327745785804812j) + assert besselj(0.25,4).ae(-0.374760630804249715) + assert besselj(1+2j,3+4j).ae(0.319247428741872131 - 0.669557748880365678j) + assert (besselj(3, 10**10) * 10**5).ae(0.76765081748139204023) + assert bessely(-0.5, 0) == 0 + assert bessely(0.5, 0) == -inf + assert bessely(1.5, 0) == -inf + assert bessely(0,0) == -inf + assert bessely(-0.4, 0) == -inf + assert bessely(-0.6, 0) == inf + assert bessely(-1, 0) == inf + assert bessely(-1.4, 0) == inf + assert bessely(-1.6, 0) == -inf + assert bessely(-1, 0) == inf + assert bessely(-2, 0) == -inf + assert bessely(-3, 0) == inf + assert bessely(0.5, 0) == -inf + assert bessely(1, 0) == -inf + assert bessely(1.5, 0) == -inf + assert bessely(2, 0) == -inf + assert bessely(2.5, 0) == -inf + assert bessely(3, 0) == -inf + assert bessely(0,0.5).ae(-0.44451873350670655715) + assert bessely(1,0.5).ae(-1.4714723926702430692) + assert bessely(-1,0.5).ae(1.4714723926702430692) + assert bessely(3.5,0.5).ae(-138.86400867242488443) + assert bessely(0,3+4j).ae(4.6047596915010138655-8.8110771408232264208j) + assert bessely(0,j).ae(-0.26803248203398854876+1.26606587775200833560j) + assert (bessely(3, 10**10) * 10**5).ae(0.21755917537013204058) + assert besseli(0,0) == 1 + assert besseli(1,0) == 0 + assert besseli(2,0) == 0 + assert besseli(-1,0) == 0 + assert besseli(-2,0) == 0 + assert besseli(0,0.5).ae(1.0634833707413235193) + assert besseli(1,0.5).ae(0.25789430539089631636) + assert besseli(-1,0.5).ae(0.25789430539089631636) + assert besseli(3.5,0.5).ae(0.00068103597085793815863) + assert besseli(0,3+4j).ae(-3.3924877882755196097-1.3239458916287264815j) + assert besseli(0,j).ae(besselj(0,1)) + assert (besseli(3, 10**10) * mpf(10)**(-4342944813)).ae(4.2996028505491271875) + assert besselk(0,0) == inf + assert besselk(1,0) == inf + assert besselk(2,0) == inf + assert besselk(-1,0) == inf + assert besselk(-2,0) == inf + assert besselk(0,0.5).ae(0.92441907122766586178) + assert besselk(1,0.5).ae(1.6564411200033008937) + assert besselk(-1,0.5).ae(1.6564411200033008937) + assert besselk(3.5,0.5).ae(207.48418747548460607) + assert besselk(0,3+4j).ae(-0.007239051213570155013+0.026510418350267677215j) + assert besselk(0,j).ae(-0.13863371520405399968-1.20196971531720649914j) + assert (besselk(3, 10**10) * mpf(10)**4342944824).ae(1.1628981033356187851) + # test for issue 331, bug reported by Michael Hartmann + for n in range(10,100,10): + mp.dps = n + assert besseli(91.5,24.7708).ae("4.00830632138673963619656140653537080438462342928377020695738635559218797348548092636896796324190271316137982810144874264e-41") + +def test_bessel_zeros(): + mp.dps = 15 + assert besseljzero(0,1).ae(2.40482555769577276869) + assert besseljzero(2,1).ae(5.1356223018406825563) + assert besseljzero(1,50).ae(157.86265540193029781) + assert besseljzero(10,1).ae(14.475500686554541220) + assert besseljzero(0.5,3).ae(9.4247779607693797153) + assert besseljzero(2,1,1).ae(3.0542369282271403228) + assert besselyzero(0,1).ae(0.89357696627916752158) + assert besselyzero(2,1).ae(3.3842417671495934727) + assert besselyzero(1,50).ae(156.29183520147840108) + assert besselyzero(10,1).ae(12.128927704415439387) + assert besselyzero(0.5,3).ae(7.8539816339744830962) + assert besselyzero(2,1,1).ae(5.0025829314460639452) + +def test_hankel(): + mp.dps = 15 + assert hankel1(0,0.5).ae(0.93846980724081290423-0.44451873350670655715j) + assert hankel1(1,0.5).ae(0.2422684576748738864-1.4714723926702430692j) + assert hankel1(-1,0.5).ae(-0.2422684576748738864+1.4714723926702430692j) + assert hankel1(1.5,0.5).ae(0.0917016996256513026-2.5214655504213378514j) + assert hankel1(1.5,3+4j).ae(0.0066806866476728165382-0.0036684231610839127106j) + assert hankel2(0,0.5).ae(0.93846980724081290423+0.44451873350670655715j) + assert hankel2(1,0.5).ae(0.2422684576748738864+1.4714723926702430692j) + assert hankel2(-1,0.5).ae(-0.2422684576748738864-1.4714723926702430692j) + assert hankel2(1.5,0.5).ae(0.0917016996256513026+2.5214655504213378514j) + assert hankel2(1.5,3+4j).ae(14.783528526098567526-7.397390270853446512j) + +def test_struve(): + mp.dps = 15 + assert struveh(2,3).ae(0.74238666967748318564) + assert struveh(-2.5,3).ae(0.41271003220971599344) + assert struvel(2,3).ae(1.7476573277362782744) + assert struvel(-2.5,3).ae(1.5153394466819651377) + +def test_whittaker(): + mp.dps = 15 + assert whitm(2,3,4).ae(49.753745589025246591) + assert whitw(2,3,4).ae(14.111656223052932215) + +def test_kelvin(): + mp.dps = 15 + assert ber(2,3).ae(0.80836846563726819091) + assert ber(3,4).ae(-0.28262680167242600233) + assert ber(-3,2).ae(-0.085611448496796363669) + assert bei(2,3).ae(-0.89102236377977331571) + assert bei(-3,2).ae(-0.14420994155731828415) + assert ker(2,3).ae(0.12839126695733458928) + assert ker(-3,2).ae(-0.29802153400559142783) + assert ker(0.5,3).ae(-0.085662378535217097524) + assert kei(2,3).ae(0.036804426134164634000) + assert kei(-3,2).ae(0.88682069845786731114) + assert kei(0.5,3).ae(0.013633041571314302948) + +def test_hyper_misc(): + mp.dps = 15 + assert hyp0f1(1,0) == 1 + assert hyp1f1(1,2,0) == 1 + assert hyp1f2(1,2,3,0) == 1 + assert hyp2f1(1,2,3,0) == 1 + assert hyp2f2(1,2,3,4,0) == 1 + assert hyp2f3(1,2,3,4,5,0) == 1 + # Degenerate case: 0F0 + assert hyper([],[],0) == 1 + assert hyper([],[],-2).ae(exp(-2)) + # Degenerate case: 1F0 + assert hyper([2],[],1.5) == 4 + # + assert hyp2f1((1,3),(2,3),(5,6),mpf(27)/32).ae(1.6) + assert hyp2f1((1,4),(1,2),(3,4),mpf(80)/81).ae(1.8) + assert hyp2f1((2,3),(1,1),(3,2),(2+j)/3).ae(1.327531603558679093+0.439585080092769253j) + mp.dps = 25 + v = mpc('1.2282306665029814734863026', '-0.1225033830118305184672133') + assert hyper([(3,4),2+j,1],[1,5,j/3],mpf(1)/5+j/8).ae(v) + mp.dps = 15 + +def test_elliptic_integrals(): + mp.dps = 15 + assert ellipk(0).ae(pi/2) + assert ellipk(0.5).ae(gamma(0.25)**2/(4*sqrt(pi))) + assert ellipk(1) == inf + assert ellipk(1+0j) == inf + assert ellipk(-1).ae('1.3110287771460599052') + assert ellipk(-2).ae('1.1714200841467698589') + assert isinstance(ellipk(-2), mpf) + assert isinstance(ellipe(-2), mpf) + assert ellipk(-50).ae('0.47103424540873331679') + mp.dps = 30 + n1 = +fraction(99999,100000) + n2 = +fraction(100001,100000) + mp.dps = 15 + assert ellipk(n1).ae('7.1427724505817781901') + assert ellipk(n2).ae(mpc('7.1427417367963090109', '-1.5707923998261688019')) + assert ellipe(n1).ae('1.0000332138990829170') + v = ellipe(n2) + assert v.real.ae('0.999966786328145474069137') + assert (v.imag*10**6).ae('7.853952181727432') + assert ellipk(2).ae(mpc('1.3110287771460599052', '-1.3110287771460599052')) + assert ellipk(50).ae(mpc('0.22326753950210985451', '-0.47434723226254522087')) + assert ellipk(3+4j).ae(mpc('0.91119556380496500866', '0.63133428324134524388')) + assert ellipk(3-4j).ae(mpc('0.91119556380496500866', '-0.63133428324134524388')) + assert ellipk(-3+4j).ae(mpc('0.95357894880405122483', '0.23093044503746114444')) + assert ellipk(-3-4j).ae(mpc('0.95357894880405122483', '-0.23093044503746114444')) + assert isnan(ellipk(nan)) + assert isnan(ellipe(nan)) + assert ellipk(inf) == 0 + assert isinstance(ellipk(inf), mpc) + assert ellipk(-inf) == 0 + assert ellipk(1+0j) == inf + assert ellipe(0).ae(pi/2) + assert ellipe(0.5).ae(pi**(mpf(3)/2)/gamma(0.25)**2 +gamma(0.25)**2/(8*sqrt(pi))) + assert ellipe(1) == 1 + assert ellipe(1+0j) == 1 + assert ellipe(inf) == mpc(0,inf) + assert ellipe(-inf) == inf + assert ellipe(3+4j).ae(1.4995535209333469543-1.5778790079127582745j) + assert ellipe(3-4j).ae(1.4995535209333469543+1.5778790079127582745j) + assert ellipe(-3+4j).ae(2.5804237855343377803-0.8306096791000413778j) + assert ellipe(-3-4j).ae(2.5804237855343377803+0.8306096791000413778j) + assert ellipe(2).ae(0.59907011736779610372+0.59907011736779610372j) + assert ellipe('1e-1000000000').ae(pi/2) + assert ellipk('1e-1000000000').ae(pi/2) + assert ellipe(-pi).ae(2.4535865983838923) + mp.dps = 50 + assert ellipk(1/pi).ae('1.724756270009501831744438120951614673874904182624739673') + assert ellipe(1/pi).ae('1.437129808135123030101542922290970050337425479058225712') + assert ellipk(-10*pi).ae('0.5519067523886233967683646782286965823151896970015484512') + assert ellipe(-10*pi).ae('5.926192483740483797854383268707108012328213431657645509') + v = ellipk(pi) + assert v.real.ae('0.973089521698042334840454592642137667227167622330325225') + assert v.imag.ae('-1.156151296372835303836814390793087600271609993858798016') + v = ellipe(pi) + assert v.real.ae('0.4632848917264710404078033487934663562998345622611263332') + assert v.imag.ae('1.0637961621753130852473300451583414489944099504180510966') + mp.dps = 15 + +def test_exp_integrals(): + mp.dps = 15 + x = +e + z = e + sqrt(3)*j + assert ei(x).ae(8.21168165538361560) + assert li(x).ae(1.89511781635593676) + assert si(x).ae(1.82104026914756705) + assert ci(x).ae(0.213958001340379779) + assert shi(x).ae(4.11520706247846193) + assert chi(x).ae(4.09647459290515367) + assert fresnels(x).ae(0.437189718149787643) + assert fresnelc(x).ae(0.401777759590243012) + assert airyai(x).ae(0.0108502401568586681) + assert airybi(x).ae(8.98245748585468627) + assert ei(z).ae(3.72597969491314951 + 7.34213212314224421j) + assert li(z).ae(2.28662658112562502 + 1.50427225297269364j) + assert si(z).ae(2.48122029237669054 + 0.12684703275254834j) + assert ci(z).ae(0.169255590269456633 - 0.892020751420780353j) + assert shi(z).ae(1.85810366559344468 + 3.66435842914920263j) + assert chi(z).ae(1.86787602931970484 + 3.67777369399304159j) + assert fresnels(z/3).ae(0.034534397197008182 + 0.754859844188218737j) + assert fresnelc(z/3).ae(1.261581645990027372 + 0.417949198775061893j) + assert airyai(z).ae(-0.0162552579839056062 - 0.0018045715700210556j) + assert airybi(z).ae(-4.98856113282883371 + 2.08558537872180623j) + assert li(0) == 0.0 + assert li(1) == -inf + assert li(inf) == inf + assert isinstance(li(0.7), mpf) + assert si(inf).ae(pi/2) + assert si(-inf).ae(-pi/2) + assert ci(inf) == 0 + assert ci(0) == -inf + assert isinstance(ei(-0.7), mpf) + assert airyai(inf) == 0 + assert airybi(inf) == inf + assert airyai(-inf) == 0 + assert airybi(-inf) == 0 + assert fresnels(inf) == 0.5 + assert fresnelc(inf) == 0.5 + assert fresnels(-inf) == -0.5 + assert fresnelc(-inf) == -0.5 + assert shi(0) == 0 + assert shi(inf) == inf + assert shi(-inf) == -inf + assert chi(0) == -inf + assert chi(inf) == inf + +def test_ei(): + mp.dps = 15 + assert ei(0) == -inf + assert ei(inf) == inf + assert ei(-inf) == -0.0 + assert ei(20+70j).ae(6.1041351911152984397e6 - 2.7324109310519928872e6j) + # tests for the asymptotic expansion + # values checked with Mathematica ExpIntegralEi + mp.dps = 50 + r = ei(20000) + s = '3.8781962825045010930273870085501819470698476975019e+8681' + assert str(r) == s + r = ei(-200) + s = '-6.8852261063076355977108174824557929738368086933303e-90' + assert str(r) == s + r =ei(20000 + 10*j) + sre = '-3.255138234032069402493850638874410725961401274106e+8681' + sim = '-2.1081929993474403520785942429469187647767369645423e+8681' + assert str(r.real) == sre and str(r.imag) == sim + mp.dps = 15 + # More asymptotic expansions + assert chi(-10**6+100j).ae('1.3077239389562548386e+434288 + 7.6808956999707408158e+434287j') + assert shi(-10**6+100j).ae('-1.3077239389562548386e+434288 - 7.6808956999707408158e+434287j') + mp.dps = 15 + assert ei(10j).ae(-0.0454564330044553726+3.2291439210137706686j) + assert ei(100j).ae(-0.0051488251426104921+3.1330217936839529126j) + u = ei(fmul(10**20, j, exact=True)) + assert u.real.ae(-6.4525128526578084421345e-21, abs_eps=0, rel_eps=8*eps) + assert u.imag.ae(pi) + assert ei(-10j).ae(-0.0454564330044553726-3.2291439210137706686j) + assert ei(-100j).ae(-0.0051488251426104921-3.1330217936839529126j) + u = ei(fmul(-10**20, j, exact=True)) + assert u.real.ae(-6.4525128526578084421345e-21, abs_eps=0, rel_eps=8*eps) + assert u.imag.ae(-pi) + assert ei(10+10j).ae(-1576.1504265768517448+436.9192317011328140j) + u = ei(-10+10j) + assert u.real.ae(7.6698978415553488362543e-7, abs_eps=0, rel_eps=8*eps) + assert u.imag.ae(3.141595611735621062025) + +def test_e1(): + mp.dps = 15 + assert e1(0) == inf + assert e1(inf) == 0 + assert e1(-inf) == mpc(-inf, -pi) + assert e1(10j).ae(0.045456433004455372635 + 0.087551267423977430100j) + assert e1(100j).ae(0.0051488251426104921444 - 0.0085708599058403258790j) + assert e1(fmul(10**20, j, exact=True)).ae(6.4525128526578084421e-21 - 7.6397040444172830039e-21j, abs_eps=0, rel_eps=8*eps) + assert e1(-10j).ae(0.045456433004455372635 - 0.087551267423977430100j) + assert e1(-100j).ae(0.0051488251426104921444 + 0.0085708599058403258790j) + assert e1(fmul(-10**20, j, exact=True)).ae(6.4525128526578084421e-21 + 7.6397040444172830039e-21j, abs_eps=0, rel_eps=8*eps) + +def test_expint(): + mp.dps = 15 + assert expint(0,0) == inf + assert expint(0,1).ae(1/e) + assert expint(0,1.5).ae(2/exp(1.5)/3) + assert expint(1,1).ae(-ei(-1)) + assert expint(2,0).ae(1) + assert expint(3,0).ae(1/2.) + assert expint(4,0).ae(1/3.) + assert expint(-2, 0.5).ae(26/sqrt(e)) + assert expint(-1,-1) == 0 + assert expint(-2,-1).ae(-e) + assert expint(5.5, 0).ae(2/9.) + assert expint(2.00000001,0).ae(100000000./100000001) + assert expint(2+3j,4-j).ae(0.0023461179581675065414+0.0020395540604713669262j) + assert expint('1.01', '1e-1000').ae(99.9999999899412802) + assert expint('1.000000000001', 3.5).ae(0.00697013985754701819446) + assert expint(2,3).ae(3*ei(-3)+exp(-3)) + assert (expint(10,20)*10**10).ae(0.694439055541231353) + assert expint(3,inf) == 0 + assert expint(3.2,inf) == 0 + assert expint(3.2+2j,inf) == 0 + assert expint(1,3j).ae(-0.11962978600800032763 + 0.27785620120457163717j) + assert expint(1,3).ae(0.013048381094197037413) + assert expint(1,-3).ae(-ei(3)-pi*j) + #assert expint(3) == expint(1,3) + assert expint(1,-20).ae(-25615652.66405658882 - 3.1415926535897932385j) + assert expint(1000000,0).ae(1./999999) + assert expint(0,2+3j).ae(-0.025019798357114678171 + 0.027980439405104419040j) + assert expint(-1,2+3j).ae(-0.022411973626262070419 + 0.038058922011377716932j) + assert expint(-1.5,0) == inf + +def test_trig_integrals(): + mp.dps = 30 + assert si(mpf(1)/1000000).ae('0.000000999999999999944444444444446111') + assert ci(mpf(1)/1000000).ae('-13.2382948930629912435014366276') + assert si(10**10).ae('1.5707963267075846569685111517747537') + assert ci(10**10).ae('-4.87506025174822653785729773959e-11') + assert si(10**100).ae(pi/2) + assert (ci(10**100)*10**100).ae('-0.372376123661276688262086695553') + assert si(-3) == -si(3) + assert ci(-3).ae(ci(3) + pi*j) + # Test complex structure + mp.dps = 15 + assert mp.ci(50).ae(-0.0056283863241163054402) + assert mp.ci(50+2j).ae(-0.018378282946133067149+0.070352808023688336193j) + assert mp.ci(20j).ae(1.28078263320282943611e7+1.5707963267949j) + assert mp.ci(-2+20j).ae(-4.050116856873293505e6+1.207476188206989909e7j) + assert mp.ci(-50+2j).ae(-0.0183782829461330671+3.0712398455661049023j) + assert mp.ci(-50).ae(-0.0056283863241163054+3.1415926535897932385j) + assert mp.ci(-50-2j).ae(-0.0183782829461330671-3.0712398455661049023j) + assert mp.ci(-2-20j).ae(-4.050116856873293505e6-1.207476188206989909e7j) + assert mp.ci(-20j).ae(1.28078263320282943611e7-1.5707963267949j) + assert mp.ci(50-2j).ae(-0.018378282946133067149-0.070352808023688336193j) + assert mp.si(50).ae(1.5516170724859358947) + assert mp.si(50+2j).ae(1.497884414277228461-0.017515007378437448j) + assert mp.si(20j).ae(1.2807826332028294459e7j) + assert mp.si(-2+20j).ae(-1.20747603112735722103e7-4.050116856873293554e6j) + assert mp.si(-50+2j).ae(-1.497884414277228461-0.017515007378437448j) + assert mp.si(-50).ae(-1.5516170724859358947) + assert mp.si(-50-2j).ae(-1.497884414277228461+0.017515007378437448j) + assert mp.si(-2-20j).ae(-1.20747603112735722103e7+4.050116856873293554e6j) + assert mp.si(-20j).ae(-1.2807826332028294459e7j) + assert mp.si(50-2j).ae(1.497884414277228461+0.017515007378437448j) + assert mp.chi(50j).ae(-0.0056283863241163054+1.5707963267948966192j) + assert mp.chi(-2+50j).ae(-0.0183782829461330671+1.6411491348185849554j) + assert mp.chi(-20).ae(1.28078263320282943611e7+3.1415926535898j) + assert mp.chi(-20-2j).ae(-4.050116856873293505e6+1.20747571696809187053e7j) + assert mp.chi(-2-50j).ae(-0.0183782829461330671-1.6411491348185849554j) + assert mp.chi(-50j).ae(-0.0056283863241163054-1.5707963267948966192j) + assert mp.chi(2-50j).ae(-0.0183782829461330671-1.500443518771208283j) + assert mp.chi(20-2j).ae(-4.050116856873293505e6-1.20747603112735722951e7j) + assert mp.chi(20).ae(1.2807826332028294361e7) + assert mp.chi(2+50j).ae(-0.0183782829461330671+1.500443518771208283j) + assert mp.shi(50j).ae(1.5516170724859358947j) + assert mp.shi(-2+50j).ae(0.017515007378437448+1.497884414277228461j) + assert mp.shi(-20).ae(-1.2807826332028294459e7) + assert mp.shi(-20-2j).ae(4.050116856873293554e6-1.20747603112735722103e7j) + assert mp.shi(-2-50j).ae(0.017515007378437448-1.497884414277228461j) + assert mp.shi(-50j).ae(-1.5516170724859358947j) + assert mp.shi(2-50j).ae(-0.017515007378437448-1.497884414277228461j) + assert mp.shi(20-2j).ae(-4.050116856873293554e6-1.20747603112735722103e7j) + assert mp.shi(20).ae(1.2807826332028294459e7) + assert mp.shi(2+50j).ae(-0.017515007378437448+1.497884414277228461j) + def ae(x,y,tol=1e-12): + return abs(x-y) <= abs(y)*tol + assert fp.ci(fp.inf) == 0 + assert ae(fp.ci(fp.ninf), fp.pi*1j) + assert ae(fp.si(fp.inf), fp.pi/2) + assert ae(fp.si(fp.ninf), -fp.pi/2) + assert fp.si(0) == 0 + assert ae(fp.ci(50), -0.0056283863241163054402) + assert ae(fp.ci(50+2j), -0.018378282946133067149+0.070352808023688336193j) + assert ae(fp.ci(20j), 1.28078263320282943611e7+1.5707963267949j) + assert ae(fp.ci(-2+20j), -4.050116856873293505e6+1.207476188206989909e7j) + assert ae(fp.ci(-50+2j), -0.0183782829461330671+3.0712398455661049023j) + assert ae(fp.ci(-50), -0.0056283863241163054+3.1415926535897932385j) + assert ae(fp.ci(-50-2j), -0.0183782829461330671-3.0712398455661049023j) + assert ae(fp.ci(-2-20j), -4.050116856873293505e6-1.207476188206989909e7j) + assert ae(fp.ci(-20j), 1.28078263320282943611e7-1.5707963267949j) + assert ae(fp.ci(50-2j), -0.018378282946133067149-0.070352808023688336193j) + assert ae(fp.si(50), 1.5516170724859358947) + assert ae(fp.si(50+2j), 1.497884414277228461-0.017515007378437448j) + assert ae(fp.si(20j), 1.2807826332028294459e7j) + assert ae(fp.si(-2+20j), -1.20747603112735722103e7-4.050116856873293554e6j) + assert ae(fp.si(-50+2j), -1.497884414277228461-0.017515007378437448j) + assert ae(fp.si(-50), -1.5516170724859358947) + assert ae(fp.si(-50-2j), -1.497884414277228461+0.017515007378437448j) + assert ae(fp.si(-2-20j), -1.20747603112735722103e7+4.050116856873293554e6j) + assert ae(fp.si(-20j), -1.2807826332028294459e7j) + assert ae(fp.si(50-2j), 1.497884414277228461+0.017515007378437448j) + assert ae(fp.chi(50j), -0.0056283863241163054+1.5707963267948966192j) + assert ae(fp.chi(-2+50j), -0.0183782829461330671+1.6411491348185849554j) + assert ae(fp.chi(-20), 1.28078263320282943611e7+3.1415926535898j) + assert ae(fp.chi(-20-2j), -4.050116856873293505e6+1.20747571696809187053e7j) + assert ae(fp.chi(-2-50j), -0.0183782829461330671-1.6411491348185849554j) + assert ae(fp.chi(-50j), -0.0056283863241163054-1.5707963267948966192j) + assert ae(fp.chi(2-50j), -0.0183782829461330671-1.500443518771208283j) + assert ae(fp.chi(20-2j), -4.050116856873293505e6-1.20747603112735722951e7j) + assert ae(fp.chi(20), 1.2807826332028294361e7) + assert ae(fp.chi(2+50j), -0.0183782829461330671+1.500443518771208283j) + assert ae(fp.shi(50j), 1.5516170724859358947j) + assert ae(fp.shi(-2+50j), 0.017515007378437448+1.497884414277228461j) + assert ae(fp.shi(-20), -1.2807826332028294459e7) + assert ae(fp.shi(-20-2j), 4.050116856873293554e6-1.20747603112735722103e7j) + assert ae(fp.shi(-2-50j), 0.017515007378437448-1.497884414277228461j) + assert ae(fp.shi(-50j), -1.5516170724859358947j) + assert ae(fp.shi(2-50j), -0.017515007378437448-1.497884414277228461j) + assert ae(fp.shi(20-2j), -4.050116856873293554e6-1.20747603112735722103e7j) + assert ae(fp.shi(20), 1.2807826332028294459e7) + assert ae(fp.shi(2+50j), -0.017515007378437448+1.497884414277228461j) + +def test_airy(): + mp.dps = 15 + assert (airyai(10)*10**10).ae(1.1047532552898687) + assert (airybi(10)/10**9).ae(0.45564115354822515) + assert (airyai(1000)*10**9158).ae(9.306933063179556004) + assert (airybi(1000)/10**9154).ae(5.4077118391949465477) + assert airyai(-1000).ae(0.055971895773019918842) + assert airybi(-1000).ae(-0.083264574117080633012) + assert (airyai(100+100j)*10**188).ae(2.9099582462207032076 + 2.353013591706178756j) + assert (airybi(100+100j)/10**185).ae(1.7086751714463652039 - 3.1416590020830804578j) + +def test_hyper_0f1(): + mp.dps = 15 + v = 8.63911136507950465 + assert hyper([],[(1,3)],1.5).ae(v) + assert hyper([],[1/3.],1.5).ae(v) + assert hyp0f1(1/3.,1.5).ae(v) + assert hyp0f1((1,3),1.5).ae(v) + # Asymptotic expansion + assert hyp0f1(3,1e9).ae('4.9679055380347771271e+27455') + assert hyp0f1(3,1e9j).ae('-2.1222788784457702157e+19410 + 5.0840597555401854116e+19410j') + +def test_hyper_1f1(): + mp.dps = 15 + v = 1.2917526488617656673 + assert hyper([(1,2)],[(3,2)],0.7).ae(v) + assert hyper([(1,2)],[(3,2)],0.7+0j).ae(v) + assert hyper([0.5],[(3,2)],0.7).ae(v) + assert hyper([0.5],[1.5],0.7).ae(v) + assert hyper([0.5],[(3,2)],0.7+0j).ae(v) + assert hyper([0.5],[1.5],0.7+0j).ae(v) + assert hyper([(1,2)],[1.5+0j],0.7).ae(v) + assert hyper([0.5+0j],[1.5],0.7).ae(v) + assert hyper([0.5+0j],[1.5+0j],0.7+0j).ae(v) + assert hyp1f1(0.5,1.5,0.7).ae(v) + assert hyp1f1((1,2),1.5,0.7).ae(v) + # Asymptotic expansion + assert hyp1f1(2,3,1e10).ae('2.1555012157015796988e+4342944809') + assert (hyp1f1(2,3,1e10j)*10**10).ae(-0.97501205020039745852 - 1.7462392454512132074j) + # Shouldn't use asymptotic expansion + assert hyp1f1(-2, 1, 10000).ae(49980001) + # Bug + assert hyp1f1(1j,fraction(1,3),0.415-69.739j).ae(25.857588206024346592 + 15.738060264515292063j) + +def test_hyper_2f1(): + mp.dps = 15 + v = 1.0652207633823291032 + assert hyper([(1,2), (3,4)], [2], 0.3).ae(v) + assert hyper([(1,2), 0.75], [2], 0.3).ae(v) + assert hyper([0.5, 0.75], [2.0], 0.3).ae(v) + assert hyper([0.5, 0.75], [2.0], 0.3+0j).ae(v) + assert hyper([0.5+0j, (3,4)], [2.0], 0.3+0j).ae(v) + assert hyper([0.5+0j, (3,4)], [2.0], 0.3).ae(v) + assert hyper([0.5, (3,4)], [2.0+0j], 0.3).ae(v) + assert hyper([0.5+0j, 0.75+0j], [2.0+0j], 0.3+0j).ae(v) + v = 1.09234681096223231717 + 0.18104859169479360380j + assert hyper([(1,2),0.75+j], [2], 0.5).ae(v) + assert hyper([0.5,0.75+j], [2.0], 0.5).ae(v) + assert hyper([0.5,0.75+j], [2.0], 0.5+0j).ae(v) + assert hyper([0.5,0.75+j], [2.0+0j], 0.5+0j).ae(v) + v = 0.9625 - 0.125j + assert hyper([(3,2),-1],[4], 0.1+j/3).ae(v) + assert hyper([1.5,-1.0],[4], 0.1+j/3).ae(v) + assert hyper([1.5,-1.0],[4+0j], 0.1+j/3).ae(v) + assert hyper([1.5+0j,-1.0+0j],[4+0j], 0.1+j/3).ae(v) + v = 1.02111069501693445001 - 0.50402252613466859521j + assert hyper([(2,10),(3,10)],[(4,10)],1.5).ae(v) + assert hyper([0.2,(3,10)],[0.4+0j],1.5).ae(v) + assert hyper([0.2,(3,10)],[0.4+0j],1.5+0j).ae(v) + v = 0.76922501362865848528 + 0.32640579593235886194j + assert hyper([(2,10),(3,10)],[(4,10)],4+2j).ae(v) + assert hyper([0.2,(3,10)],[0.4+0j],4+2j).ae(v) + assert hyper([0.2,(3,10)],[(4,10)],4+2j).ae(v) + +def test_hyper_2f1_hard(): + mp.dps = 15 + # Singular cases + assert hyp2f1(2,-1,-1,3).ae(7) + assert hyp2f1(2,-1,-1,3,eliminate_all=True).ae(0.25) + assert hyp2f1(2,-2,-2,3).ae(34) + assert hyp2f1(2,-2,-2,3,eliminate_all=True).ae(0.25) + assert hyp2f1(2,-2,-3,3) == 14 + assert hyp2f1(2,-3,-2,3) == inf + assert hyp2f1(2,-1.5,-1.5,3) == 0.25 + assert hyp2f1(1,2,3,0) == 1 + assert hyp2f1(0,1,0,0) == 1 + assert hyp2f1(0,0,0,0) == 1 + assert isnan(hyp2f1(1,1,0,0)) + assert hyp2f1(2,-1,-5, 0.25+0.25j).ae(1.1+0.1j) + assert hyp2f1(2,-5,-5, 0.25+0.25j, eliminate=False).ae(163./128 + 125./128*j) + assert hyp2f1(0.7235, -1, -5, 0.3).ae(1.04341) + assert hyp2f1(0.7235, -5, -5, 0.3, eliminate=False).ae(1.2939225017815903812) + assert hyp2f1(-1,-2,4,1) == 1.5 + assert hyp2f1(1,2,-3,1) == inf + assert hyp2f1(-2,-2,1,1) == 6 + assert hyp2f1(1,-2,-4,1).ae(5./3) + assert hyp2f1(0,-6,-4,1) == 1 + assert hyp2f1(0,-3,-4,1) == 1 + assert hyp2f1(0,0,0,1) == 1 + assert hyp2f1(1,0,0,1,eliminate=False) == 1 + assert hyp2f1(1,1,0,1) == inf + assert hyp2f1(1,-6,-4,1) == inf + assert hyp2f1(-7.2,-0.5,-4.5,1) == 0 + assert hyp2f1(-7.2,-1,-2,1).ae(-2.6) + assert hyp2f1(1,-0.5,-4.5, 1) == inf + assert hyp2f1(1,0.5,-4.5, 1) == -inf + # Check evaluation on / close to unit circle + z = exp(j*pi/3) + w = (nthroot(2,3)+1)*exp(j*pi/12)/nthroot(3,4)**3 + assert hyp2f1('1/2','1/6','1/3', z).ae(w) + assert hyp2f1('1/2','1/6','1/3', z.conjugate()).ae(w.conjugate()) + assert hyp2f1(0.25, (1,3), 2, '0.999').ae(1.06826449496030635) + assert hyp2f1(0.25, (1,3), 2, '1.001').ae(1.06867299254830309446-0.00001446586793975874j) + assert hyp2f1(0.25, (1,3), 2, -1).ae(0.96656584492524351673) + assert hyp2f1(0.25, (1,3), 2, j).ae(0.99041766248982072266+0.03777135604180735522j) + assert hyp2f1(2,3,5,'0.99').ae(27.699347904322690602) + assert hyp2f1((3,2),-0.5,3,'0.99').ae(0.68403036843911661388) + assert hyp2f1(2,3,5,1j).ae(0.37290667145974386127+0.59210004902748285917j) + assert fsum([hyp2f1((7,10),(2,3),(-1,2), 0.95*exp(j*k)) for k in range(1,15)]).ae(52.851400204289452922+6.244285013912953225j) + assert fsum([hyp2f1((7,10),(2,3),(-1,2), 1.05*exp(j*k)) for k in range(1,15)]).ae(54.506013786220655330-3.000118813413217097j) + assert fsum([hyp2f1((7,10),(2,3),(-1,2), exp(j*k)) for k in range(1,15)]).ae(55.792077935955314887+1.731986485778500241j) + assert hyp2f1(2,2.5,-3.25,0.999).ae(218373932801217082543180041.33) + # Branches + assert hyp2f1(1,1,2,1.01).ae(4.5595744415723676911-3.1104877758314784539j) + assert hyp2f1(1,1,2,1.01+0.1j).ae(2.4149427480552782484+1.4148224796836938829j) + assert hyp2f1(1,1,2,3+4j).ae(0.14576709331407297807+0.48379185417980360773j) + assert hyp2f1(1,1,2,4).ae(-0.27465307216702742285 - 0.78539816339744830962j) + assert hyp2f1(1,1,2,-4).ae(0.40235947810852509365) + # Other: + # Cancellation with a large parameter involved (bug reported on sage-devel) + assert hyp2f1(112, (51,10), (-9,10), -0.99999).ae(-1.6241361047970862961e-24, abs_eps=0, rel_eps=eps*16) + +def test_hyper_3f2_etc(): + assert hyper([1,2,3],[1.5,8],-1).ae(0.67108992351533333030) + assert hyper([1,2,3,4],[5,6,7], -1).ae(0.90232988035425506008) + assert hyper([1,2,3],[1.25,5], 1).ae(28.924181329701905701) + assert hyper([1,2,3,4],[5,6,7],5).ae(1.5192307344006649499-1.1529845225075537461j) + assert hyper([1,2,3,4,5],[6,7,8,9],-1).ae(0.96288759462882357253) + assert hyper([1,2,3,4,5],[6,7,8,9],1).ae(1.0428697385885855841) + assert hyper([1,2,3,4,5],[6,7,8,9],5).ae(1.33980653631074769423-0.07143405251029226699j) + assert hyper([1,2.79,3.08,4.37],[5.2,6.1,7.3],5).ae(1.0996321464692607231-1.7748052293979985001j) + assert hyper([1,1,1],[1,2],1) == inf + assert hyper([1,1,1],[2,(101,100)],1).ae(100.01621213528313220) + # slow -- covered by doctests + #assert hyper([1,1,1],[2,3],0.9999).ae(1.2897972005319693905) + +def test_hyper_u(): + mp.dps = 15 + assert hyperu(2,-3,0).ae(0.05) + assert hyperu(2,-3.5,0).ae(4./99) + assert hyperu(2,0,0) == 0.5 + assert hyperu(-5,1,0) == -120 + assert hyperu(-5,2,0) == inf + assert hyperu(-5,-2,0) == 0 + assert hyperu(7,7,3).ae(0.00014681269365593503986) #exp(3)*gammainc(-6,3) + assert hyperu(2,-3,4).ae(0.011836478100271995559) + assert hyperu(3,4,5).ae(1./125) + assert hyperu(2,3,0.0625) == 256 + assert hyperu(-1,2,0.25+0.5j) == -1.75+0.5j + assert hyperu(0.5,1.5,7.25).ae(2/sqrt(29)) + assert hyperu(2,6,pi).ae(0.55804439825913399130) + assert (hyperu((3,2),8,100+201j)*10**4).ae(-0.3797318333856738798 - 2.9974928453561707782j) + assert (hyperu((5,2),(-1,2),-5000)*10**10).ae(-5.6681877926881664678j) + # XXX: fails because of undetected cancellation in low level series code + # Alternatively: could use asymptotic series here, if convergence test + # tweaked back to recognize this one + #assert (hyperu((5,2),(-1,2),-500)*10**7).ae(-1.82526906001593252847j) + +def test_hyper_2f0(): + mp.dps = 15 + assert hyper([1,2],[],3) == hyp2f0(1,2,3) + assert hyp2f0(2,3,7).ae(0.0116108068639728714668 - 0.0073727413865865802130j) + assert hyp2f0(2,3,0) == 1 + assert hyp2f0(0,0,0) == 1 + assert hyp2f0(-1,-1,1).ae(2) + assert hyp2f0(-4,1,1.5).ae(62.5) + assert hyp2f0(-4,1,50).ae(147029801) + assert hyp2f0(-4,1,0.0001).ae(0.99960011997600240000) + assert hyp2f0(0.5,0.25,0.001).ae(1.0001251174078538115) + assert hyp2f0(0.5,0.25,3+4j).ae(0.85548875824755163518 + 0.21636041283392292973j) + # Important: cancellation check + assert hyp2f0((1,6),(5,6),-0.02371708245126284498).ae(0.996785723120804309) + # Should be exact; polynomial case + assert hyp2f0(-2,1,0.5+0.5j,zeroprec=200) == 0 + assert hyp2f0(1,-2,0.5+0.5j,zeroprec=200) == 0 + # There used to be a bug in thresholds that made one of the following hang + for d in [15, 50, 80]: + mp.dps = d + assert hyp2f0(1.5, 0.5, 0.009).ae('1.006867007239309717945323585695344927904000945829843527398772456281301440034218290443367270629519483 + 1.238277162240704919639384945859073461954721356062919829456053965502443570466701567100438048602352623e-46j') + +def test_hyper_1f2(): + mp.dps = 15 + assert hyper([1],[2,3],4) == hyp1f2(1,2,3,4) + a1,b1,b2 = (1,10),(2,3),1./16 + assert hyp1f2(a1,b1,b2,10).ae(298.7482725554557568) + assert hyp1f2(a1,b1,b2,100).ae(224128961.48602947604) + assert hyp1f2(a1,b1,b2,1000).ae(1.1669528298622675109e+27) + assert hyp1f2(a1,b1,b2,10000).ae(2.4780514622487212192e+86) + assert hyp1f2(a1,b1,b2,100000).ae(1.3885391458871523997e+274) + assert hyp1f2(a1,b1,b2,1000000).ae('9.8851796978960318255e+867') + assert hyp1f2(a1,b1,b2,10**7).ae('1.1505659189516303646e+2746') + assert hyp1f2(a1,b1,b2,10**8).ae('1.4672005404314334081e+8685') + assert hyp1f2(a1,b1,b2,10**20).ae('3.6888217332150976493e+8685889636') + assert hyp1f2(a1,b1,b2,10*j).ae(-16.163252524618572878 - 44.321567896480184312j) + assert hyp1f2(a1,b1,b2,100*j).ae(61938.155294517848171 + 637349.45215942348739j) + assert hyp1f2(a1,b1,b2,1000*j).ae(8455057657257695958.7 + 6261969266997571510.6j) + assert hyp1f2(a1,b1,b2,10000*j).ae(-8.9771211184008593089e+60 + 4.6550528111731631456e+59j) + assert hyp1f2(a1,b1,b2,100000*j).ae(2.6398091437239324225e+193 + 4.1658080666870618332e+193j) + assert hyp1f2(a1,b1,b2,1000000*j).ae('3.5999042951925965458e+613 + 1.5026014707128947992e+613j') + assert hyp1f2(a1,b1,b2,10**7*j).ae('-8.3208715051623234801e+1939 - 3.6752883490851869429e+1941j') + assert hyp1f2(a1,b1,b2,10**8*j).ae('2.0724195707891484454e+6140 - 1.3276619482724266387e+6141j') + assert hyp1f2(a1,b1,b2,10**20*j).ae('-1.1734497974795488504e+6141851462 + 1.1498106965385471542e+6141851462j') + +def test_hyper_2f3(): + mp.dps = 15 + assert hyper([1,2],[3,4,5],6) == hyp2f3(1,2,3,4,5,6) + a1,a2,b1,b2,b3 = (1,10),(2,3),(3,10), 2, 1./16 + # Check asymptotic expansion + assert hyp2f3(a1,a2,b1,b2,b3,10).ae(128.98207160698659976) + assert hyp2f3(a1,a2,b1,b2,b3,1000).ae(6.6309632883131273141e25) + assert hyp2f3(a1,a2,b1,b2,b3,10000).ae(4.6863639362713340539e84) + assert hyp2f3(a1,a2,b1,b2,b3,100000).ae(8.6632451236103084119e271) + assert hyp2f3(a1,a2,b1,b2,b3,10**6).ae('2.0291718386574980641e865') + assert hyp2f3(a1,a2,b1,b2,b3,10**7).ae('7.7639836665710030977e2742') + assert hyp2f3(a1,a2,b1,b2,b3,10**8).ae('3.2537462584071268759e8681') + assert hyp2f3(a1,a2,b1,b2,b3,10**20).ae('1.2966030542911614163e+8685889627') + assert hyp2f3(a1,a2,b1,b2,b3,10*j).ae(-18.551602185587547854 - 13.348031097874113552j) + assert hyp2f3(a1,a2,b1,b2,b3,100*j).ae(78634.359124504488695 + 74459.535945281973996j) + assert hyp2f3(a1,a2,b1,b2,b3,1000*j).ae(597682550276527901.59 - 65136194809352613.078j) + assert hyp2f3(a1,a2,b1,b2,b3,10000*j).ae(-1.1779696326238582496e+59 + 1.2297607505213133872e+59j) + assert hyp2f3(a1,a2,b1,b2,b3,100000*j).ae(2.9844228969804380301e+191 + 7.5587163231490273296e+190j) + assert hyp2f3(a1,a2,b1,b2,b3,1000000*j).ae('7.4859161049322370311e+610 - 2.8467477015940090189e+610j') + assert hyp2f3(a1,a2,b1,b2,b3,10**7*j).ae('-1.7477645579418800826e+1938 - 1.7606522995808116405e+1938j') + assert hyp2f3(a1,a2,b1,b2,b3,10**8*j).ae('-1.6932731942958401784e+6137 - 2.4521909113114629368e+6137j') + assert hyp2f3(a1,a2,b1,b2,b3,10**20*j).ae('-2.0988815677627225449e+6141851451 + 5.7708223542739208681e+6141851452j') + +def test_hyper_2f2(): + mp.dps = 15 + assert hyper([1,2],[3,4],5) == hyp2f2(1,2,3,4,5) + a1,a2,b1,b2 = (3,10),4,(1,2),1./16 + assert hyp2f2(a1,a2,b1,b2,10).ae(448225936.3377556696) + assert hyp2f2(a1,a2,b1,b2,10000).ae('1.2012553712966636711e+4358') + assert hyp2f2(a1,a2,b1,b2,-20000).ae(-0.04182343755661214626) + assert hyp2f2(a1,a2,b1,b2,10**20).ae('1.1148680024303263661e+43429448190325182840') + +def test_orthpoly(): + mp.dps = 15 + assert jacobi(-4,2,3,0.7).ae(22800./4913) + assert jacobi(3,2,4,5.5) == 4133.125 + assert jacobi(1.5,5/6.,4,0).ae(-1.0851951434075508417) + assert jacobi(-2, 1, 2, 4).ae(-0.16) + assert jacobi(2, -1, 2.5, 4).ae(34.59375) + #assert jacobi(2, -1, 2, 4) == 28.5 + assert legendre(5, 7) == 129367 + assert legendre(0.5,0).ae(0.53935260118837935667) + assert legendre(-1,-1) == 1 + assert legendre(0,-1) == 1 + assert legendre(0, 1) == 1 + assert legendre(1, -1) == -1 + assert legendre(7, 1) == 1 + assert legendre(7, -1) == -1 + assert legendre(8,1.5).ae(15457523./32768) + assert legendre(j,-j).ae(2.4448182735671431011 + 0.6928881737669934843j) + assert chebyu(5,1) == 6 + assert chebyt(3,2) == 26 + assert legendre(3.5,-1) == inf + assert legendre(4.5,-1) == -inf + assert legendre(3.5+1j,-1) == mpc(inf,inf) + assert legendre(4.5+1j,-1) == mpc(-inf,-inf) + assert laguerre(4, -2, 3).ae(-1.125) + assert laguerre(3, 1+j, 0.5).ae(0.2291666666666666667 + 2.5416666666666666667j) + +def test_hermite(): + mp.dps = 15 + assert hermite(-2, 0).ae(0.5) + assert hermite(-1, 0).ae(0.88622692545275801365) + assert hermite(0, 0).ae(1) + assert hermite(1, 0) == 0 + assert hermite(2, 0).ae(-2) + assert hermite(0, 2).ae(1) + assert hermite(1, 2).ae(4) + assert hermite(1, -2).ae(-4) + assert hermite(2, -2).ae(14) + assert hermite(0.5, 0).ae(0.69136733903629335053) + assert hermite(9, 0) == 0 + assert hermite(4,4).ae(3340) + assert hermite(3,4).ae(464) + assert hermite(-4,4).ae(0.00018623860287512396181) + assert hermite(-3,4).ae(0.0016540169879668766270) + assert hermite(9, 2.5j).ae(13638725j) + assert hermite(9, -2.5j).ae(-13638725j) + assert hermite(9, 100).ae(511078883759363024000) + assert hermite(9, -100).ae(-511078883759363024000) + assert hermite(9, 100j).ae(512922083920643024000j) + assert hermite(9, -100j).ae(-512922083920643024000j) + assert hermite(-9.5, 2.5j).ae(-2.9004951258126778174e-6 + 1.7601372934039951100e-6j) + assert hermite(-9.5, -2.5j).ae(-2.9004951258126778174e-6 - 1.7601372934039951100e-6j) + assert hermite(-9.5, 100).ae(1.3776300722767084162e-22, abs_eps=0, rel_eps=eps) + assert hermite(-9.5, -100).ae('1.3106082028470671626e4355') + assert hermite(-9.5, 100j).ae(-9.7900218581864768430e-23 - 9.7900218581864768430e-23j, abs_eps=0, rel_eps=eps) + assert hermite(-9.5, -100j).ae(-9.7900218581864768430e-23 + 9.7900218581864768430e-23j, abs_eps=0, rel_eps=eps) + assert hermite(2+3j, -1-j).ae(851.3677063883687676 - 1496.4373467871007997j) + +def test_gegenbauer(): + mp.dps = 15 + assert gegenbauer(1,2,3).ae(12) + assert gegenbauer(2,3,4).ae(381) + assert gegenbauer(0,0,0) == 0 + assert gegenbauer(2,-1,3) == 0 + assert gegenbauer(-7, 0.5, 3).ae(8989) + assert gegenbauer(1, -0.5, 3).ae(-3) + assert gegenbauer(1, -1.5, 3).ae(-9) + assert gegenbauer(1, -0.5, 3).ae(-3) + assert gegenbauer(-0.5, -0.5, 3).ae(-2.6383553159023906245) + assert gegenbauer(2+3j, 1-j, 3+4j).ae(14.880536623203696780 + 20.022029711598032898j) + #assert gegenbauer(-2, -0.5, 3).ae(-12) + +def test_legenp(): + mp.dps = 15 + assert legenp(2,0,4) == legendre(2,4) + assert legenp(-2, -1, 0.5).ae(0.43301270189221932338) + assert legenp(-2, -1, 0.5, type=3).ae(0.43301270189221932338j) + assert legenp(-2, 1, 0.5).ae(-0.86602540378443864676) + assert legenp(2+j, 3+4j, -j).ae(134742.98773236786148 + 429782.72924463851745j) + assert legenp(2+j, 3+4j, -j, type=3).ae(802.59463394152268507 - 251.62481308942906447j) + assert legenp(2,4,3).ae(0) + assert legenp(2,4,3,type=3).ae(0) + assert legenp(2,1,0.5).ae(-1.2990381056766579701) + assert legenp(2,1,0.5,type=3).ae(1.2990381056766579701j) + assert legenp(3,2,3).ae(-360) + assert legenp(3,3,3).ae(240j*2**0.5) + assert legenp(3,4,3).ae(0) + assert legenp(0,0.5,2).ae(0.52503756790433198939 - 0.52503756790433198939j) + assert legenp(-1,-0.5,2).ae(0.60626116232846498110 + 0.60626116232846498110j) + assert legenp(-2,0.5,2).ae(1.5751127037129959682 - 1.5751127037129959682j) + assert legenp(-2,0.5,-0.5).ae(-0.85738275810499171286) + +def test_legenq(): + mp.dps = 15 + f = legenq + # Evaluation at poles + assert isnan(f(3,2,1)) + assert isnan(f(3,2,-1)) + assert isnan(f(3,2,1,type=3)) + assert isnan(f(3,2,-1,type=3)) + # Evaluation at 0 + assert f(0,1,0,type=2).ae(-1) + assert f(-2,2,0,type=2,zeroprec=200).ae(0) + assert f(1.5,3,0,type=2).ae(-2.2239343475841951023) + assert f(0,1,0,type=3).ae(j) + assert f(-2,2,0,type=3,zeroprec=200).ae(0) + assert f(1.5,3,0,type=3).ae(2.2239343475841951022*(1-1j)) + # Standard case, degree 0 + assert f(0,0,-1.5).ae(-0.8047189562170501873 + 1.5707963267948966192j) + assert f(0,0,-0.5).ae(-0.54930614433405484570) + assert f(0,0,0,zeroprec=200).ae(0) + assert f(0,0,0.5).ae(0.54930614433405484570) + assert f(0,0,1.5).ae(0.8047189562170501873 - 1.5707963267948966192j) + assert f(0,0,-1.5,type=3).ae(-0.80471895621705018730) + assert f(0,0,-0.5,type=3).ae(-0.5493061443340548457 - 1.5707963267948966192j) + assert f(0,0,0,type=3).ae(-1.5707963267948966192j) + assert f(0,0,0.5,type=3).ae(0.5493061443340548457 - 1.5707963267948966192j) + assert f(0,0,1.5,type=3).ae(0.80471895621705018730) + # Standard case, degree 1 + assert f(1,0,-1.5).ae(0.2070784343255752810 - 2.3561944901923449288j) + assert f(1,0,-0.5).ae(-0.72534692783297257715) + assert f(1,0,0).ae(-1) + assert f(1,0,0.5).ae(-0.72534692783297257715) + assert f(1,0,1.5).ae(0.2070784343255752810 - 2.3561944901923449288j) + # Standard case, degree 2 + assert f(2,0,-1.5).ae(-0.0635669991240192885 + 4.5160394395353277803j) + assert f(2,0,-0.5).ae(0.81866326804175685571) + assert f(2,0,0,zeroprec=200).ae(0) + assert f(2,0,0.5).ae(-0.81866326804175685571) + assert f(2,0,1.5).ae(0.0635669991240192885 - 4.5160394395353277803j) + # Misc orders and degrees + assert f(2,3,1.5,type=2).ae(-5.7243340223994616228j) + assert f(2,3,1.5,type=3).ae(-5.7243340223994616228) + assert f(2,3,0.5,type=2).ae(-12.316805742712016310) + assert f(2,3,0.5,type=3).ae(-12.316805742712016310j) + assert f(2,3,-1.5,type=2).ae(-5.7243340223994616228j) + assert f(2,3,-1.5,type=3).ae(5.7243340223994616228) + assert f(2,3,-0.5,type=2).ae(-12.316805742712016310) + assert f(2,3,-0.5,type=3).ae(-12.316805742712016310j) + assert f(2+3j, 3+4j, 0.5, type=3).ae(0.0016119404873235186807 - 0.0005885900510718119836j) + assert f(2+3j, 3+4j, -1.5, type=3).ae(0.008451400254138808670 + 0.020645193304593235298j) + assert f(-2.5,1,-1.5).ae(3.9553395527435335749j) + assert f(-2.5,1,-0.5).ae(1.9290561746445456908) + assert f(-2.5,1,0).ae(1.2708196271909686299) + assert f(-2.5,1,0.5).ae(-0.31584812990742202869) + assert f(-2.5,1,1.5).ae(-3.9553395527435335742 + 0.2993235655044701706j) + assert f(-2.5,1,-1.5,type=3).ae(0.29932356550447017254j) + assert f(-2.5,1,-0.5,type=3).ae(-0.3158481299074220287 - 1.9290561746445456908j) + assert f(-2.5,1,0,type=3).ae(1.2708196271909686292 - 1.2708196271909686299j) + assert f(-2.5,1,0.5,type=3).ae(1.9290561746445456907 + 0.3158481299074220287j) + assert f(-2.5,1,1.5,type=3).ae(-0.29932356550447017254) + +def test_agm(): + mp.dps = 15 + assert agm(0,0) == 0 + assert agm(0,1) == 0 + assert agm(1,1) == 1 + assert agm(7,7) == 7 + assert agm(j,j) == j + assert (1/agm(1,sqrt(2))).ae(0.834626841674073186) + assert agm(1,2).ae(1.4567910310469068692) + assert agm(1,3).ae(1.8636167832448965424) + assert agm(1,j).ae(0.599070117367796104+0.599070117367796104j) + assert agm(2) == agm(1,2) + assert agm(-3,4).ae(0.63468509766550907+1.3443087080896272j) + +def test_gammainc(): + mp.dps = 15 + assert gammainc(2,5).ae(6*exp(-5)) + assert gammainc(2,0,5).ae(1-6*exp(-5)) + assert gammainc(2,3,5).ae(-6*exp(-5)+4*exp(-3)) + assert gammainc(-2.5,-0.5).ae(-0.9453087204829418812-5.3164237738936178621j) + assert gammainc(0,2,4).ae(0.045121158298212213088) + assert gammainc(0,3).ae(0.013048381094197037413) + assert gammainc(0,2+j,1-j).ae(0.00910653685850304839-0.22378752918074432574j) + assert gammainc(0,1-j).ae(0.00028162445198141833+0.17932453503935894015j) + assert gammainc(3,4,5,True).ae(0.11345128607046320253) + assert gammainc(3.5,0,inf).ae(gamma(3.5)) + assert gammainc(-150.5,500).ae('6.9825435345798951153e-627') + assert gammainc(-150.5,800).ae('4.6885137549474089431e-788') + assert gammainc(-3.5, -20.5).ae(0.27008820585226911 - 1310.31447140574997636j) + assert gammainc(-3.5, -200.5).ae(0.27008820585226911 - 5.3264597096208368435e76j) # XXX real part + assert gammainc(0,0,2) == inf + assert gammainc(1,b=1).ae(0.6321205588285576784) + assert gammainc(3,2,2) == 0 + assert gammainc(2,3+j,3-j).ae(-0.28135485191849314194j) + assert gammainc(4+0j,1).ae(5.8860710587430771455) + # GH issue #301 + assert gammainc(-1,-1).ae(-0.8231640121031084799 + 3.1415926535897932385j) + assert gammainc(-2,-1).ae(1.7707229202810768576 - 1.5707963267948966192j) + assert gammainc(-3,-1).ae(-1.4963349162467073643 + 0.5235987755982988731j) + assert gammainc(-4,-1).ae(1.05365418617643814992 - 0.13089969389957471827j) + # Regularized upper gamma + assert isnan(gammainc(0, 0, regularized=True)) + assert gammainc(-1, 0, regularized=True) == inf + assert gammainc(1, 0, regularized=True) == 1 + assert gammainc(0, 5, regularized=True) == 0 + assert gammainc(0, 2+3j, regularized=True) == 0 + assert gammainc(0, 5000, regularized=True) == 0 + assert gammainc(0, 10**30, regularized=True) == 0 + assert gammainc(-1, 5, regularized=True) == 0 + assert gammainc(-1, 5000, regularized=True) == 0 + assert gammainc(-1, 10**30, regularized=True) == 0 + assert gammainc(-1, -5, regularized=True) == 0 + assert gammainc(-1, -5000, regularized=True) == 0 + assert gammainc(-1, -10**30, regularized=True) == 0 + assert gammainc(-1, 3+4j, regularized=True) == 0 + assert gammainc(1, 5, regularized=True).ae(exp(-5)) + assert gammainc(1, 5000, regularized=True).ae(exp(-5000)) + assert gammainc(1, 10**30, regularized=True).ae(exp(-10**30)) + assert gammainc(1, 3+4j, regularized=True).ae(exp(-3-4j)) + assert gammainc(-1000000,2).ae('1.3669297209397347754e-301037', abs_eps=0, rel_eps=8*eps) + assert gammainc(-1000000,2,regularized=True) == 0 + assert gammainc(-1000000,3+4j).ae('-1.322575609404222361e-698979 - 4.9274570591854533273e-698978j', abs_eps=0, rel_eps=8*eps) + assert gammainc(-1000000,3+4j,regularized=True) == 0 + assert gammainc(2+3j, 4+5j, regularized=True).ae(0.085422013530993285774-0.052595379150390078503j) + assert gammainc(1000j, 1000j, regularized=True).ae(0.49702647628921131761 + 0.00297355675013575341j) + # Generalized + assert gammainc(3,4,2) == -gammainc(3,2,4) + assert gammainc(4, 2, 3).ae(1.2593494302978947396) + assert gammainc(4, 2, 3, regularized=True).ae(0.20989157171631578993) + assert gammainc(0, 2, 3).ae(0.035852129613864082155) + assert gammainc(0, 2, 3, regularized=True) == 0 + assert gammainc(-1, 2, 3).ae(0.015219822548487616132) + assert gammainc(-1, 2, 3, regularized=True) == 0 + assert gammainc(0, 2, 3).ae(0.035852129613864082155) + assert gammainc(0, 2, 3, regularized=True) == 0 + # Should use upper gammas + assert gammainc(5, 10000, 12000).ae('1.1359381951461801687e-4327', abs_eps=0, rel_eps=8*eps) + # Should use lower gammas + assert gammainc(10000, 2, 3).ae('8.1244514125995785934e4765') + # GH issue 306 + assert gammainc(3,-1-1j) == 0 + assert gammainc(3,-1+1j) == 0 + assert gammainc(2,-1) == 0 + assert gammainc(2,-1+0j) == 0 + assert gammainc(2+0j,-1) == 0 + +def test_gammainc_expint_n(): + # These tests are intended to check all cases of the low-level code + # for upper gamma and expint with small integer index. + # Need to cover positive/negative arguments; small/large/huge arguments + # for both positive and negative indices, as well as indices 0 and 1 + # which may be special-cased + mp.dps = 15 + assert expint(-3,3.5).ae(0.021456366563296693987) + assert expint(-2,3.5).ae(0.014966633183073309405) + assert expint(-1,3.5).ae(0.011092916359219041088) + assert expint(0,3.5).ae(0.0086278238349481430685) + assert expint(1,3.5).ae(0.0069701398575483929193) + assert expint(2,3.5).ae(0.0058018939208991255223) + assert expint(3,3.5).ae(0.0049453773495857807058) + assert expint(-3,-3.5).ae(-4.6618170604073311319) + assert expint(-2,-3.5).ae(-5.5996974157555515963) + assert expint(-1,-3.5).ae(-6.7582555017739415818) + assert expint(0,-3.5).ae(-9.4615577024835182145) + assert expint(1,-3.5).ae(-13.925353995152335292 - 3.1415926535897932385j) + assert expint(2,-3.5).ae(-15.62328702434085977 - 10.995574287564276335j) + assert expint(3,-3.5).ae(-10.783026313250347722 - 19.242255003237483586j) + assert expint(-3,350).ae(2.8614825451252838069e-155, abs_eps=0, rel_eps=8*eps) + assert expint(-2,350).ae(2.8532837224504675901e-155, abs_eps=0, rel_eps=8*eps) + assert expint(-1,350).ae(2.8451316155828634555e-155, abs_eps=0, rel_eps=8*eps) + assert expint(0,350).ae(2.8370258275042797989e-155, abs_eps=0, rel_eps=8*eps) + assert expint(1,350).ae(2.8289659656701459404e-155, abs_eps=0, rel_eps=8*eps) + assert expint(2,350).ae(2.8209516419468505006e-155, abs_eps=0, rel_eps=8*eps) + assert expint(3,350).ae(2.8129824725501272171e-155, abs_eps=0, rel_eps=8*eps) + assert expint(-3,-350).ae(-2.8528796154044839443e+149) + assert expint(-2,-350).ae(-2.8610072121701264351e+149) + assert expint(-1,-350).ae(-2.8691813842677537647e+149) + assert expint(0,-350).ae(-2.8774025343659421709e+149) + u = expint(1,-350) + assert u.ae(-2.8856710698020863568e+149) + assert u.imag.ae(-3.1415926535897932385) + u = expint(2,-350) + assert u.ae(-2.8939874026504650534e+149) + assert u.imag.ae(-1099.5574287564276335) + u = expint(3,-350) + assert u.ae(-2.9023519497915044349e+149) + assert u.imag.ae(-192422.55003237483586) + assert expint(-3,350000000000000000000000).ae('2.1592908471792544286e-152003068666138139677919', abs_eps=0, rel_eps=8*eps) + assert expint(-2,350000000000000000000000).ae('2.1592908471792544286e-152003068666138139677919', abs_eps=0, rel_eps=8*eps) + assert expint(-1,350000000000000000000000).ae('2.1592908471792544286e-152003068666138139677919', abs_eps=0, rel_eps=8*eps) + assert expint(0,350000000000000000000000).ae('2.1592908471792544286e-152003068666138139677919', abs_eps=0, rel_eps=8*eps) + assert expint(1,350000000000000000000000).ae('2.1592908471792544286e-152003068666138139677919', abs_eps=0, rel_eps=8*eps) + assert expint(2,350000000000000000000000).ae('2.1592908471792544286e-152003068666138139677919', abs_eps=0, rel_eps=8*eps) + assert expint(3,350000000000000000000000).ae('2.1592908471792544286e-152003068666138139677919', abs_eps=0, rel_eps=8*eps) + assert expint(-3,-350000000000000000000000).ae('-3.7805306852415755699e+152003068666138139677871') + assert expint(-2,-350000000000000000000000).ae('-3.7805306852415755699e+152003068666138139677871') + assert expint(-1,-350000000000000000000000).ae('-3.7805306852415755699e+152003068666138139677871') + assert expint(0,-350000000000000000000000).ae('-3.7805306852415755699e+152003068666138139677871') + u = expint(1,-350000000000000000000000) + assert u.ae('-3.7805306852415755699e+152003068666138139677871') + assert u.imag.ae(-3.1415926535897932385) + u = expint(2,-350000000000000000000000) + assert u.imag.ae(-1.0995574287564276335e+24) + assert u.ae('-3.7805306852415755699e+152003068666138139677871') + u = expint(3,-350000000000000000000000) + assert u.imag.ae(-1.9242255003237483586e+47) + assert u.ae('-3.7805306852415755699e+152003068666138139677871') + # Small case; no branch cut + assert gammainc(-3,3.5).ae(0.00010020262545203707109) + assert gammainc(-2,3.5).ae(0.00040370427343557393517) + assert gammainc(-1,3.5).ae(0.0016576839773997501492) + assert gammainc(0,3.5).ae(0.0069701398575483929193) + assert gammainc(1,3.5).ae(0.03019738342231850074) + assert gammainc(2,3.5).ae(0.13588822540043325333) + assert gammainc(3,3.5).ae(0.64169439772426814072) + # Small case; with branch cut + assert gammainc(-3,-3.5).ae(0.03595832954467563286 + 0.52359877559829887308j) + assert gammainc(-2,-3.5).ae(-0.88024704597962022221 - 1.5707963267948966192j) + assert gammainc(-1,-3.5).ae(4.4637962926688170771 + 3.1415926535897932385j) + assert gammainc(0,-3.5).ae(-13.925353995152335292 - 3.1415926535897932385j) + assert gammainc(1,-3.5).ae(33.115451958692313751) + assert gammainc(2,-3.5).ae(-82.788629896730784377) + assert gammainc(3,-3.5).ae(240.08702670051927469) + # Asymptotic case; no branch cut + assert gammainc(-3,350).ae(6.5424095113340358813e-163, abs_eps=0, rel_eps=8*eps) + assert gammainc(-2,350).ae(2.296312222489899769e-160, abs_eps=0, rel_eps=8*eps) + assert gammainc(-1,350).ae(8.059861834133858573e-158, abs_eps=0, rel_eps=8*eps) + assert gammainc(0,350).ae(2.8289659656701459404e-155, abs_eps=0, rel_eps=8*eps) + assert gammainc(1,350).ae(9.9295903962649792963e-153, abs_eps=0, rel_eps=8*eps) + assert gammainc(2,350).ae(3.485286229089007733e-150, abs_eps=0, rel_eps=8*eps) + assert gammainc(3,350).ae(1.2233453960006379793e-147, abs_eps=0, rel_eps=8*eps) + # Asymptotic case; branch cut + u = gammainc(-3,-350) + assert u.ae(6.7889565783842895085e+141) + assert u.imag.ae(0.52359877559829887308) + u = gammainc(-2,-350) + assert u.ae(-2.3692668977889832121e+144) + assert u.imag.ae(-1.5707963267948966192) + u = gammainc(-1,-350) + assert u.ae(8.2685354361441858669e+146) + assert u.imag.ae(3.1415926535897932385) + u = gammainc(0,-350) + assert u.ae(-2.8856710698020863568e+149) + assert u.imag.ae(-3.1415926535897932385) + u = gammainc(1,-350) + assert u.ae(1.0070908870280797598e+152) + assert u.imag == 0 + u = gammainc(2,-350) + assert u.ae(-3.5147471957279983618e+154) + assert u.imag == 0 + u = gammainc(3,-350) + assert u.ae(1.2266568422179417091e+157) + assert u.imag == 0 + # Extreme asymptotic case + assert gammainc(-3,350000000000000000000000).ae('5.0362468738874738859e-152003068666138139677990', abs_eps=0, rel_eps=8*eps) + assert gammainc(-2,350000000000000000000000).ae('1.7626864058606158601e-152003068666138139677966', abs_eps=0, rel_eps=8*eps) + assert gammainc(-1,350000000000000000000000).ae('6.1694024205121555102e-152003068666138139677943', abs_eps=0, rel_eps=8*eps) + assert gammainc(0,350000000000000000000000).ae('2.1592908471792544286e-152003068666138139677919', abs_eps=0, rel_eps=8*eps) + assert gammainc(1,350000000000000000000000).ae('7.5575179651273905e-152003068666138139677896', abs_eps=0, rel_eps=8*eps) + assert gammainc(2,350000000000000000000000).ae('2.645131287794586675e-152003068666138139677872', abs_eps=0, rel_eps=8*eps) + assert gammainc(3,350000000000000000000000).ae('9.2579595072810533625e-152003068666138139677849', abs_eps=0, rel_eps=8*eps) + u = gammainc(-3,-350000000000000000000000) + assert u.ae('8.8175642804468234866e+152003068666138139677800') + assert u.imag.ae(0.52359877559829887308) + u = gammainc(-2,-350000000000000000000000) + assert u.ae('-3.0861474981563882203e+152003068666138139677824') + assert u.imag.ae(-1.5707963267948966192) + u = gammainc(-1,-350000000000000000000000) + assert u.ae('1.0801516243547358771e+152003068666138139677848') + assert u.imag.ae(3.1415926535897932385) + u = gammainc(0,-350000000000000000000000) + assert u.ae('-3.7805306852415755699e+152003068666138139677871') + assert u.imag.ae(-3.1415926535897932385) + assert gammainc(1,-350000000000000000000000).ae('1.3231857398345514495e+152003068666138139677895') + assert gammainc(2,-350000000000000000000000).ae('-4.6311500894209300731e+152003068666138139677918') + assert gammainc(3,-350000000000000000000000).ae('1.6209025312973255256e+152003068666138139677942') + +def test_incomplete_beta(): + mp.dps = 15 + assert betainc(-2,-3,0.5,0.75).ae(63.4305673311255413583969) + assert betainc(4.5,0.5+2j,2.5,6).ae(0.2628801146130621387903065 + 0.5162565234467020592855378j) + assert betainc(4,5,0,6).ae(90747.77142857142857142857) + +def test_erf(): + mp.dps = 15 + assert erf(0) == 0 + assert erf(1).ae(0.84270079294971486934) + assert erf(3+4j).ae(-120.186991395079444098 - 27.750337293623902498j) + assert erf(-4-3j).ae(-0.99991066178539168236 + 0.00004972026054496604j) + assert erf(pi).ae(0.99999112385363235839) + assert erf(1j).ae(1.6504257587975428760j) + assert erf(-1j).ae(-1.6504257587975428760j) + assert isinstance(erf(1), mpf) + assert isinstance(erf(-1), mpf) + assert isinstance(erf(0), mpf) + assert isinstance(erf(0j), mpc) + assert erf(inf) == 1 + assert erf(-inf) == -1 + assert erfi(0) == 0 + assert erfi(1/pi).ae(0.371682698493894314) + assert erfi(inf) == inf + assert erfi(-inf) == -inf + assert erf(1+0j) == erf(1) + assert erfc(1+0j) == erfc(1) + assert erf(0.2+0.5j).ae(1 - erfc(0.2+0.5j)) + assert erfc(0) == 1 + assert erfc(1).ae(1-erf(1)) + assert erfc(-1).ae(1-erf(-1)) + assert erfc(1/pi).ae(1-erf(1/pi)) + assert erfc(-10) == 2 + assert erfc(-1000000) == 2 + assert erfc(-inf) == 2 + assert erfc(inf) == 0 + assert isnan(erfc(nan)) + assert (erfc(10**4)*mpf(10)**43429453).ae('3.63998738656420') + assert erf(8+9j).ae(-1072004.2525062051158 + 364149.91954310255423j) + assert erfc(8+9j).ae(1072005.2525062051158 - 364149.91954310255423j) + assert erfc(-8-9j).ae(-1072003.2525062051158 + 364149.91954310255423j) + mp.dps = 50 + # This one does not use the asymptotic series + assert (erfc(10)*10**45).ae('2.0884875837625447570007862949577886115608181193212') + # This one does + assert (erfc(50)*10**1088).ae('2.0709207788416560484484478751657887929322509209954') + mp.dps = 15 + assert str(erfc(10**50)) == '3.66744826532555e-4342944819032518276511289189166050822943970058036665661144537831658646492088707747292249493384317534' + assert erfinv(0) == 0 + assert erfinv(0.5).ae(0.47693627620446987338) + assert erfinv(-0.5).ae(-0.47693627620446987338) + assert erfinv(1) == inf + assert erfinv(-1) == -inf + assert erf(erfinv(0.95)).ae(0.95) + assert erf(erfinv(0.999999999995)).ae(0.999999999995) + assert erf(erfinv(-0.999999999995)).ae(-0.999999999995) + mp.dps = 50 + assert erf(erfinv('0.99999999999999999999999999999995')).ae('0.99999999999999999999999999999995') + assert erf(erfinv('0.999999999999999999999999999999995')).ae('0.999999999999999999999999999999995') + assert erf(erfinv('-0.999999999999999999999999999999995')).ae('-0.999999999999999999999999999999995') + mp.dps = 15 + # Complex asymptotic expansions + v = erfc(50j) + assert v.real == 1 + assert v.imag.ae('-6.1481820666053078736e+1083') + assert erfc(-100+5j).ae(2) + assert (erfc(100+5j)*10**4335).ae(2.3973567853824133572 - 3.9339259530609420597j) + assert erfc(100+100j).ae(0.00065234366376857698698 - 0.0039357263629214118437j) + +def test_pdf(): + mp.dps = 15 + assert npdf(-inf) == 0 + assert npdf(inf) == 0 + assert npdf(5,0,2).ae(npdf(5+4,4,2)) + assert quadts(lambda x: npdf(x,-0.5,0.8), [-inf, inf]) == 1 + assert ncdf(0) == 0.5 + assert ncdf(3,3) == 0.5 + assert ncdf(-inf) == 0 + assert ncdf(inf) == 1 + assert ncdf(10) == 1 + # Verify that this is computed accurately + assert (ncdf(-10)*10**24).ae(7.619853024160526) + +def test_lambertw(): + mp.dps = 15 + assert lambertw(0) == 0 + assert lambertw(0+0j) == 0 + assert lambertw(inf) == inf + assert isnan(lambertw(nan)) + assert lambertw(inf,1).real == inf + assert lambertw(inf,1).imag.ae(2*pi) + assert lambertw(-inf,1).real == inf + assert lambertw(-inf,1).imag.ae(3*pi) + assert lambertw(0,-1) == -inf + assert lambertw(0,1) == -inf + assert lambertw(0,3) == -inf + assert lambertw(e).ae(1) + assert lambertw(1).ae(0.567143290409783873) + assert lambertw(-pi/2).ae(j*pi/2) + assert lambertw(-log(2)/2).ae(-log(2)) + assert lambertw(0.25).ae(0.203888354702240164) + assert lambertw(-0.25).ae(-0.357402956181388903) + assert lambertw(-1./10000,0).ae(-0.000100010001500266719) + assert lambertw(-0.25,-1).ae(-2.15329236411034965) + assert lambertw(0.25,-1).ae(-3.00899800997004620-4.07652978899159763j) + assert lambertw(-0.25,-1).ae(-2.15329236411034965) + assert lambertw(0.25,1).ae(-3.00899800997004620+4.07652978899159763j) + assert lambertw(-0.25,1).ae(-3.48973228422959210+7.41405453009603664j) + assert lambertw(-4).ae(0.67881197132094523+1.91195078174339937j) + assert lambertw(-4,1).ae(-0.66743107129800988+7.76827456802783084j) + assert lambertw(-4,-1).ae(0.67881197132094523-1.91195078174339937j) + assert lambertw(1000).ae(5.24960285240159623) + assert lambertw(1000,1).ae(4.91492239981054535+5.44652615979447070j) + assert lambertw(1000,-1).ae(4.91492239981054535-5.44652615979447070j) + assert lambertw(1000,5).ae(3.5010625305312892+29.9614548941181328j) + assert lambertw(3+4j).ae(1.281561806123775878+0.533095222020971071j) + assert lambertw(-0.4+0.4j).ae(-0.10396515323290657+0.61899273315171632j) + assert lambertw(3+4j,1).ae(-0.11691092896595324+5.61888039871282334j) + assert lambertw(3+4j,-1).ae(0.25856740686699742-3.85211668616143559j) + assert lambertw(-0.5,-1).ae(-0.794023632344689368-0.770111750510379110j) + assert lambertw(-1./10000,1).ae(-11.82350837248724344+6.80546081842002101j) + assert lambertw(-1./10000,-1).ae(-11.6671145325663544) + assert lambertw(-1./10000,-2).ae(-11.82350837248724344-6.80546081842002101j) + assert lambertw(-1./100000,4).ae(-14.9186890769540539+26.1856750178782046j) + assert lambertw(-1./100000,5).ae(-15.0931437726379218666+32.5525721210262290086j) + assert lambertw((2+j)/10).ae(0.173704503762911669+0.071781336752835511j) + assert lambertw((2+j)/10,1).ae(-3.21746028349820063+4.56175438896292539j) + assert lambertw((2+j)/10,-1).ae(-3.03781405002993088-3.53946629633505737j) + assert lambertw((2+j)/10,4).ae(-4.6878509692773249+23.8313630697683291j) + assert lambertw(-(2+j)/10).ae(-0.226933772515757933-0.164986470020154580j) + assert lambertw(-(2+j)/10,1).ae(-2.43569517046110001+0.76974067544756289j) + assert lambertw(-(2+j)/10,-1).ae(-3.54858738151989450-6.91627921869943589j) + assert lambertw(-(2+j)/10,4).ae(-4.5500846928118151+20.6672982215434637j) + mp.dps = 50 + assert lambertw(pi).ae('1.073658194796149172092178407024821347547745350410314531') + mp.dps = 15 + # Former bug in generated branch + assert lambertw(-0.5+0.002j).ae(-0.78917138132659918344 + 0.76743539379990327749j) + assert lambertw(-0.5-0.002j).ae(-0.78917138132659918344 - 0.76743539379990327749j) + assert lambertw(-0.448+0.4j).ae(-0.11855133765652382241 + 0.66570534313583423116j) + assert lambertw(-0.448-0.4j).ae(-0.11855133765652382241 - 0.66570534313583423116j) + assert lambertw(-0.65475+0.0001j).ae(-0.61053421111385310898+1.0396534993944097723803j) + # Huge branch index + w = lambertw(1,10**20) + assert w.real.ae(-47.889578926290259164) + assert w.imag.ae(6.2831853071795864769e+20) + +def test_lambertw_hard(): + def check(x,y): + y = convert(y) + type_ok = True + if isinstance(y, mpf): + type_ok = isinstance(x, mpf) + real_ok = abs(x.real-y.real) <= abs(y.real)*8*eps + imag_ok = abs(x.imag-y.imag) <= abs(y.imag)*8*eps + #print x, y, abs(x.real-y.real), abs(x.imag-y.imag) + return real_ok and imag_ok + # Evaluation near 0 + mp.dps = 15 + assert check(lambertw(1e-10), 9.999999999000000000e-11) + assert check(lambertw(-1e-10), -1.000000000100000000e-10) + assert check(lambertw(1e-10j), 9.999999999999999999733e-21 + 9.99999999999999999985e-11j) + assert check(lambertw(-1e-10j), 9.999999999999999999733e-21 - 9.99999999999999999985e-11j) + assert check(lambertw(1e-10,1), -26.303186778379041559 + 3.265093911703828397j) + assert check(lambertw(-1e-10,1), -26.326236166739163892 + 6.526183280686333315j) + assert check(lambertw(1e-10j,1), -26.312931726911421551 + 4.896366881798013421j) + assert check(lambertw(-1e-10j,1), -26.297238779529035066 + 1.632807161345576513j) + assert check(lambertw(1e-10,-1), -26.303186778379041559 - 3.265093911703828397j) + assert check(lambertw(-1e-10,-1), -26.295238819246925694) + assert check(lambertw(1e-10j,-1), -26.297238779529035028 - 1.6328071613455765135j) + assert check(lambertw(-1e-10j,-1), -26.312931726911421551 - 4.896366881798013421j) + # Test evaluation very close to the branch point -1/e + # on the -1, 0, and 1 branches + add = lambda x, y: fadd(x,y,exact=True) + sub = lambda x, y: fsub(x,y,exact=True) + addj = lambda x, y: fadd(x,fmul(y,1j,exact=True),exact=True) + subj = lambda x, y: fadd(x,fmul(y,-1j,exact=True),exact=True) + mp.dps = 1500 + a = -1/e + 10*eps + d3 = mpf('1e-3') + d10 = mpf('1e-10') + d20 = mpf('1e-20') + d40 = mpf('1e-40') + d80 = mpf('1e-80') + d300 = mpf('1e-300') + d1000 = mpf('1e-1000') + mp.dps = 15 + # ---- Branch 0 ---- + # -1/e + eps + assert check(lambertw(add(a,d3)), -0.92802015005456704876) + assert check(lambertw(add(a,d10)), -0.99997668374140088071) + assert check(lambertw(add(a,d20)), -0.99999999976683560186) + assert lambertw(add(a,d40)) == -1 + assert lambertw(add(a,d80)) == -1 + assert lambertw(add(a,d300)) == -1 + assert lambertw(add(a,d1000)) == -1 + # -1/e - eps + assert check(lambertw(sub(a,d3)), -0.99819016149860989001+0.07367191188934638577j) + assert check(lambertw(sub(a,d10)), -0.9999999998187812114595992+0.0000233164398140346109194j) + assert check(lambertw(sub(a,d20)), -0.99999999999999999998187+2.331643981597124203344e-10j) + assert check(lambertw(sub(a,d40)), -1.0+2.33164398159712420336e-20j) + assert check(lambertw(sub(a,d80)), -1.0+2.33164398159712420336e-40j) + assert check(lambertw(sub(a,d300)), -1.0+2.33164398159712420336e-150j) + assert check(lambertw(sub(a,d1000)), mpc(-1,'2.33164398159712420336e-500')) + # -1/e + eps*j + assert check(lambertw(addj(a,d3)), -0.94790387486938526634+0.05036819639190132490j) + assert check(lambertw(addj(a,d10)), -0.9999835127872943680999899+0.0000164870314895821225256j) + assert check(lambertw(addj(a,d20)), -0.999999999835127872929987+1.64872127051890935830e-10j) + assert check(lambertw(addj(a,d40)), -0.9999999999999999999835+1.6487212707001281468305e-20j) + assert check(lambertw(addj(a,d80)), -1.0 + 1.64872127070012814684865e-40j) + assert check(lambertw(addj(a,d300)), -1.0 + 1.64872127070012814684865e-150j) + assert check(lambertw(addj(a,d1000)), mpc(-1.0,'1.64872127070012814684865e-500')) + # -1/e - eps*j + assert check(lambertw(subj(a,d3)), -0.94790387486938526634-0.05036819639190132490j) + assert check(lambertw(subj(a,d10)), -0.9999835127872943680999899-0.0000164870314895821225256j) + assert check(lambertw(subj(a,d20)), -0.999999999835127872929987-1.64872127051890935830e-10j) + assert check(lambertw(subj(a,d40)), -0.9999999999999999999835-1.6487212707001281468305e-20j) + assert check(lambertw(subj(a,d80)), -1.0 - 1.64872127070012814684865e-40j) + assert check(lambertw(subj(a,d300)), -1.0 - 1.64872127070012814684865e-150j) + assert check(lambertw(subj(a,d1000)), mpc(-1.0,'-1.64872127070012814684865e-500')) + # ---- Branch 1 ---- + assert check(lambertw(addj(a,d3),1), -3.088501303219933378005990 + 7.458676867597474813950098j) + assert check(lambertw(addj(a,d80),1), -3.088843015613043855957087 + 7.461489285654254556906117j) + assert check(lambertw(addj(a,d300),1), -3.088843015613043855957087 + 7.461489285654254556906117j) + assert check(lambertw(addj(a,d1000),1), -3.088843015613043855957087 + 7.461489285654254556906117j) + assert check(lambertw(subj(a,d3),1), -1.0520914180450129534365906 + 0.0539925638125450525673175j) + assert check(lambertw(subj(a,d10),1), -1.0000164872127056318529390 + 0.000016487393927159250398333077j) + assert check(lambertw(subj(a,d20),1), -1.0000000001648721270700128 + 1.64872127088134693542628e-10j) + assert check(lambertw(subj(a,d40),1), -1.000000000000000000016487 + 1.64872127070012814686677e-20j) + assert check(lambertw(subj(a,d80),1), -1.0 + 1.64872127070012814684865e-40j) + assert check(lambertw(subj(a,d300),1), -1.0 + 1.64872127070012814684865e-150j) + assert check(lambertw(subj(a,d1000),1), mpc(-1.0, '1.64872127070012814684865e-500')) + # ---- Branch -1 ---- + # -1/e + eps + assert check(lambertw(add(a,d3),-1), -1.075608941186624989414945) + assert check(lambertw(add(a,d10),-1), -1.000023316621036696460620) + assert check(lambertw(add(a,d20),-1), -1.000000000233164398177834) + assert lambertw(add(a,d40),-1) == -1 + assert lambertw(add(a,d80),-1) == -1 + assert lambertw(add(a,d300),-1) == -1 + assert lambertw(add(a,d1000),-1) == -1 + # -1/e - eps + assert check(lambertw(sub(a,d3),-1), -0.99819016149860989001-0.07367191188934638577j) + assert check(lambertw(sub(a,d10),-1), -0.9999999998187812114595992-0.0000233164398140346109194j) + assert check(lambertw(sub(a,d20),-1), -0.99999999999999999998187-2.331643981597124203344e-10j) + assert check(lambertw(sub(a,d40),-1), -1.0-2.33164398159712420336e-20j) + assert check(lambertw(sub(a,d80),-1), -1.0-2.33164398159712420336e-40j) + assert check(lambertw(sub(a,d300),-1), -1.0-2.33164398159712420336e-150j) + assert check(lambertw(sub(a,d1000),-1), mpc(-1,'-2.33164398159712420336e-500')) + # -1/e + eps*j + assert check(lambertw(addj(a,d3),-1), -1.0520914180450129534365906 - 0.0539925638125450525673175j) + assert check(lambertw(addj(a,d10),-1), -1.0000164872127056318529390 - 0.0000164873939271592503983j) + assert check(lambertw(addj(a,d20),-1), -1.0000000001648721270700 - 1.64872127088134693542628e-10j) + assert check(lambertw(addj(a,d40),-1), -1.00000000000000000001648 - 1.6487212707001281468667726e-20j) + assert check(lambertw(addj(a,d80),-1), -1.0 - 1.64872127070012814684865e-40j) + assert check(lambertw(addj(a,d300),-1), -1.0 - 1.64872127070012814684865e-150j) + assert check(lambertw(addj(a,d1000),-1), mpc(-1.0,'-1.64872127070012814684865e-500')) + # -1/e - eps*j + assert check(lambertw(subj(a,d3),-1), -3.088501303219933378005990-7.458676867597474813950098j) + assert check(lambertw(subj(a,d10),-1), -3.088843015579260686911033-7.461489285372968780020716j) + assert check(lambertw(subj(a,d20),-1), -3.088843015613043855953708-7.461489285654254556877988j) + assert check(lambertw(subj(a,d40),-1), -3.088843015613043855957087-7.461489285654254556906117j) + assert check(lambertw(subj(a,d80),-1), -3.088843015613043855957087 - 7.461489285654254556906117j) + assert check(lambertw(subj(a,d300),-1), -3.088843015613043855957087 - 7.461489285654254556906117j) + assert check(lambertw(subj(a,d1000),-1), -3.088843015613043855957087 - 7.461489285654254556906117j) + # One more case, testing higher precision + mp.dps = 500 + x = -1/e + mpf('1e-13') + ans = "-0.99999926266961377166355784455394913638782494543377383"\ + "744978844374498153493943725364881490261187530235150668593869563"\ + "168276697689459394902153960200361935311512317183678882" + mp.dps = 15 + assert lambertw(x).ae(ans) + mp.dps = 50 + assert lambertw(x).ae(ans) + mp.dps = 150 + assert lambertw(x).ae(ans) + +def test_meijerg(): + mp.dps = 15 + assert meijerg([[2,3],[1]],[[0.5,2],[3,4]], 2.5).ae(4.2181028074787439386) + assert meijerg([[],[1+j]],[[1],[1]], 3+4j).ae(271.46290321152464592 - 703.03330399954820169j) + assert meijerg([[0.25],[1]],[[0.5],[2]],0) == 0 + assert meijerg([[0],[]],[[0,0,'1/3','2/3'], []], '2/27').ae(2.2019391389653314120) + # Verify 1/z series being used + assert meijerg([[-3],[-0.5]], [[-1],[-2.5]], -0.5).ae(-1.338096165935754898687431) + assert meijerg([[1-(-1)],[1-(-2.5)]], [[1-(-3)],[1-(-0.5)]], -2.0).ae(-1.338096165935754898687431) + assert meijerg([[-3],[-0.5]], [[-1],[-2.5]], -1).ae(-(pi+4)/(4*pi)) + a = 2.5 + b = 1.25 + for z in [mpf(0.25), mpf(2)]: + x1 = hyp1f1(a,b,z) + x2 = gamma(b)/gamma(a)*meijerg([[1-a],[]],[[0],[1-b]],-z) + x3 = gamma(b)/gamma(a)*meijerg([[1-0],[1-(1-b)]],[[1-(1-a)],[]],-1/z) + assert x1.ae(x2) + assert x1.ae(x3) + +def test_appellf1(): + mp.dps = 15 + assert appellf1(2,-2,1,1,2,3).ae(-1.75) + assert appellf1(2,1,-2,1,2,3).ae(-8) + assert appellf1(2,1,-2,1,0.5,0.25).ae(1.5) + assert appellf1(-2,1,3,2,3,3).ae(19) + assert appellf1(1,2,3,4,0.5,0.125).ae( 1.53843285792549786518) + +def test_coulomb(): + # Note: most tests are doctests + # Test for a bug: + mp.dps = 15 + assert coulombg(mpc(-5,0),2,3).ae(20.087729487721430394) + +def test_hyper_param_accuracy(): + mp.dps = 15 + As = [n+1e-10 for n in range(-5,-1)] + Bs = [n+1e-10 for n in range(-12,-5)] + assert hyper(As,Bs,10).ae(-381757055858.652671927) + assert legenp(0.5, 100, 0.25).ae(-2.4124576567211311755e+144) + assert (hyp1f1(1000,1,-100)*10**24).ae(5.2589445437370169113) + assert (hyp2f1(10, -900, 10.5, 0.99)*10**24).ae(1.9185370579660768203) + assert (hyp2f1(1000,1.5,-3.5,-1.5)*10**385).ae(-2.7367529051334000764) + assert hyp2f1(-5, 10, 3, 0.5, zeroprec=500) == 0 + assert (hyp1f1(-10000, 1000, 100)*10**424).ae(-3.1046080515824859974) + assert (hyp2f1(1000,1.5,-3.5,-0.75,maxterms=100000)*10**231).ae(-4.0534790813913998643) + assert legenp(2, 3, 0.25) == 0 + pytest.raises(ValueError, lambda: hypercomb(lambda a: [([],[],[],[],[a],[-a],0.5)], [3])) + assert hypercomb(lambda a: [([],[],[],[],[a],[-a],0.5)], [3], infprec=200) == inf + assert meijerg([[],[]],[[0,0,0,0],[]],0.1).ae(1.5680822343832351418) + assert (besselk(400,400)*10**94).ae(1.4387057277018550583) + mp.dps = 5 + (hyp1f1(-5000.5, 1500, 100)*10**185).ae(8.5185229673381935522) + (hyp1f1(-5000, 1500, 100)*10**185).ae(9.1501213424563944311) + mp.dps = 15 + (hyp1f1(-5000.5, 1500, 100)*10**185).ae(8.5185229673381935522) + (hyp1f1(-5000, 1500, 100)*10**185).ae(9.1501213424563944311) + assert hyp0f1(fadd(-20,'1e-100',exact=True), 0.25).ae(1.85014429040102783e+49) + assert hyp0f1((-20*10**100+1, 10**100), 0.25).ae(1.85014429040102783e+49) + +def test_hypercomb_zero_pow(): + # check that 0^0 = 1 + assert hypercomb(lambda a: (([0],[a],[],[],[],[],0),), [0]) == 1 + assert meijerg([[-1.5],[]],[[0],[-0.75]],0).ae(1.4464090846320771425) + +def test_spherharm(): + mp.dps = 15 + t = 0.5; r = 0.25 + assert spherharm(0,0,t,r).ae(0.28209479177387814347) + assert spherharm(1,-1,t,r).ae(0.16048941205971996369 - 0.04097967481096344271j) + assert spherharm(1,0,t,r).ae(0.42878904414183579379) + assert spherharm(1,1,t,r).ae(-0.16048941205971996369 - 0.04097967481096344271j) + assert spherharm(2,-2,t,r).ae(0.077915886919031181734 - 0.042565643022253962264j) + assert spherharm(2,-1,t,r).ae(0.31493387233497459884 - 0.08041582001959297689j) + assert spherharm(2,0,t,r).ae(0.41330596756220761898) + assert spherharm(2,1,t,r).ae(-0.31493387233497459884 - 0.08041582001959297689j) + assert spherharm(2,2,t,r).ae(0.077915886919031181734 + 0.042565643022253962264j) + assert spherharm(3,-3,t,r).ae(0.033640236589690881646 - 0.031339125318637082197j) + assert spherharm(3,-2,t,r).ae(0.18091018743101461963 - 0.09883168583167010241j) + assert spherharm(3,-1,t,r).ae(0.42796713930907320351 - 0.10927795157064962317j) + assert spherharm(3,0,t,r).ae(0.27861659336351639787) + assert spherharm(3,1,t,r).ae(-0.42796713930907320351 - 0.10927795157064962317j) + assert spherharm(3,2,t,r).ae(0.18091018743101461963 + 0.09883168583167010241j) + assert spherharm(3,3,t,r).ae(-0.033640236589690881646 - 0.031339125318637082197j) + assert spherharm(0,-1,t,r) == 0 + assert spherharm(0,-2,t,r) == 0 + assert spherharm(0,1,t,r) == 0 + assert spherharm(0,2,t,r) == 0 + assert spherharm(1,2,t,r) == 0 + assert spherharm(1,3,t,r) == 0 + assert spherharm(1,-2,t,r) == 0 + assert spherharm(1,-3,t,r) == 0 + assert spherharm(2,3,t,r) == 0 + assert spherharm(2,4,t,r) == 0 + assert spherharm(2,-3,t,r) == 0 + assert spherharm(2,-4,t,r) == 0 + assert spherharm(3,4.5,0.5,0.25).ae(-22.831053442240790148 + 10.910526059510013757j) + assert spherharm(2+3j, 1-j, 1+j, 3+4j).ae(-2.6582752037810116935 - 1.0909214905642160211j) + assert spherharm(-6,2.5,t,r).ae(0.39383644983851448178 + 0.28414687085358299021j) + assert spherharm(-3.5, 3, 0.5, 0.25).ae(0.014516852987544698924 - 0.015582769591477628495j) + assert spherharm(-3, 3, 0.5, 0.25) == 0 + assert spherharm(-6, 3, 0.5, 0.25).ae(-0.16544349818782275459 - 0.15412657723253924562j) + assert spherharm(-6, 1.5, 0.5, 0.25).ae(0.032208193499767402477 + 0.012678000924063664921j) + assert spherharm(3,0,0,1).ae(0.74635266518023078283) + assert spherharm(3,-2,0,1) == 0 + assert spherharm(3,-2,1,1).ae(-0.16270707338254028971 - 0.35552144137546777097j) + +def test_qfunctions(): + mp.dps = 15 + assert qp(2,3,100).ae('2.7291482267247332183e2391') + +def test_issue_239(): + mp.prec = 150 + x = ldexp(2476979795053773,-52) + assert betainc(206, 385, 0, 0.55, 1).ae('0.99999999999999999999996570910644857895771110649954') + mp.dps = 15 + pytest.raises(ValueError, lambda: hyp2f1(-5,5,0.5,0.5)) + +# Extra stress testing for Bessel functions +# Reference zeros generated with the aid of scipy.special +# jn_zero, jnp_zero, yn_zero, ynp_zero + +V = 15 +M = 15 + +jn_small_zeros = \ +[[2.4048255576957728, + 5.5200781102863106, + 8.6537279129110122, + 11.791534439014282, + 14.930917708487786, + 18.071063967910923, + 21.211636629879259, + 24.352471530749303, + 27.493479132040255, + 30.634606468431975, + 33.775820213573569, + 36.917098353664044, + 40.058425764628239, + 43.19979171317673, + 46.341188371661814], + [3.8317059702075123, + 7.0155866698156188, + 10.173468135062722, + 13.323691936314223, + 16.470630050877633, + 19.615858510468242, + 22.760084380592772, + 25.903672087618383, + 29.046828534916855, + 32.189679910974404, + 35.332307550083865, + 38.474766234771615, + 41.617094212814451, + 44.759318997652822, + 47.901460887185447], + [5.1356223018406826, + 8.4172441403998649, + 11.619841172149059, + 14.795951782351261, + 17.959819494987826, + 21.116997053021846, + 24.270112313573103, + 27.420573549984557, + 30.569204495516397, + 33.7165195092227, + 36.86285651128381, + 40.008446733478192, + 43.153453778371463, + 46.297996677236919, + 49.442164110416873], + [6.3801618959239835, + 9.7610231299816697, + 13.015200721698434, + 16.223466160318768, + 19.409415226435012, + 22.582729593104442, + 25.748166699294978, + 28.908350780921758, + 32.064852407097709, + 35.218670738610115, + 38.370472434756944, + 41.520719670406776, + 44.669743116617253, + 47.817785691533302, + 50.965029906205183], + [7.5883424345038044, + 11.064709488501185, + 14.37253667161759, + 17.615966049804833, + 20.826932956962388, + 24.01901952477111, + 27.199087765981251, + 30.371007667117247, + 33.537137711819223, + 36.699001128744649, + 39.857627302180889, + 43.01373772335443, + 46.167853512924375, + 49.320360686390272, + 52.471551398458023], + [8.771483815959954, + 12.338604197466944, + 15.700174079711671, + 18.980133875179921, + 22.217799896561268, + 25.430341154222704, + 28.626618307291138, + 31.811716724047763, + 34.988781294559295, + 38.159868561967132, + 41.326383254047406, + 44.489319123219673, + 47.649399806697054, + 50.80716520300633, + 53.963026558378149], + [9.9361095242176849, + 13.589290170541217, + 17.003819667816014, + 20.320789213566506, + 23.58608443558139, + 26.820151983411405, + 30.033722386570469, + 33.233041762847123, + 36.422019668258457, + 39.603239416075404, + 42.778481613199507, + 45.949015998042603, + 49.11577372476426, + 52.279453903601052, + 55.440592068853149], + [11.086370019245084, + 14.821268727013171, + 18.287582832481726, + 21.641541019848401, + 24.934927887673022, + 28.191188459483199, + 31.42279419226558, + 34.637089352069324, + 37.838717382853611, + 41.030773691585537, + 44.21540850526126, + 47.394165755570512, + 50.568184679795566, + 53.738325371963291, + 56.905249991978781], + [12.225092264004655, + 16.037774190887709, + 19.554536430997055, + 22.94517313187462, + 26.266814641176644, + 29.54565967099855, + 32.795800037341462, + 36.025615063869571, + 39.240447995178135, + 42.443887743273558, + 45.638444182199141, + 48.825930381553857, + 52.007691456686903, + 55.184747939289049, + 58.357889025269694], + [13.354300477435331, + 17.241220382489128, + 20.807047789264107, + 24.233885257750552, + 27.583748963573006, + 30.885378967696675, + 34.154377923855096, + 37.400099977156589, + 40.628553718964528, + 43.843801420337347, + 47.048700737654032, + 50.245326955305383, + 53.435227157042058, + 56.619580266508436, + 59.799301630960228], + [14.475500686554541, + 18.433463666966583, + 22.046985364697802, + 25.509450554182826, + 28.887375063530457, + 32.211856199712731, + 35.499909205373851, + 38.761807017881651, + 42.004190236671805, + 45.231574103535045, + 48.447151387269394, + 51.653251668165858, + 54.851619075963349, + 58.043587928232478, + 61.230197977292681], + [15.589847884455485, + 19.61596690396692, + 23.275853726263409, + 26.773322545509539, + 30.17906117878486, + 33.526364075588624, + 36.833571341894905, + 40.111823270954241, + 43.368360947521711, + 46.608132676274944, + 49.834653510396724, + 53.050498959135054, + 56.257604715114484, + 59.457456908388002, + 62.651217388202912], + [16.698249933848246, + 20.789906360078443, + 24.494885043881354, + 28.026709949973129, + 31.45996003531804, + 34.829986990290238, + 38.156377504681354, + 41.451092307939681, + 44.721943543191147, + 47.974293531269048, + 51.211967004101068, + 54.437776928325074, + 57.653844811906946, + 60.8618046824805, + 64.062937824850136], + [17.801435153282442, + 21.95624406783631, + 25.705103053924724, + 29.270630441874802, + 32.731053310978403, + 36.123657666448762, + 39.469206825243883, + 42.780439265447158, + 46.06571091157561, + 49.330780096443524, + 52.579769064383396, + 55.815719876305778, + 59.040934037249271, + 62.257189393731728, + 65.465883797232125], + [18.899997953174024, + 23.115778347252756, + 26.907368976182104, + 30.505950163896036, + 33.993184984781542, + 37.408185128639695, + 40.772827853501868, + 44.100590565798301, + 47.400347780543231, + 50.678236946479898, + 53.93866620912693, + 57.184898598119301, + 60.419409852130297, + 63.644117508962281, + 66.860533012260103]] + +jnp_small_zeros = \ +[[0.0, + 3.8317059702075123, + 7.0155866698156188, + 10.173468135062722, + 13.323691936314223, + 16.470630050877633, + 19.615858510468242, + 22.760084380592772, + 25.903672087618383, + 29.046828534916855, + 32.189679910974404, + 35.332307550083865, + 38.474766234771615, + 41.617094212814451, + 44.759318997652822], + [1.8411837813406593, + 5.3314427735250326, + 8.5363163663462858, + 11.706004902592064, + 14.863588633909033, + 18.015527862681804, + 21.16436985918879, + 24.311326857210776, + 27.457050571059246, + 30.601922972669094, + 33.746182898667383, + 36.889987409236811, + 40.033444053350675, + 43.176628965448822, + 46.319597561173912], + [3.0542369282271403, + 6.7061331941584591, + 9.9694678230875958, + 13.170370856016123, + 16.347522318321783, + 19.512912782488205, + 22.671581772477426, + 25.826037141785263, + 28.977672772993679, + 32.127327020443474, + 35.275535050674691, + 38.422654817555906, + 41.568934936074314, + 44.714553532819734, + 47.859641607992093], + [4.2011889412105285, + 8.0152365983759522, + 11.345924310743006, + 14.585848286167028, + 17.78874786606647, + 20.9724769365377, + 24.144897432909265, + 27.310057930204349, + 30.470268806290424, + 33.626949182796679, + 36.781020675464386, + 39.933108623659488, + 43.083652662375079, + 46.232971081836478, + 49.381300092370349], + [5.3175531260839944, + 9.2823962852416123, + 12.681908442638891, + 15.964107037731551, + 19.196028800048905, + 22.401032267689004, + 25.589759681386733, + 28.767836217666503, + 31.938539340972783, + 35.103916677346764, + 38.265316987088158, + 41.423666498500732, + 44.579623137359257, + 47.733667523865744, + 50.886159153182682], + [6.4156163757002403, + 10.519860873772308, + 13.9871886301403, + 17.312842487884625, + 20.575514521386888, + 23.803581476593863, + 27.01030789777772, + 30.20284907898166, + 33.385443901010121, + 36.560777686880356, + 39.730640230067416, + 42.896273163494417, + 46.058566273567043, + 49.218174614666636, + 52.375591529563596], + [7.501266144684147, + 11.734935953042708, + 15.268181461097873, + 18.637443009666202, + 21.931715017802236, + 25.183925599499626, + 28.409776362510085, + 31.617875716105035, + 34.81339298429743, + 37.999640897715301, + 41.178849474321413, + 44.352579199070217, + 47.521956905768113, + 50.687817781723741, + 53.85079463676896], + [8.5778364897140741, + 12.932386237089576, + 16.529365884366944, + 19.941853366527342, + 23.268052926457571, + 26.545032061823576, + 29.790748583196614, + 33.015178641375142, + 36.224380548787162, + 39.422274578939259, + 42.611522172286684, + 45.793999658055002, + 48.971070951900596, + 52.143752969301988, + 55.312820330403446], + [9.6474216519972168, + 14.115518907894618, + 17.774012366915256, + 21.229062622853124, + 24.587197486317681, + 27.889269427955092, + 31.155326556188325, + 34.39662855427218, + 37.620078044197086, + 40.830178681822041, + 44.030010337966153, + 47.221758471887113, + 50.407020967034367, + 53.586995435398319, + 56.762598475105272], + [10.711433970699945, + 15.28673766733295, + 19.004593537946053, + 22.501398726777283, + 25.891277276839136, + 29.218563499936081, + 32.505247352375523, + 35.763792928808799, + 39.001902811514218, + 42.224638430753279, + 45.435483097475542, + 48.636922645305525, + 51.830783925834728, + 55.01844255063594, + 58.200955824859509], + [11.770876674955582, + 16.447852748486498, + 20.223031412681701, + 23.760715860327448, + 27.182021527190532, + 30.534504754007074, + 33.841965775135715, + 37.118000423665604, + 40.371068905333891, + 43.606764901379516, + 46.828959446564562, + 50.040428970943456, + 53.243223214220535, + 56.438892058982552, + 59.628631306921512], + [12.826491228033465, + 17.600266557468326, + 21.430854238060294, + 25.008518704644261, + 28.460857279654847, + 31.838424458616998, + 35.166714427392629, + 38.460388720328256, + 41.728625562624312, + 44.977526250903469, + 48.211333836373288, + 51.433105171422278, + 54.645106240447105, + 57.849056857839799, + 61.046288512821078], + [13.878843069697276, + 18.745090916814406, + 22.629300302835503, + 26.246047773946584, + 29.72897816891134, + 33.131449953571661, + 36.480548302231658, + 39.791940718940855, + 43.075486800191012, + 46.337772104541405, + 49.583396417633095, + 52.815686826850452, + 56.037118687012179, + 59.249577075517968, + 62.454525995970462], + [14.928374492964716, + 19.88322436109951, + 23.81938909003628, + 27.474339750968247, + 30.987394331665278, + 34.414545662167183, + 37.784378506209499, + 41.113512376883377, + 44.412454519229281, + 47.688252845993366, + 50.945849245830813, + 54.188831071035124, + 57.419876154678179, + 60.641030026538746, + 63.853885828967512], + [15.975438807484321, + 21.015404934568315, + 25.001971500138194, + 28.694271223110755, + 32.236969407878118, + 35.688544091185301, + 39.078998185245057, + 42.425854432866141, + 45.740236776624833, + 49.029635055514276, + 52.299319390331728, + 55.553127779547459, + 58.793933759028134, + 62.02393848337554, + 65.244860767043859]] + +yn_small_zeros = \ +[[0.89357696627916752, + 3.9576784193148579, + 7.0860510603017727, + 10.222345043496417, + 13.361097473872763, + 16.500922441528091, + 19.64130970088794, + 22.782028047291559, + 25.922957653180923, + 29.064030252728398, + 32.205204116493281, + 35.346452305214321, + 38.487756653081537, + 41.629104466213808, + 44.770486607221993], + [2.197141326031017, + 5.4296810407941351, + 8.5960058683311689, + 11.749154830839881, + 14.897442128336725, + 18.043402276727856, + 21.188068934142213, + 24.331942571356912, + 27.475294980449224, + 30.618286491641115, + 33.761017796109326, + 36.90355531614295, + 40.045944640266876, + 43.188218097393211, + 46.330399250701687], + [3.3842417671495935, + 6.7938075132682675, + 10.023477979360038, + 13.209986710206416, + 16.378966558947457, + 19.539039990286384, + 22.69395593890929, + 25.845613720902269, + 28.995080395650151, + 32.143002257627551, + 35.289793869635804, + 38.435733485446343, + 41.581014867297885, + 44.725777117640461, + 47.870122696676504], + [4.5270246611496439, + 8.0975537628604907, + 11.396466739595867, + 14.623077742393873, + 17.81845523294552, + 20.997284754187761, + 24.166235758581828, + 27.328799850405162, + 30.486989604098659, + 33.642049384702463, + 36.794791029185579, + 39.945767226378749, + 43.095367507846703, + 46.2438744334407, + 49.391498015725107], + [5.6451478942208959, + 9.3616206152445429, + 12.730144474090465, + 15.999627085382479, + 19.22442895931681, + 22.424810599698521, + 25.610267054939328, + 28.785893657666548, + 31.954686680031668, + 35.118529525584828, + 38.278668089521758, + 41.435960629910073, + 44.591018225353424, + 47.744288086361052, + 50.896105199722123], + [6.7471838248710219, + 10.597176726782031, + 14.033804104911233, + 17.347086393228382, + 20.602899017175335, + 23.826536030287532, + 27.030134937138834, + 30.220335654231385, + 33.401105611047908, + 36.574972486670962, + 39.743627733020277, + 42.908248189569535, + 46.069679073215439, + 49.228543693445843, + 52.385312123112282], + [7.8377378223268716, + 11.811037107609447, + 15.313615118517857, + 18.670704965906724, + 21.958290897126571, + 25.206207715021249, + 28.429037095235496, + 31.634879502950644, + 34.828638524084437, + 38.013473399691765, + 41.19151880917741, + 44.364272633271975, + 47.53281875312084, + 50.697961822183806, + 53.860312300118388], + [8.919605734873789, + 13.007711435388313, + 16.573915129085334, + 19.974342312352426, + 23.293972585596648, + 26.5667563757203, + 29.809531451608321, + 33.031769327150685, + 36.239265816598239, + 39.435790312675323, + 42.623910919472727, + 45.805442883111651, + 48.981708325514764, + 52.153694518185572, + 55.322154420959698], + [9.9946283820824834, + 14.190361295800141, + 17.817887841179873, + 21.26093227125945, + 24.612576377421522, + 27.910524883974868, + 31.173701563441602, + 34.412862242025045, + 37.634648706110989, + 40.843415321050884, + 44.04214994542435, + 47.232978012841169, + 50.417456447370186, + 53.596753874948731, + 56.771765754432457], + [11.064090256031013, + 15.361301343575925, + 19.047949646361388, + 22.532765416313869, + 25.91620496332662, + 29.2394205079349, + 32.523270869465881, + 35.779715464475261, + 39.016196664616095, + 42.237627509803703, + 45.4474001519274, + 48.647941127433196, + 51.841036928216499, + 55.028034667184916, + 58.209970905250097], + [12.128927704415439, + 16.522284394784426, + 20.265984501212254, + 23.791669719454272, + 27.206568881574774, + 30.555020011020762, + 33.859683872746356, + 37.133649760307504, + 40.385117593813002, + 43.619533085646856, + 46.840676630553575, + 50.051265851897857, + 53.253310556711732, + 56.448332488918971, + 59.637507005589829], + [13.189846995683845, + 17.674674253171487, + 21.473493977824902, + 25.03913093040942, + 28.485081336558058, + 31.858644293774859, + 35.184165245422787, + 38.475796636190897, + 41.742455848758449, + 44.990096293791186, + 48.222870660068338, + 51.443777308699826, + 54.655042589416311, + 57.858358441436511, + 61.055036135780528], + [14.247395665073945, + 18.819555894710682, + 22.671697117872794, + 26.276375544903892, + 29.752925495549038, + 33.151412708998983, + 36.497763772987645, + 39.807134090704376, + 43.089121522203808, + 46.350163579538652, + 49.594769786270069, + 52.82620892320143, + 56.046916910756961, + 59.258751140598783, + 62.463155567737854], + [15.30200785858925, + 19.957808654258601, + 23.861599172945054, + 27.504429642227545, + 31.011103429019229, + 34.434283425782942, + 37.801385632318459, + 41.128514139788358, + 44.425913324440663, + 47.700482714581842, + 50.957073905278458, + 54.199216028087261, + 57.429547607017405, + 60.65008661807661, + 63.862406280068586], + [16.354034360047551, + 21.090156519983806, + 25.044040298785627, + 28.724161640881914, + 32.260472459522644, + 35.708083982611664, + 39.095820003878235, + 42.440684315990936, + 45.75353669045622, + 49.041718113283529, + 52.310408280968073, + 55.56338698149062, + 58.803488508906895, + 62.032886550960831, + 65.253280088312461]] + +ynp_small_zeros = \ +[[2.197141326031017, + 5.4296810407941351, + 8.5960058683311689, + 11.749154830839881, + 14.897442128336725, + 18.043402276727856, + 21.188068934142213, + 24.331942571356912, + 27.475294980449224, + 30.618286491641115, + 33.761017796109326, + 36.90355531614295, + 40.045944640266876, + 43.188218097393211, + 46.330399250701687], + [3.6830228565851777, + 6.9414999536541757, + 10.123404655436613, + 13.285758156782854, + 16.440058007293282, + 19.590241756629495, + 22.738034717396327, + 25.884314618788867, + 29.029575819372535, + 32.174118233366201, + 35.318134458192094, + 38.461753870997549, + 41.605066618873108, + 44.74813744908079, + 47.891014070791065], + [5.0025829314460639, + 8.3507247014130795, + 11.574195465217647, + 14.760909306207676, + 17.931285939466855, + 21.092894504412739, + 24.249231678519058, + 27.402145837145258, + 30.552708880564553, + 33.70158627151572, + 36.849213419846257, + 39.995887376143356, + 43.141817835750686, + 46.287157097544201, + 49.432018469138281], + [6.2536332084598136, + 9.6987879841487711, + 12.972409052292216, + 16.19044719506921, + 19.38238844973613, + 22.559791857764261, + 25.728213194724094, + 28.890678419054777, + 32.048984005266337, + 35.204266606440635, + 38.357281675961019, + 41.508551443818436, + 44.658448731963676, + 47.807246956681162, + 50.95515126455207], + [7.4649217367571329, + 11.005169149809189, + 14.3317235192331, + 17.58443601710272, + 20.801062338411128, + 23.997004122902644, + 27.179886689853435, + 30.353960608554323, + 33.521797098666792, + 36.685048382072301, + 39.844826969405863, + 43.001910515625288, + 46.15685955107263, + 49.310088614282257, + 52.461911043685864], + [8.6495562436971983, + 12.280868725807848, + 15.660799304540377, + 18.949739756016503, + 22.192841809428241, + 25.409072788867674, + 28.608039283077593, + 31.795195353138159, + 34.973890634255288, + 38.14630522169358, + 41.313923188794905, + 44.477791768537617, + 47.638672065035628, + 50.797131066967842, + 53.953600129601663], + [9.8147970120105779, + 13.532811875789828, + 16.965526446046053, + 20.291285512443867, + 23.56186260680065, + 26.799499736027237, + 30.015665481543419, + 33.216968050039509, + 36.407516858984748, + 39.590015243560459, + 42.766320595957378, + 45.937754257017323, + 49.105283450953203, + 52.269633324547373, + 55.431358715604255], + [10.965152105242974, + 14.765687379508912, + 18.250123150217555, + 21.612750053384621, + 24.911310600813573, + 28.171051927637585, + 31.40518108895689, + 34.621401012564177, + 37.824552065973114, + 41.017847386464902, + 44.203512240871601, + 47.3831408366063, + 50.557907466622796, + 53.728697478957026, + 56.896191727313342], + [12.103641941939539, + 15.982840905145284, + 19.517731005559611, + 22.916962141504605, + 26.243700855690533, + 29.525960140695407, + 32.778568197561124, + 36.010261572392516, + 39.226578757802172, + 42.43122493258747, + 45.626783824134354, + 48.815117837929515, + 51.997606404328863, + 55.175294723956816, + 58.348990221754937], + [13.232403808592215, + 17.186756572616758, + 20.770762917490496, + 24.206152448722253, + 27.561059462697153, + 30.866053571250639, + 34.137476603379774, + 37.385039772270268, + 40.614946085165892, + 43.831373184731238, + 47.037251786726299, + 50.234705848765229, + 53.425316228549359, + 56.610286079882087, + 59.790548623216652], + [14.35301374369987, + 18.379337301642568, + 22.011118775283494, + 25.482116178696707, + 28.865046588695164, + 32.192853922166294, + 35.483296655830277, + 38.747005493021857, + 41.990815194320955, + 45.219355876831731, + 48.435892856078888, + 51.642803925173029, + 54.84186659475857, + 58.034439083840155, + 61.221578745109862], + [15.466672066554263, + 19.562077985759503, + 23.240325531101082, + 26.746322986645901, + 30.157042415639891, + 33.507642948240263, + 36.817212798512775, + 40.097251300178642, + 43.355193847719752, + 46.596103410173672, + 49.823567279972794, + 53.040208868780832, + 56.247996968470062, + 59.448441365714251, + 62.642721301357187], + [16.574317035530872, + 20.73617763753932, + 24.459631728238804, + 27.999993668839644, + 31.438208790267783, + 34.811512070805535, + 38.140243708611251, + 41.436725143893739, + 44.708963264433333, + 47.962435051891027, + 51.201037321915983, + 54.427630745992975, + 57.644369734615238, + 60.852911791989989, + 64.054555435720397], + [17.676697936439624, + 21.9026148697762, + 25.670073356263225, + 29.244155124266438, + 32.709534477396028, + 36.105399554497548, + 39.453272918267025, + 42.766255701958017, + 46.052899215578358, + 49.319076602061401, + 52.568982147952547, + 55.805705507386287, + 59.031580956740466, + 62.248409689597653, + 65.457606670836759], + [18.774423978290318, + 23.06220035979272, + 26.872520985976736, + 30.479680663499762, + 33.971869047372436, + 37.390118854896324, + 40.757072537673599, + 44.086572292170345, + 47.387688809191869, + 50.66667461073936, + 53.928009929563275, + 57.175005343085052, + 60.410169281219877, + 63.635442539153021, + 66.85235358587768]] + +@pytest.mark.slow +def test_bessel_zeros_extra(): + mp.dps = 15 + for v in range(V): + for m in range(1,M+1): + print(v, m, "of", V, M) + # Twice to test cache (if used) + assert besseljzero(v,m).ae(jn_small_zeros[v][m-1]) + assert besseljzero(v,m).ae(jn_small_zeros[v][m-1]) + assert besseljzero(v,m,1).ae(jnp_small_zeros[v][m-1]) + assert besseljzero(v,m,1).ae(jnp_small_zeros[v][m-1]) + assert besselyzero(v,m).ae(yn_small_zeros[v][m-1]) + assert besselyzero(v,m).ae(yn_small_zeros[v][m-1]) + assert besselyzero(v,m,1).ae(ynp_small_zeros[v][m-1]) + assert besselyzero(v,m,1).ae(ynp_small_zeros[v][m-1]) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_gammazeta.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_gammazeta.py new file mode 100644 index 0000000000000000000000000000000000000000..6a18a7964d746561dfd5f81177cd78ccc46d2a5d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_gammazeta.py @@ -0,0 +1,698 @@ +from mpmath import * +from mpmath.libmp import round_up, from_float, mpf_zeta_int + +def test_zeta_int_bug(): + assert mpf_zeta_int(0, 10) == from_float(-0.5) + +def test_bernoulli(): + assert bernfrac(0) == (1,1) + assert bernfrac(1) == (-1,2) + assert bernfrac(2) == (1,6) + assert bernfrac(3) == (0,1) + assert bernfrac(4) == (-1,30) + assert bernfrac(5) == (0,1) + assert bernfrac(6) == (1,42) + assert bernfrac(8) == (-1,30) + assert bernfrac(10) == (5,66) + assert bernfrac(12) == (-691,2730) + assert bernfrac(18) == (43867,798) + p, q = bernfrac(228) + assert p % 10**10 == 164918161 + assert q == 625170 + p, q = bernfrac(1000) + assert p % 10**10 == 7950421099 + assert q == 342999030 + mp.dps = 15 + assert bernoulli(0) == 1 + assert bernoulli(1) == -0.5 + assert bernoulli(2).ae(1./6) + assert bernoulli(3) == 0 + assert bernoulli(4).ae(-1./30) + assert bernoulli(5) == 0 + assert bernoulli(6).ae(1./42) + assert str(bernoulli(10)) == '0.0757575757575758' + assert str(bernoulli(234)) == '7.62772793964344e+267' + assert str(bernoulli(10**5)) == '-5.82229431461335e+376755' + assert str(bernoulli(10**8+2)) == '1.19570355039953e+676752584' + + mp.dps = 50 + assert str(bernoulli(10)) == '0.075757575757575757575757575757575757575757575757576' + assert str(bernoulli(234)) == '7.6277279396434392486994969020496121553385863373331e+267' + assert str(bernoulli(10**5)) == '-5.8222943146133508236497045360612887555320691004308e+376755' + assert str(bernoulli(10**8+2)) == '1.1957035503995297272263047884604346914602088317782e+676752584' + + mp.dps = 1000 + assert bernoulli(10).ae(mpf(5)/66) + + mp.dps = 50000 + assert bernoulli(10).ae(mpf(5)/66) + + mp.dps = 15 + +def test_bernpoly_eulerpoly(): + mp.dps = 15 + assert bernpoly(0,-1).ae(1) + assert bernpoly(0,0).ae(1) + assert bernpoly(0,'1/2').ae(1) + assert bernpoly(0,'3/4').ae(1) + assert bernpoly(0,1).ae(1) + assert bernpoly(0,2).ae(1) + assert bernpoly(1,-1).ae('-3/2') + assert bernpoly(1,0).ae('-1/2') + assert bernpoly(1,'1/2').ae(0) + assert bernpoly(1,'3/4').ae('1/4') + assert bernpoly(1,1).ae('1/2') + assert bernpoly(1,2).ae('3/2') + assert bernpoly(2,-1).ae('13/6') + assert bernpoly(2,0).ae('1/6') + assert bernpoly(2,'1/2').ae('-1/12') + assert bernpoly(2,'3/4').ae('-1/48') + assert bernpoly(2,1).ae('1/6') + assert bernpoly(2,2).ae('13/6') + assert bernpoly(3,-1).ae(-3) + assert bernpoly(3,0).ae(0) + assert bernpoly(3,'1/2').ae(0) + assert bernpoly(3,'3/4').ae('-3/64') + assert bernpoly(3,1).ae(0) + assert bernpoly(3,2).ae(3) + assert bernpoly(4,-1).ae('119/30') + assert bernpoly(4,0).ae('-1/30') + assert bernpoly(4,'1/2').ae('7/240') + assert bernpoly(4,'3/4').ae('7/3840') + assert bernpoly(4,1).ae('-1/30') + assert bernpoly(4,2).ae('119/30') + assert bernpoly(5,-1).ae(-5) + assert bernpoly(5,0).ae(0) + assert bernpoly(5,'1/2').ae(0) + assert bernpoly(5,'3/4').ae('25/1024') + assert bernpoly(5,1).ae(0) + assert bernpoly(5,2).ae(5) + assert bernpoly(10,-1).ae('665/66') + assert bernpoly(10,0).ae('5/66') + assert bernpoly(10,'1/2').ae('-2555/33792') + assert bernpoly(10,'3/4').ae('-2555/34603008') + assert bernpoly(10,1).ae('5/66') + assert bernpoly(10,2).ae('665/66') + assert bernpoly(11,-1).ae(-11) + assert bernpoly(11,0).ae(0) + assert bernpoly(11,'1/2').ae(0) + assert bernpoly(11,'3/4').ae('-555731/4194304') + assert bernpoly(11,1).ae(0) + assert bernpoly(11,2).ae(11) + assert eulerpoly(0,-1).ae(1) + assert eulerpoly(0,0).ae(1) + assert eulerpoly(0,'1/2').ae(1) + assert eulerpoly(0,'3/4').ae(1) + assert eulerpoly(0,1).ae(1) + assert eulerpoly(0,2).ae(1) + assert eulerpoly(1,-1).ae('-3/2') + assert eulerpoly(1,0).ae('-1/2') + assert eulerpoly(1,'1/2').ae(0) + assert eulerpoly(1,'3/4').ae('1/4') + assert eulerpoly(1,1).ae('1/2') + assert eulerpoly(1,2).ae('3/2') + assert eulerpoly(2,-1).ae(2) + assert eulerpoly(2,0).ae(0) + assert eulerpoly(2,'1/2').ae('-1/4') + assert eulerpoly(2,'3/4').ae('-3/16') + assert eulerpoly(2,1).ae(0) + assert eulerpoly(2,2).ae(2) + assert eulerpoly(3,-1).ae('-9/4') + assert eulerpoly(3,0).ae('1/4') + assert eulerpoly(3,'1/2').ae(0) + assert eulerpoly(3,'3/4').ae('-11/64') + assert eulerpoly(3,1).ae('-1/4') + assert eulerpoly(3,2).ae('9/4') + assert eulerpoly(4,-1).ae(2) + assert eulerpoly(4,0).ae(0) + assert eulerpoly(4,'1/2').ae('5/16') + assert eulerpoly(4,'3/4').ae('57/256') + assert eulerpoly(4,1).ae(0) + assert eulerpoly(4,2).ae(2) + assert eulerpoly(5,-1).ae('-3/2') + assert eulerpoly(5,0).ae('-1/2') + assert eulerpoly(5,'1/2').ae(0) + assert eulerpoly(5,'3/4').ae('361/1024') + assert eulerpoly(5,1).ae('1/2') + assert eulerpoly(5,2).ae('3/2') + assert eulerpoly(10,-1).ae(2) + assert eulerpoly(10,0).ae(0) + assert eulerpoly(10,'1/2').ae('-50521/1024') + assert eulerpoly(10,'3/4').ae('-36581523/1048576') + assert eulerpoly(10,1).ae(0) + assert eulerpoly(10,2).ae(2) + assert eulerpoly(11,-1).ae('-699/4') + assert eulerpoly(11,0).ae('691/4') + assert eulerpoly(11,'1/2').ae(0) + assert eulerpoly(11,'3/4').ae('-512343611/4194304') + assert eulerpoly(11,1).ae('-691/4') + assert eulerpoly(11,2).ae('699/4') + # Potential accuracy issues + assert bernpoly(10000,10000).ae('5.8196915936323387117e+39999') + assert bernpoly(200,17.5).ae(3.8048418524583064909e244) + assert eulerpoly(200,17.5).ae(-3.7309911582655785929e275) + +def test_gamma(): + mp.dps = 15 + assert gamma(0.25).ae(3.6256099082219083119) + assert gamma(0.0001).ae(9999.4228832316241908) + assert gamma(300).ae('1.0201917073881354535e612') + assert gamma(-0.5).ae(-3.5449077018110320546) + assert gamma(-7.43).ae(0.00026524416464197007186) + #assert gamma(Rational(1,2)) == gamma(0.5) + #assert gamma(Rational(-7,3)).ae(gamma(mpf(-7)/3)) + assert gamma(1+1j).ae(0.49801566811835604271 - 0.15494982830181068512j) + assert gamma(-1+0.01j).ae(-0.422733904013474115 + 99.985883082635367436j) + assert gamma(20+30j).ae(-1453876687.5534810 + 1163777777.8031573j) + # Should always give exact factorials when they can + # be represented as mpfs under the current working precision + fact = 1 + for i in range(1, 18): + assert gamma(i) == fact + fact *= i + for dps in [170, 600]: + fact = 1 + mp.dps = dps + for i in range(1, 105): + assert gamma(i) == fact + fact *= i + mp.dps = 100 + assert gamma(0.5).ae(sqrt(pi)) + mp.dps = 15 + assert factorial(0) == fac(0) == 1 + assert factorial(3) == 6 + assert isnan(gamma(nan)) + assert gamma(1100).ae('4.8579168073569433667e2866') + assert rgamma(0) == 0 + assert rgamma(-1) == 0 + assert rgamma(2) == 1.0 + assert rgamma(3) == 0.5 + assert loggamma(2+8j).ae(-8.5205176753667636926 + 10.8569497125597429366j) + assert loggamma('1e10000').ae('2.302485092994045684017991e10004') + assert loggamma('1e10000j').ae(mpc('-1.570796326794896619231322e10000','2.302485092994045684017991e10004')) + +def test_fac2(): + mp.dps = 15 + assert [fac2(n) for n in range(10)] == [1,1,2,3,8,15,48,105,384,945] + assert fac2(-5).ae(1./3) + assert fac2(-11).ae(-1./945) + assert fac2(50).ae(5.20469842636666623e32) + assert fac2(0.5+0.75j).ae(0.81546769394688069176-0.34901016085573266889j) + assert fac2(inf) == inf + assert isnan(fac2(-inf)) + +def test_gamma_quotients(): + mp.dps = 15 + h = 1e-8 + ep = 1e-4 + G = gamma + assert gammaprod([-1],[-3,-4]) == 0 + assert gammaprod([-1,0],[-5]) == inf + assert abs(gammaprod([-1],[-2]) - G(-1+h)/G(-2+h)) < 1e-4 + assert abs(gammaprod([-4,-3],[-2,0]) - G(-4+h)*G(-3+h)/G(-2+h)/G(0+h)) < 1e-4 + assert rf(3,0) == 1 + assert rf(2.5,1) == 2.5 + assert rf(-5,2) == 20 + assert rf(j,j).ae(gamma(2*j)/gamma(j)) + assert rf('-255.5815971722918','-0.5119253100282322').ae('-0.1952720278805729485') # issue 421 + assert ff(-2,0) == 1 + assert ff(-2,1) == -2 + assert ff(4,3) == 24 + assert ff(3,4) == 0 + assert binomial(0,0) == 1 + assert binomial(1,0) == 1 + assert binomial(0,-1) == 0 + assert binomial(3,2) == 3 + assert binomial(5,2) == 10 + assert binomial(5,3) == 10 + assert binomial(5,5) == 1 + assert binomial(-1,0) == 1 + assert binomial(-2,-4) == 3 + assert binomial(4.5, 1.5) == 6.5625 + assert binomial(1100,1) == 1100 + assert binomial(1100,2) == 604450 + assert beta(1,1) == 1 + assert beta(0,0) == inf + assert beta(3,0) == inf + assert beta(-1,-1) == inf + assert beta(1.5,1).ae(2/3.) + assert beta(1.5,2.5).ae(pi/16) + assert (10**15*beta(10,100)).ae(2.3455339739604649879) + assert beta(inf,inf) == 0 + assert isnan(beta(-inf,inf)) + assert isnan(beta(-3,inf)) + assert isnan(beta(0,inf)) + assert beta(inf,0.5) == beta(0.5,inf) == 0 + assert beta(inf,-1.5) == inf + assert beta(inf,-0.5) == -inf + assert beta(1+2j,-1-j/2).ae(1.16396542451069943086+0.08511695947832914640j) + assert beta(-0.5,0.5) == 0 + assert beta(-3,3).ae(-1/3.) + assert beta('-255.5815971722918','-0.5119253100282322').ae('18.157330562703710339') # issue 421 + +def test_zeta(): + mp.dps = 15 + assert zeta(2).ae(pi**2 / 6) + assert zeta(2.0).ae(pi**2 / 6) + assert zeta(mpc(2)).ae(pi**2 / 6) + assert zeta(100).ae(1) + assert zeta(0).ae(-0.5) + assert zeta(0.5).ae(-1.46035450880958681) + assert zeta(-1).ae(-mpf(1)/12) + assert zeta(-2) == 0 + assert zeta(-3).ae(mpf(1)/120) + assert zeta(-4) == 0 + assert zeta(-100) == 0 + assert isnan(zeta(nan)) + assert zeta(1e-30).ae(-0.5) + assert zeta(-1e-30).ae(-0.5) + # Zeros in the critical strip + assert zeta(mpc(0.5, 14.1347251417346937904)).ae(0) + assert zeta(mpc(0.5, 21.0220396387715549926)).ae(0) + assert zeta(mpc(0.5, 25.0108575801456887632)).ae(0) + assert zeta(mpc(1e-30,1e-40)).ae(-0.5) + assert zeta(mpc(-1e-30,1e-40)).ae(-0.5) + mp.dps = 50 + im = '236.5242296658162058024755079556629786895294952121891237' + assert zeta(mpc(0.5, im)).ae(0, 1e-46) + mp.dps = 15 + # Complex reflection formula + assert (zeta(-60+3j) / 10**34).ae(8.6270183987866146+15.337398548226238j) + # issue #358 + assert zeta(0,0.5) == 0 + assert zeta(0,0) == 0.5 + assert zeta(0,0.5,1).ae(-0.34657359027997265) + # see issue #390 + assert zeta(-1.5,0.5j).ae(-0.13671400162512768475 + 0.11411333638426559139j) + +def test_altzeta(): + mp.dps = 15 + assert altzeta(-2) == 0 + assert altzeta(-4) == 0 + assert altzeta(-100) == 0 + assert altzeta(0) == 0.5 + assert altzeta(-1) == 0.25 + assert altzeta(-3) == -0.125 + assert altzeta(-5) == 0.25 + assert altzeta(-21) == 1180529130.25 + assert altzeta(1).ae(log(2)) + assert altzeta(2).ae(pi**2/12) + assert altzeta(10).ae(73*pi**10/6842880) + assert altzeta(50) < 1 + assert altzeta(60, rounding='d') < 1 + assert altzeta(60, rounding='u') == 1 + assert altzeta(10000, rounding='d') < 1 + assert altzeta(10000, rounding='u') == 1 + assert altzeta(3+0j) == altzeta(3) + s = 3+4j + assert altzeta(s).ae((1-2**(1-s))*zeta(s)) + s = -3+4j + assert altzeta(s).ae((1-2**(1-s))*zeta(s)) + assert altzeta(-100.5).ae(4.58595480083585913e+108) + assert altzeta(1.3).ae(0.73821404216623045) + assert altzeta(1e-30).ae(0.5) + assert altzeta(-1e-30).ae(0.5) + assert altzeta(mpc(1e-30,1e-40)).ae(0.5) + assert altzeta(mpc(-1e-30,1e-40)).ae(0.5) + +def test_zeta_huge(): + mp.dps = 15 + assert zeta(inf) == 1 + mp.dps = 50 + assert zeta(100).ae('1.0000000000000000000000000000007888609052210118073522') + assert zeta(40*pi).ae('1.0000000000000000000000000000000000000148407238666182') + mp.dps = 10000 + v = zeta(33000) + mp.dps = 15 + assert str(v-1) == '1.02363019598118e-9934' + assert zeta(pi*1000, rounding=round_up) > 1 + assert zeta(3000, rounding=round_up) > 1 + assert zeta(pi*1000) == 1 + assert zeta(3000) == 1 + +def test_zeta_negative(): + mp.dps = 150 + a = -pi*10**40 + mp.dps = 15 + assert str(zeta(a)) == '2.55880492708712e+1233536161668617575553892558646631323374078' + mp.dps = 50 + assert str(zeta(a)) == '2.5588049270871154960875033337384432038436330847333e+1233536161668617575553892558646631323374078' + mp.dps = 15 + +def test_polygamma(): + mp.dps = 15 + psi0 = lambda z: psi(0,z) + psi1 = lambda z: psi(1,z) + assert psi0(3) == psi(0,3) == digamma(3) + #assert psi2(3) == psi(2,3) == tetragamma(3) + #assert psi3(3) == psi(3,3) == pentagamma(3) + assert psi0(pi).ae(0.97721330794200673) + assert psi0(-pi).ae(7.8859523853854902) + assert psi0(-pi+1).ae(7.5676424992016996) + assert psi0(pi+j).ae(1.04224048313859376 + 0.35853686544063749j) + assert psi0(-pi-j).ae(1.3404026194821986 - 2.8824392476809402j) + assert findroot(psi0, 1).ae(1.4616321449683622) + assert psi0(1e-10).ae(-10000000000.57722) + assert psi0(1e-40).ae(-1.000000000000000e+40) + assert psi0(1e-10+1e-10j).ae(-5000000000.577215 + 5000000000.000000j) + assert psi0(1e-40+1e-40j).ae(-5.000000000000000e+39 + 5.000000000000000e+39j) + assert psi0(inf) == inf + assert psi1(inf) == 0 + assert psi(2,inf) == 0 + assert psi1(pi).ae(0.37424376965420049) + assert psi1(-pi).ae(53.030438740085385) + assert psi1(pi+j).ae(0.32935710377142464 - 0.12222163911221135j) + assert psi1(-pi-j).ae(-0.30065008356019703 + 0.01149892486928227j) + assert (10**6*psi(4,1+10*pi*j)).ae(-6.1491803479004446 - 0.3921316371664063j) + assert psi0(1+10*pi*j).ae(3.4473994217222650 + 1.5548808324857071j) + assert isnan(psi0(nan)) + assert isnan(psi0(-inf)) + assert psi0(-100.5).ae(4.615124601338064) + assert psi0(3+0j).ae(psi0(3)) + assert psi0(-100+3j).ae(4.6106071768714086321+3.1117510556817394626j) + assert isnan(psi(2,mpc(0,inf))) + assert isnan(psi(2,mpc(0,nan))) + assert isnan(psi(2,mpc(0,-inf))) + assert isnan(psi(2,mpc(1,inf))) + assert isnan(psi(2,mpc(1,nan))) + assert isnan(psi(2,mpc(1,-inf))) + assert isnan(psi(2,mpc(inf,inf))) + assert isnan(psi(2,mpc(nan,nan))) + assert isnan(psi(2,mpc(-inf,-inf))) + mp.dps = 30 + # issue #534 + assert digamma(-0.75+1j).ae(mpc('0.46317279488182026118963809283042317', '2.4821070143037957102007677817351115')) + mp.dps = 15 + +def test_polygamma_high_prec(): + mp.dps = 100 + assert str(psi(0,pi)) == "0.9772133079420067332920694864061823436408346099943256380095232865318105924777141317302075654362928734" + assert str(psi(10,pi)) == "-12.98876181434889529310283769414222588307175962213707170773803550518307617769657562747174101900659238" + +def test_polygamma_identities(): + mp.dps = 15 + psi0 = lambda z: psi(0,z) + psi1 = lambda z: psi(1,z) + psi2 = lambda z: psi(2,z) + assert psi0(0.5).ae(-euler-2*log(2)) + assert psi0(1).ae(-euler) + assert psi1(0.5).ae(0.5*pi**2) + assert psi1(1).ae(pi**2/6) + assert psi1(0.25).ae(pi**2 + 8*catalan) + assert psi2(1).ae(-2*apery) + mp.dps = 20 + u = -182*apery+4*sqrt(3)*pi**3 + mp.dps = 15 + assert psi(2,5/6.).ae(u) + assert psi(3,0.5).ae(pi**4) + +def test_foxtrot_identity(): + # A test of the complex digamma function. + # See http://mathworld.wolfram.com/FoxTrotSeries.html and + # http://mathworld.wolfram.com/DigammaFunction.html + psi0 = lambda z: psi(0,z) + mp.dps = 50 + a = (-1)**fraction(1,3) + b = (-1)**fraction(2,3) + x = -psi0(0.5*a) - psi0(-0.5*b) + psi0(0.5*(1+a)) + psi0(0.5*(1-b)) + y = 2*pi*sech(0.5*sqrt(3)*pi) + assert x.ae(y) + mp.dps = 15 + +def test_polygamma_high_order(): + mp.dps = 100 + assert str(psi(50, pi)) == "-1344100348958402765749252447726432491812.641985273160531055707095989227897753035823152397679626136483" + assert str(psi(50, pi + 14*e)) == "-0.00000000000000000189793739550804321623512073101895801993019919886375952881053090844591920308111549337295143780341396" + assert str(psi(50, pi + 14*e*j)) == ("(-0.0000000000000000522516941152169248975225472155683565752375889510631513244785" + "9377385233700094871256507814151956624433 - 0.00000000000000001813157041407010184" + "702414110218205348527862196327980417757665282244728963891298080199341480881811613j)") + mp.dps = 15 + assert str(psi(50, pi)) == "-1.34410034895841e+39" + assert str(psi(50, pi + 14*e)) == "-1.89793739550804e-18" + assert str(psi(50, pi + 14*e*j)) == "(-5.2251694115217e-17 - 1.81315704140701e-17j)" + +def test_harmonic(): + mp.dps = 15 + assert harmonic(0) == 0 + assert harmonic(1) == 1 + assert harmonic(2) == 1.5 + assert harmonic(3).ae(1. + 1./2 + 1./3) + assert harmonic(10**10).ae(23.603066594891989701) + assert harmonic(10**1000).ae(2303.162308658947) + assert harmonic(0.5).ae(2-2*log(2)) + assert harmonic(inf) == inf + assert harmonic(2+0j) == 1.5+0j + assert harmonic(1+2j).ae(1.4918071802755104+0.92080728264223022j) + +def test_gamma_huge_1(): + mp.dps = 500 + x = mpf(10**10) / 7 + mp.dps = 15 + assert str(gamma(x)) == "6.26075321389519e+12458010678" + mp.dps = 50 + assert str(gamma(x)) == "6.2607532138951929201303779291707455874010420783933e+12458010678" + mp.dps = 15 + +def test_gamma_huge_2(): + mp.dps = 500 + x = mpf(10**100) / 19 + mp.dps = 15 + assert str(gamma(x)) == (\ + "1.82341134776679e+5172997469323364168990133558175077136829182824042201886051511" + "9656908623426021308685461258226190190661") + mp.dps = 50 + assert str(gamma(x)) == (\ + "1.82341134776678875374414910350027596939980412984e+5172997469323364168990133558" + "1750771368291828240422018860515119656908623426021308685461258226190190661") + +def test_gamma_huge_3(): + mp.dps = 500 + x = 10**80 // 3 + 10**70*j / 7 + mp.dps = 15 + y = gamma(x) + assert str(y.real) == (\ + "-6.82925203918106e+2636286142112569524501781477865238132302397236429627932441916" + "056964386399485392600") + assert str(y.imag) == (\ + "8.54647143678418e+26362861421125695245017814778652381323023972364296279324419160" + "56964386399485392600") + mp.dps = 50 + y = gamma(x) + assert str(y.real) == (\ + "-6.8292520391810548460682736226799637356016538421817e+26362861421125695245017814" + "77865238132302397236429627932441916056964386399485392600") + assert str(y.imag) == (\ + "8.5464714367841748507479306948130687511711420234015e+263628614211256952450178147" + "7865238132302397236429627932441916056964386399485392600") + +def test_gamma_huge_4(): + x = 3200+11500j + mp.dps = 15 + assert str(gamma(x)) == \ + "(8.95783268539713e+5164 - 1.94678798329735e+5164j)" + mp.dps = 50 + assert str(gamma(x)) == (\ + "(8.9578326853971339570292952697675570822206567327092e+5164" + " - 1.9467879832973509568895402139429643650329524144794e+51" + "64j)") + mp.dps = 15 + +def test_gamma_huge_5(): + mp.dps = 500 + x = 10**60 * j / 3 + mp.dps = 15 + y = gamma(x) + assert str(y.real) == "-3.27753899634941e-227396058973640224580963937571892628368354580620654233316839" + assert str(y.imag) == "-7.1519888950416e-227396058973640224580963937571892628368354580620654233316841" + mp.dps = 50 + y = gamma(x) + assert str(y.real) == (\ + "-3.2775389963494132168950056995974690946983219123935e-22739605897364022458096393" + "7571892628368354580620654233316839") + assert str(y.imag) == (\ + "-7.1519888950415979749736749222530209713136588885897e-22739605897364022458096393" + "7571892628368354580620654233316841") + mp.dps = 15 + +def test_gamma_huge_7(): + mp.dps = 100 + a = 3 + j/mpf(10)**1000 + mp.dps = 15 + y = gamma(a) + assert str(y.real) == "2.0" + # wrong + #assert str(y.imag) == "2.16735365342606e-1000" + assert str(y.imag) == "1.84556867019693e-1000" + mp.dps = 50 + y = gamma(a) + assert str(y.real) == "2.0" + #assert str(y.imag) == "2.1673536534260596065418805612488708028522563689298e-1000" + assert str(y.imag) == "1.8455686701969342787869758198351951379156813281202e-1000" + +def test_stieltjes(): + mp.dps = 15 + assert stieltjes(0).ae(+euler) + mp.dps = 25 + assert stieltjes(1).ae('-0.07281584548367672486058637587') + assert stieltjes(2).ae('-0.009690363192872318484530386035') + assert stieltjes(3).ae('0.002053834420303345866160046543') + assert stieltjes(4).ae('0.002325370065467300057468170178') + mp.dps = 15 + assert stieltjes(1).ae(-0.07281584548367672486058637587) + assert stieltjes(2).ae(-0.009690363192872318484530386035) + assert stieltjes(3).ae(0.002053834420303345866160046543) + assert stieltjes(4).ae(0.0023253700654673000574681701775) + +def test_barnesg(): + mp.dps = 15 + assert barnesg(0) == barnesg(-1) == 0 + assert [superfac(i) for i in range(8)] == [1, 1, 2, 12, 288, 34560, 24883200, 125411328000] + assert str(superfac(1000)) == '3.24570818422368e+1177245' + assert isnan(barnesg(nan)) + assert isnan(superfac(nan)) + assert isnan(hyperfac(nan)) + assert barnesg(inf) == inf + assert superfac(inf) == inf + assert hyperfac(inf) == inf + assert isnan(superfac(-inf)) + assert barnesg(0.7).ae(0.8068722730141471) + assert barnesg(2+3j).ae(-0.17810213864082169+0.04504542715447838j) + assert [hyperfac(n) for n in range(7)] == [1, 1, 4, 108, 27648, 86400000, 4031078400000] + assert [hyperfac(n) for n in range(0,-7,-1)] == [1,1,-1,-4,108,27648,-86400000] + a = barnesg(-3+0j) + assert a == 0 and isinstance(a, mpc) + a = hyperfac(-3+0j) + assert a == -4 and isinstance(a, mpc) + +def test_polylog(): + mp.dps = 15 + zs = [mpmathify(z) for z in [0, 0.5, 0.99, 4, -0.5, -4, 1j, 3+4j]] + for z in zs: assert polylog(1, z).ae(-log(1-z)) + for z in zs: assert polylog(0, z).ae(z/(1-z)) + for z in zs: assert polylog(-1, z).ae(z/(1-z)**2) + for z in zs: assert polylog(-2, z).ae(z*(1+z)/(1-z)**3) + for z in zs: assert polylog(-3, z).ae(z*(1+4*z+z**2)/(1-z)**4) + assert polylog(3, 7).ae(5.3192579921456754382-5.9479244480803301023j) + assert polylog(3, -7).ae(-4.5693548977219423182) + assert polylog(2, 0.9).ae(1.2997147230049587252) + assert polylog(2, -0.9).ae(-0.75216317921726162037) + assert polylog(2, 0.9j).ae(-0.17177943786580149299+0.83598828572550503226j) + assert polylog(2, 1.1).ae(1.9619991013055685931-0.2994257606855892575j) + assert polylog(2, -1.1).ae(-0.89083809026228260587) + assert polylog(2, 1.1*sqrt(j)).ae(0.58841571107611387722+1.09962542118827026011j) + assert polylog(-2, 0.9).ae(1710) + assert polylog(-2, -0.9).ae(-90/6859.) + assert polylog(3, 0.9).ae(1.0496589501864398696) + assert polylog(-3, 0.9).ae(48690) + assert polylog(-3, -4).ae(-0.0064) + assert polylog(0.5+j/3, 0.5+j/2).ae(0.31739144796565650535 + 0.99255390416556261437j) + assert polylog(3+4j,1).ae(zeta(3+4j)) + assert polylog(3+4j,-1).ae(-altzeta(3+4j)) + # issue 390 + assert polylog(1.5, -48.910886523731889).ae(-6.272992229311817) + assert polylog(1.5, 200).ae(-8.349608319033686529 - 8.159694826434266042j) + assert polylog(-2+0j, -2).ae(mpf(1)/13.5) + assert polylog(-2+0j, 1.25).ae(-180) + +def test_bell_polyexp(): + mp.dps = 15 + # TODO: more tests for polyexp + assert (polyexp(0,1e-10)*10**10).ae(1.00000000005) + assert (polyexp(1,1e-10)*10**10).ae(1.0000000001) + assert polyexp(5,3j).ae(-607.7044517476176454+519.962786482001476087j) + assert polyexp(-1,3.5).ae(12.09537536175543444) + # bell(0,x) = 1 + assert bell(0,0) == 1 + assert bell(0,1) == 1 + assert bell(0,2) == 1 + assert bell(0,inf) == 1 + assert bell(0,-inf) == 1 + assert isnan(bell(0,nan)) + # bell(1,x) = x + assert bell(1,4) == 4 + assert bell(1,0) == 0 + assert bell(1,inf) == inf + assert bell(1,-inf) == -inf + assert isnan(bell(1,nan)) + # bell(2,x) = x*(1+x) + assert bell(2,-1) == 0 + assert bell(2,0) == 0 + # large orders / arguments + assert bell(10) == 115975 + assert bell(10,1) == 115975 + assert bell(10, -8) == 11054008 + assert bell(5,-50) == -253087550 + assert bell(50,-50).ae('3.4746902914629720259e74') + mp.dps = 80 + assert bell(50,-50) == 347469029146297202586097646631767227177164818163463279814268368579055777450 + assert bell(40,50) == 5575520134721105844739265207408344706846955281965031698187656176321717550 + assert bell(74) == 5006908024247925379707076470957722220463116781409659160159536981161298714301202 + mp.dps = 15 + assert bell(10,20j) == 7504528595600+15649605360020j + # continuity of the generalization + assert bell(0.5,0).ae(sinc(pi*0.5)) + +def test_primezeta(): + mp.dps = 15 + assert primezeta(0.9).ae(1.8388316154446882243 + 3.1415926535897932385j) + assert primezeta(4).ae(0.076993139764246844943) + assert primezeta(1) == inf + assert primezeta(inf) == 0 + assert isnan(primezeta(nan)) + +def test_rs_zeta(): + mp.dps = 15 + assert zeta(0.5+100000j).ae(1.0730320148577531321 + 5.7808485443635039843j) + assert zeta(0.75+100000j).ae(1.837852337251873704 + 1.9988492668661145358j) + assert zeta(0.5+1000000j, derivative=3).ae(1647.7744105852674733 - 1423.1270943036622097j) + assert zeta(1+1000000j, derivative=3).ae(3.4085866124523582894 - 18.179184721525947301j) + assert zeta(1+1000000j, derivative=1).ae(-0.10423479366985452134 - 0.74728992803359056244j) + assert zeta(0.5-1000000j, derivative=1).ae(11.636804066002521459 + 17.127254072212996004j) + # Additional sanity tests using fp arithmetic. + # Some more high-precision tests are found in the docstrings + def ae(x, y, tol=1e-6): + return abs(x-y) < tol*abs(y) + assert ae(fp.zeta(0.5-100000j), 1.0730320148577531321 - 5.7808485443635039843j) + assert ae(fp.zeta(0.75-100000j), 1.837852337251873704 - 1.9988492668661145358j) + assert ae(fp.zeta(0.5+1e6j), 0.076089069738227100006 + 2.8051021010192989554j) + assert ae(fp.zeta(0.5+1e6j, derivative=1), 11.636804066002521459 - 17.127254072212996004j) + assert ae(fp.zeta(1+1e6j), 0.94738726251047891048 + 0.59421999312091832833j) + assert ae(fp.zeta(1+1e6j, derivative=1), -0.10423479366985452134 - 0.74728992803359056244j) + assert ae(fp.zeta(0.5+100000j, derivative=1), 10.766962036817482375 - 30.92705282105996714j) + assert ae(fp.zeta(0.5+100000j, derivative=2), -119.40515625740538429 + 217.14780631141830251j) + assert ae(fp.zeta(0.5+100000j, derivative=3), 1129.7550282628460881 - 1685.4736895169690346j) + assert ae(fp.zeta(0.5+100000j, derivative=4), -10407.160819314958615 + 13777.786698628045085j) + assert ae(fp.zeta(0.75+100000j, derivative=1), -0.41742276699594321475 - 6.4453816275049955949j) + assert ae(fp.zeta(0.75+100000j, derivative=2), -9.214314279161977266 + 35.07290795337967899j) + assert ae(fp.zeta(0.75+100000j, derivative=3), 110.61331857820103469 - 236.87847130518129926j) + assert ae(fp.zeta(0.75+100000j, derivative=4), -1054.334275898559401 + 1769.9177890161596383j) + +def test_siegelz(): + mp.dps = 15 + assert siegelz(100000).ae(5.87959246868176504171) + assert siegelz(100000, derivative=2).ae(-54.1172711010126452832) + assert siegelz(100000, derivative=3).ae(-278.930831343966552538) + assert siegelz(100000+j,derivative=1).ae(678.214511857070283307-379.742160779916375413j) + + + +def test_zeta_near_1(): + # Test for a former bug in mpf_zeta and mpc_zeta + mp.dps = 15 + s1 = fadd(1, '1e-10', exact=True) + s2 = fadd(1, '-1e-10', exact=True) + s3 = fadd(1, '1e-10j', exact=True) + assert zeta(s1).ae(1.000000000057721566490881444e10) + assert zeta(s2).ae(-9.99999999942278433510574872e9) + z = zeta(s3) + assert z.real.ae(0.57721566490153286060) + assert z.imag.ae(-9.9999999999999999999927184e9) + mp.dps = 30 + s1 = fadd(1, '1e-50', exact=True) + s2 = fadd(1, '-1e-50', exact=True) + s3 = fadd(1, '1e-50j', exact=True) + assert zeta(s1).ae('1e50') + assert zeta(s2).ae('-1e50') + z = zeta(s3) + assert z.real.ae('0.57721566490153286060651209008240243104215933593992') + assert z.imag.ae('-1e50') diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_hp.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_hp.py new file mode 100644 index 0000000000000000000000000000000000000000..9eba0af798f64ac3f8d464e2d3bf231567a48c9b --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_hp.py @@ -0,0 +1,291 @@ +""" +Check that the output from irrational functions is accurate for +high-precision input, from 5 to 200 digits. The reference values were +verified with Mathematica. +""" + +import time +from mpmath import * + +precs = [5, 15, 28, 35, 57, 80, 100, 150, 200] + +# sqrt(3) + pi/2 +a = \ +"3.302847134363773912758768033145623809041389953497933538543279275605"\ +"841220051904536395163599428307109666700184672047856353516867399774243594"\ +"67433521615861420725323528325327484262075464241255915238845599752675" + +# e + 1/euler**2 +b = \ +"5.719681166601007617111261398629939965860873957353320734275716220045750"\ +"31474116300529519620938123730851145473473708966080207482581266469342214"\ +"824842256999042984813905047895479210702109260221361437411947323431" + +# sqrt(a) +sqrt_a = \ +"1.817373691447021556327498239690365674922395036495564333152483422755"\ +"144321726165582817927383239308173567921345318453306994746434073691275094"\ +"484777905906961689902608644112196725896908619756404253109722911487" + +# sqrt(a+b*i).real +sqrt_abi_real = \ +"2.225720098415113027729407777066107959851146508557282707197601407276"\ +"89160998185797504198062911768240808839104987021515555650875977724230130"\ +"3584116233925658621288393930286871862273400475179312570274423840384" + +# sqrt(a+b*i).imag +sqrt_abi_imag = \ +"1.2849057639084690902371581529110949983261182430040898147672052833653668"\ +"0629534491275114877090834296831373498336559849050755848611854282001250"\ +"1924311019152914021365263161630765255610885489295778894976075186" + +# log(a) +log_a = \ +"1.194784864491089550288313512105715261520511949410072046160598707069"\ +"4336653155025770546309137440687056366757650909754708302115204338077595203"\ +"83005773986664564927027147084436553262269459110211221152925732612" + +# log(a+b*i).real +log_abi_real = \ +"1.8877985921697018111624077550443297276844736840853590212962006811663"\ +"04949387789489704203167470111267581371396245317618589339274243008242708"\ +"014251531496104028712866224020066439049377679709216784954509456421" + +# log(a+b*i).imag +log_abi_imag = \ +"1.0471204952840802663567714297078763189256357109769672185219334169734948"\ +"4265809854092437285294686651806426649541504240470168212723133326542181"\ +"8300136462287639956713914482701017346851009323172531601894918640" + +# exp(a) +exp_a = \ +"27.18994224087168661137253262213293847994194869430518354305430976149"\ +"382792035050358791398632888885200049857986258414049540376323785711941636"\ +"100358982497583832083513086941635049329804685212200507288797531143" + +# exp(a+b*i).real +exp_abi_real = \ +"22.98606617170543596386921087657586890620262522816912505151109385026"\ +"40160179326569526152851983847133513990281518417211964710397233157168852"\ +"4963130831190142571659948419307628119985383887599493378056639916701" + +# exp(a+b*i).imag +exp_abi_imag = \ +"-14.523557450291489727214750571590272774669907424478129280902375851196283"\ +"3377162379031724734050088565710975758824441845278120105728824497308303"\ +"6065619788140201636218705414429933685889542661364184694108251449" + +# a**b +pow_a_b = \ +"928.7025342285568142947391505837660251004990092821305668257284426997"\ +"361966028275685583421197860603126498884545336686124793155581311527995550"\ +"580229264427202446131740932666832138634013168125809402143796691154" + +# (a**(a+b*i)).real +pow_a_abi_real = \ +"44.09156071394489511956058111704382592976814280267142206420038656267"\ +"67707916510652790502399193109819563864568986234654864462095231138500505"\ +"8197456514795059492120303477512711977915544927440682508821426093455" + +# (a**(a+b*i)).imag +pow_a_abi_imag = \ +"27.069371511573224750478105146737852141664955461266218367212527612279886"\ +"9322304536553254659049205414427707675802193810711302947536332040474573"\ +"8166261217563960235014674118610092944307893857862518964990092301" + +# ((a+b*i)**(a+b*i)).real +pow_abi_abi_real = \ +"-0.15171310677859590091001057734676423076527145052787388589334350524"\ +"8084195882019497779202452975350579073716811284169068082670778986235179"\ +"0813026562962084477640470612184016755250592698408112493759742219150452"\ + +# ((a+b*i)**(a+b*i)).imag +pow_abi_abi_imag = \ +"1.2697592504953448936553147870155987153192995316950583150964099070426"\ +"4736837932577176947632535475040521749162383347758827307504526525647759"\ +"97547638617201824468382194146854367480471892602963428122896045019902" + +# sin(a) +sin_a = \ +"-0.16055653857469062740274792907968048154164433772938156243509084009"\ +"38437090841460493108570147191289893388608611542655654723437248152535114"\ +"528368009465836614227575701220612124204622383149391870684288862269631" + +# sin(1000*a) +sin_1000a = \ +"-0.85897040577443833776358106803777589664322997794126153477060795801"\ +"09151695416961724733492511852267067419573754315098042850381158563024337"\ +"216458577140500488715469780315833217177634490142748614625281171216863" + +# sin(a+b*i) +sin_abi_real = \ +"-24.4696999681556977743346798696005278716053366404081910969773939630"\ +"7149215135459794473448465734589287491880563183624997435193637389884206"\ +"02151395451271809790360963144464736839412254746645151672423256977064" + +sin_abi_imag = \ +"-150.42505378241784671801405965872972765595073690984080160750785565810981"\ +"8314482499135443827055399655645954830931316357243750839088113122816583"\ +"7169201254329464271121058839499197583056427233866320456505060735" + +# cos +cos_a = \ +"-0.98702664499035378399332439243967038895709261414476495730788864004"\ +"05406821549361039745258003422386169330787395654908532996287293003581554"\ +"257037193284199198069707141161341820684198547572456183525659969145501" + +cos_1000a = \ +"-0.51202523570982001856195696460663971099692261342827540426136215533"\ +"52686662667660613179619804463250686852463876088694806607652218586060613"\ +"951310588158830695735537073667299449753951774916401887657320950496820" + +# tan +tan_a = \ +"0.162666873675188117341401059858835168007137819495998960250142156848"\ +"639654718809412181543343168174807985559916643549174530459883826451064966"\ +"7996119428949951351938178809444268785629011625179962457123195557310" + +tan_abi_real = \ +"6.822696615947538488826586186310162599974827139564433912601918442911"\ +"1026830824380070400102213741875804368044342309515353631134074491271890"\ +"467615882710035471686578162073677173148647065131872116479947620E-6" + +tan_abi_imag = \ +"0.9999795833048243692245661011298447587046967777739649018690797625964167"\ +"1446419978852235960862841608081413169601038230073129482874832053357571"\ +"62702259309150715669026865777947502665936317953101462202542168429" + + +def test_hp(): + for dps in precs: + mp.dps = dps + 8 + aa = mpf(a) + bb = mpf(b) + a1000 = 1000*mpf(a) + abi = mpc(aa, bb) + mp.dps = dps + assert (sqrt(3) + pi/2).ae(aa) + assert (e + 1/euler**2).ae(bb) + + assert sqrt(aa).ae(mpf(sqrt_a)) + assert sqrt(abi).ae(mpc(sqrt_abi_real, sqrt_abi_imag)) + + assert log(aa).ae(mpf(log_a)) + assert log(abi).ae(mpc(log_abi_real, log_abi_imag)) + + assert exp(aa).ae(mpf(exp_a)) + assert exp(abi).ae(mpc(exp_abi_real, exp_abi_imag)) + + assert (aa**bb).ae(mpf(pow_a_b)) + assert (aa**abi).ae(mpc(pow_a_abi_real, pow_a_abi_imag)) + assert (abi**abi).ae(mpc(pow_abi_abi_real, pow_abi_abi_imag)) + + assert sin(a).ae(mpf(sin_a)) + assert sin(a1000).ae(mpf(sin_1000a)) + assert sin(abi).ae(mpc(sin_abi_real, sin_abi_imag)) + + assert cos(a).ae(mpf(cos_a)) + assert cos(a1000).ae(mpf(cos_1000a)) + + assert tan(a).ae(mpf(tan_a)) + assert tan(abi).ae(mpc(tan_abi_real, tan_abi_imag)) + + # check that complex cancellation is avoided so that both + # real and imaginary parts have high relative accuracy. + # abs_eps should be 0, but has to be set to 1e-205 to pass the + # 200-digit case, probably due to slight inaccuracy in the + # precomputed input + assert (tan(abi).real).ae(mpf(tan_abi_real), abs_eps=1e-205) + assert (tan(abi).imag).ae(mpf(tan_abi_imag), abs_eps=1e-205) + mp.dps = 460 + assert str(log(3))[-20:] == '02166121184001409826' + mp.dps = 15 + +# Since str(a) can differ in the last digit from rounded a, and I want +# to compare the last digits of big numbers with the results in Mathematica, +# I made this hack to get the last 20 digits of rounded a + +def last_digits(a): + r = repr(a) + s = str(a) + #dps = mp.dps + #mp.dps += 3 + m = 10 + r = r.replace(s[:-m],'') + r = r.replace("mpf('",'').replace("')",'') + num0 = 0 + for c in r: + if c == '0': + num0 += 1 + else: + break + b = float(int(r))/10**(len(r) - m) + if b >= 10**m - 0.5: # pragma: no cover + raise NotImplementedError + n = int(round(b)) + sn = str(n) + s = s[:-m] + '0'*num0 + sn + return s[-20:] + +# values checked with Mathematica +def test_log_hp(): + mp.dps = 2000 + a = mpf(10)**15000/3 + r = log(a) + res = last_digits(r) + # Mathematica N[Log[10^15000/3], 2000] + # ...7443804441768333470331 + assert res == '43804441768333470331' + + # see issue 145 + r = log(mpf(3)/2) + # Mathematica N[Log[3/2], 2000] + # ...69653749808140753263288 + res = last_digits(r) + assert res == '53749808140753263288' + + mp.dps = 10000 + r = log(2) + res = last_digits(r) + # Mathematica N[Log[2], 10000] + # ...695615913401856601359655561 + assert res == '13401856601359655561' + r = log(mpf(10)**10/3) + res = last_digits(r) + # Mathematica N[Log[10^10/3], 10000] + # ...587087654020631943060007154 + assert res == '54020631943060007154', res + r = log(mpf(10)**100/3) + res = last_digits(r) + # Mathematica N[Log[10^100/3], 10000] + # ,,,59246336539088351652334666 + assert res == '36539088351652334666', res + mp.dps += 10 + a = 1 - mpf(1)/10**10 + mp.dps -= 10 + r = log(a) + res = last_digits(r) + # ...3310334360482956137216724048322957404 + # 372167240483229574038733026370 + # Mathematica N[Log[1 - 10^-10]*10^10, 10000] + # ...60482956137216724048322957404 + assert res == '37216724048322957404', res + mp.dps = 10000 + mp.dps += 100 + a = 1 + mpf(1)/10**100 + mp.dps -= 100 + + r = log(a) + res = last_digits(+r) + # Mathematica N[Log[1 + 10^-100]*10^10, 10030] + # ...3994733877377412241546890854692521568292338268273 10^-91 + assert res == '39947338773774122415', res + + mp.dps = 15 + +def test_exp_hp(): + mp.dps = 4000 + r = exp(mpf(1)/10) + # IntegerPart[N[Exp[1/10] * 10^4000, 4000]] + # ...92167105162069688129 + assert int(r * 10**mp.dps) % 10**20 == 92167105162069688129 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_identify.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_identify.py new file mode 100644 index 0000000000000000000000000000000000000000..f75ab0bc4f04ecb614011e7f4599989465cab785 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_identify.py @@ -0,0 +1,19 @@ +from mpmath import * + +def test_pslq(): + mp.dps = 15 + assert pslq([3*pi+4*e/7, pi, e, log(2)]) == [7, -21, -4, 0] + assert pslq([4.9999999999999991, 1]) == [1, -5] + assert pslq([2,1]) == [1, -2] + +def test_identify(): + mp.dps = 20 + assert identify(zeta(4), ['log(2)', 'pi**4']) == '((1/90)*pi**4)' + mp.dps = 15 + assert identify(exp(5)) == 'exp(5)' + assert identify(exp(4)) == 'exp(4)' + assert identify(log(5)) == 'log(5)' + assert identify(exp(3*pi), ['pi']) == 'exp((3*pi))' + assert identify(3, full=True) == ['3', '3', '1/(1/3)', 'sqrt(9)', + '1/sqrt((1/9))', '(sqrt(12)/2)**2', '1/(sqrt(12)/6)**2'] + assert identify(pi+1, {'a':+pi}) == '(1 + 1*a)' diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_interval.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_interval.py new file mode 100644 index 0000000000000000000000000000000000000000..251fd8b7ddb00074e8ae27cce4a01d8f4f8fe151 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_interval.py @@ -0,0 +1,453 @@ +from mpmath import * + +def test_interval_identity(): + iv.dps = 15 + assert mpi(2) == mpi(2, 2) + assert mpi(2) != mpi(-2, 2) + assert not (mpi(2) != mpi(2, 2)) + assert mpi(-1, 1) == mpi(-1, 1) + assert str(mpi('0.1')) == "[0.099999999999999991673, 0.10000000000000000555]" + assert repr(mpi('0.1')) == "mpi('0.099999999999999992', '0.10000000000000001')" + u = mpi(-1, 3) + assert -1 in u + assert 2 in u + assert 3 in u + assert -1.1 not in u + assert 3.1 not in u + assert mpi(-1, 3) in u + assert mpi(0, 1) in u + assert mpi(-1.1, 2) not in u + assert mpi(2.5, 3.1) not in u + w = mpi(-inf, inf) + assert mpi(-5, 5) in w + assert mpi(2, inf) in w + assert mpi(0, 2) in mpi(0, 10) + assert not (3 in mpi(-inf, 0)) + +def test_interval_hash(): + assert hash(mpi(3)) == hash(3) + assert hash(mpi(3.25)) == hash(3.25) + assert hash(mpi(3,4)) == hash(mpi(3,4)) + assert hash(iv.mpc(3)) == hash(3) + assert hash(iv.mpc(3,4)) == hash(3+4j) + assert hash(iv.mpc((1,3),(2,4))) == hash(iv.mpc((1,3),(2,4))) + +def test_interval_arithmetic(): + iv.dps = 15 + assert mpi(2) + mpi(3,4) == mpi(5,6) + assert mpi(1, 2)**2 == mpi(1, 4) + assert mpi(1) + mpi(0, 1e-50) == mpi(1, mpf('1.0000000000000002')) + x = 1 / (1 / mpi(3)) + assert x.a < 3 < x.b + x = mpi(2) ** mpi(0.5) + iv.dps += 5 + sq = iv.sqrt(2) + iv.dps -= 5 + assert x.a < sq < x.b + assert mpi(1) / mpi(1, inf) + assert mpi(2, 3) / inf == mpi(0, 0) + assert mpi(0) / inf == 0 + assert mpi(0) / 0 == mpi(-inf, inf) + assert mpi(inf) / 0 == mpi(-inf, inf) + assert mpi(0) * inf == mpi(-inf, inf) + assert 1 / mpi(2, inf) == mpi(0, 0.5) + assert str((mpi(50, 50) * mpi(-10, -10)) / 3) == \ + '[-166.66666666666668561, -166.66666666666665719]' + assert mpi(0, 4) ** 3 == mpi(0, 64) + assert mpi(2,4).mid == 3 + iv.dps = 30 + a = mpi(iv.pi) + iv.dps = 15 + b = +a + assert b.a < a.a + assert b.b > a.b + a = mpi(iv.pi) + assert a == +a + assert abs(mpi(-1,2)) == mpi(0,2) + assert abs(mpi(0.5,2)) == mpi(0.5,2) + assert abs(mpi(-3,2)) == mpi(0,3) + assert abs(mpi(-3,-0.5)) == mpi(0.5,3) + assert mpi(0) * mpi(2,3) == mpi(0) + assert mpi(2,3) * mpi(0) == mpi(0) + assert mpi(1,3).delta == 2 + assert mpi(1,2) - mpi(3,4) == mpi(-3,-1) + assert mpi(-inf,0) - mpi(0,inf) == mpi(-inf,0) + assert mpi(-inf,0) - mpi(-inf,inf) == mpi(-inf,inf) + assert mpi(0,inf) - mpi(-inf,1) == mpi(-1,inf) + +def test_interval_mul(): + assert mpi(-1, 0) * inf == mpi(-inf, 0) + assert mpi(-1, 0) * -inf == mpi(0, inf) + assert mpi(0, 1) * inf == mpi(0, inf) + assert mpi(0, 1) * mpi(0, inf) == mpi(0, inf) + assert mpi(-1, 1) * inf == mpi(-inf, inf) + assert mpi(-1, 1) * mpi(0, inf) == mpi(-inf, inf) + assert mpi(-1, 1) * mpi(-inf, inf) == mpi(-inf, inf) + assert mpi(-inf, 0) * mpi(0, 1) == mpi(-inf, 0) + assert mpi(-inf, 0) * mpi(0, 0) * mpi(-inf, 0) + assert mpi(-inf, 0) * mpi(-inf, inf) == mpi(-inf, inf) + assert mpi(-5,0)*mpi(-32,28) == mpi(-140,160) + assert mpi(2,3) * mpi(-1,2) == mpi(-3,6) + # Should be undefined? + assert mpi(inf, inf) * 0 == mpi(-inf, inf) + assert mpi(-inf, -inf) * 0 == mpi(-inf, inf) + assert mpi(0) * mpi(-inf,2) == mpi(-inf,inf) + assert mpi(0) * mpi(-2,inf) == mpi(-inf,inf) + assert mpi(-2,inf) * mpi(0) == mpi(-inf,inf) + assert mpi(-inf,2) * mpi(0) == mpi(-inf,inf) + +def test_interval_pow(): + assert mpi(3)**2 == mpi(9, 9) + assert mpi(-3)**2 == mpi(9, 9) + assert mpi(-3, 1)**2 == mpi(0, 9) + assert mpi(-3, -1)**2 == mpi(1, 9) + assert mpi(-3, -1)**3 == mpi(-27, -1) + assert mpi(-3, 1)**3 == mpi(-27, 1) + assert mpi(-2, 3)**2 == mpi(0, 9) + assert mpi(-3, 2)**2 == mpi(0, 9) + assert mpi(4) ** -1 == mpi(0.25, 0.25) + assert mpi(-4) ** -1 == mpi(-0.25, -0.25) + assert mpi(4) ** -2 == mpi(0.0625, 0.0625) + assert mpi(-4) ** -2 == mpi(0.0625, 0.0625) + assert mpi(0, 1) ** inf == mpi(0, 1) + assert mpi(0, 1) ** -inf == mpi(1, inf) + assert mpi(0, inf) ** inf == mpi(0, inf) + assert mpi(0, inf) ** -inf == mpi(0, inf) + assert mpi(1, inf) ** inf == mpi(1, inf) + assert mpi(1, inf) ** -inf == mpi(0, 1) + assert mpi(2, 3) ** 1 == mpi(2, 3) + assert mpi(2, 3) ** 0 == 1 + assert mpi(1,3) ** mpi(2) == mpi(1,9) + +def test_interval_sqrt(): + assert mpi(4) ** 0.5 == mpi(2) + +def test_interval_div(): + assert mpi(0.5, 1) / mpi(-1, 0) == mpi(-inf, -0.5) + assert mpi(0, 1) / mpi(0, 1) == mpi(0, inf) + assert mpi(inf, inf) / mpi(inf, inf) == mpi(0, inf) + assert mpi(inf, inf) / mpi(2, inf) == mpi(0, inf) + assert mpi(inf, inf) / mpi(2, 2) == mpi(inf, inf) + assert mpi(0, inf) / mpi(2, inf) == mpi(0, inf) + assert mpi(0, inf) / mpi(2, 2) == mpi(0, inf) + assert mpi(2, inf) / mpi(2, 2) == mpi(1, inf) + assert mpi(2, inf) / mpi(2, inf) == mpi(0, inf) + assert mpi(-4, 8) / mpi(1, inf) == mpi(-4, 8) + assert mpi(-4, 8) / mpi(0.5, inf) == mpi(-8, 16) + assert mpi(-inf, 8) / mpi(0.5, inf) == mpi(-inf, 16) + assert mpi(-inf, inf) / mpi(0.5, inf) == mpi(-inf, inf) + assert mpi(8, inf) / mpi(0.5, inf) == mpi(0, inf) + assert mpi(-8, inf) / mpi(0.5, inf) == mpi(-16, inf) + assert mpi(-4, 8) / mpi(inf, inf) == mpi(0, 0) + assert mpi(0, 8) / mpi(inf, inf) == mpi(0, 0) + assert mpi(0, 0) / mpi(inf, inf) == mpi(0, 0) + assert mpi(-inf, 0) / mpi(inf, inf) == mpi(-inf, 0) + assert mpi(-inf, 8) / mpi(inf, inf) == mpi(-inf, 0) + assert mpi(-inf, inf) / mpi(inf, inf) == mpi(-inf, inf) + assert mpi(-8, inf) / mpi(inf, inf) == mpi(0, inf) + assert mpi(0, inf) / mpi(inf, inf) == mpi(0, inf) + assert mpi(8, inf) / mpi(inf, inf) == mpi(0, inf) + assert mpi(inf, inf) / mpi(inf, inf) == mpi(0, inf) + assert mpi(-1, 2) / mpi(0, 1) == mpi(-inf, +inf) + assert mpi(0, 1) / mpi(0, 1) == mpi(0.0, +inf) + assert mpi(-1, 0) / mpi(0, 1) == mpi(-inf, 0.0) + assert mpi(-0.5, -0.25) / mpi(0, 1) == mpi(-inf, -0.25) + assert mpi(0.5, 1) / mpi(0, 1) == mpi(0.5, +inf) + assert mpi(0.5, 4) / mpi(0, 1) == mpi(0.5, +inf) + assert mpi(-1, -0.5) / mpi(0, 1) == mpi(-inf, -0.5) + assert mpi(-4, -0.5) / mpi(0, 1) == mpi(-inf, -0.5) + assert mpi(-1, 2) / mpi(-2, 0.5) == mpi(-inf, +inf) + assert mpi(0, 1) / mpi(-2, 0.5) == mpi(-inf, +inf) + assert mpi(-1, 0) / mpi(-2, 0.5) == mpi(-inf, +inf) + assert mpi(-0.5, -0.25) / mpi(-2, 0.5) == mpi(-inf, +inf) + assert mpi(0.5, 1) / mpi(-2, 0.5) == mpi(-inf, +inf) + assert mpi(0.5, 4) / mpi(-2, 0.5) == mpi(-inf, +inf) + assert mpi(-1, -0.5) / mpi(-2, 0.5) == mpi(-inf, +inf) + assert mpi(-4, -0.5) / mpi(-2, 0.5) == mpi(-inf, +inf) + assert mpi(-1, 2) / mpi(-1, 0) == mpi(-inf, +inf) + assert mpi(0, 1) / mpi(-1, 0) == mpi(-inf, 0.0) + assert mpi(-1, 0) / mpi(-1, 0) == mpi(0.0, +inf) + assert mpi(-0.5, -0.25) / mpi(-1, 0) == mpi(0.25, +inf) + assert mpi(0.5, 1) / mpi(-1, 0) == mpi(-inf, -0.5) + assert mpi(0.5, 4) / mpi(-1, 0) == mpi(-inf, -0.5) + assert mpi(-1, -0.5) / mpi(-1, 0) == mpi(0.5, +inf) + assert mpi(-4, -0.5) / mpi(-1, 0) == mpi(0.5, +inf) + assert mpi(-1, 2) / mpi(0.5, 1) == mpi(-2.0, 4.0) + assert mpi(0, 1) / mpi(0.5, 1) == mpi(0.0, 2.0) + assert mpi(-1, 0) / mpi(0.5, 1) == mpi(-2.0, 0.0) + assert mpi(-0.5, -0.25) / mpi(0.5, 1) == mpi(-1.0, -0.25) + assert mpi(0.5, 1) / mpi(0.5, 1) == mpi(0.5, 2.0) + assert mpi(0.5, 4) / mpi(0.5, 1) == mpi(0.5, 8.0) + assert mpi(-1, -0.5) / mpi(0.5, 1) == mpi(-2.0, -0.5) + assert mpi(-4, -0.5) / mpi(0.5, 1) == mpi(-8.0, -0.5) + assert mpi(-1, 2) / mpi(-2, -0.5) == mpi(-4.0, 2.0) + assert mpi(0, 1) / mpi(-2, -0.5) == mpi(-2.0, 0.0) + assert mpi(-1, 0) / mpi(-2, -0.5) == mpi(0.0, 2.0) + assert mpi(-0.5, -0.25) / mpi(-2, -0.5) == mpi(0.125, 1.0) + assert mpi(0.5, 1) / mpi(-2, -0.5) == mpi(-2.0, -0.25) + assert mpi(0.5, 4) / mpi(-2, -0.5) == mpi(-8.0, -0.25) + assert mpi(-1, -0.5) / mpi(-2, -0.5) == mpi(0.25, 2.0) + assert mpi(-4, -0.5) / mpi(-2, -0.5) == mpi(0.25, 8.0) + # Should be undefined? + assert mpi(0, 0) / mpi(0, 0) == mpi(-inf, inf) + assert mpi(0, 0) / mpi(0, 1) == mpi(-inf, inf) + +def test_interval_cos_sin(): + iv.dps = 15 + cos = iv.cos + sin = iv.sin + tan = iv.tan + pi = iv.pi + # Around 0 + assert cos(mpi(0)) == 1 + assert sin(mpi(0)) == 0 + assert cos(mpi(0,1)) == mpi(0.54030230586813965399, 1.0) + assert sin(mpi(0,1)) == mpi(0, 0.8414709848078966159) + assert cos(mpi(1,2)) == mpi(-0.4161468365471424069, 0.54030230586813976501) + assert sin(mpi(1,2)) == mpi(0.84147098480789650488, 1.0) + assert sin(mpi(1,2.5)) == mpi(0.59847214410395643824, 1.0) + assert cos(mpi(-1, 1)) == mpi(0.54030230586813965399, 1.0) + assert cos(mpi(-1, 0.5)) == mpi(0.54030230586813965399, 1.0) + assert cos(mpi(-1, 1.5)) == mpi(0.070737201667702906405, 1.0) + assert sin(mpi(-1,1)) == mpi(-0.8414709848078966159, 0.8414709848078966159) + assert sin(mpi(-1,0.5)) == mpi(-0.8414709848078966159, 0.47942553860420300538) + assert mpi(-0.8414709848078966159, 1.00000000000000002e-100) in sin(mpi(-1,1e-100)) + assert mpi(-2.00000000000000004e-100, 1.00000000000000002e-100) in sin(mpi(-2e-100,1e-100)) + # Same interval + assert cos(mpi(2, 2.5)) + assert cos(mpi(3.5, 4)) == mpi(-0.93645668729079634129, -0.65364362086361182946) + assert cos(mpi(5, 5.5)) == mpi(0.28366218546322624627, 0.70866977429126010168) + assert mpi(0.59847214410395654927, 0.90929742682568170942) in sin(mpi(2, 2.5)) + assert sin(mpi(3.5, 4)) == mpi(-0.75680249530792831347, -0.35078322768961983646) + assert sin(mpi(5, 5.5)) == mpi(-0.95892427466313856499, -0.70554032557039181306) + # Higher roots + iv.dps = 55 + w = 4*10**50 + mpi(0.5) + for p in [15, 40, 80]: + iv.dps = p + assert 0 in sin(4*mpi(pi)) + assert 0 in sin(4*10**50*mpi(pi)) + assert 0 in cos((4+0.5)*mpi(pi)) + assert 0 in cos(w*mpi(pi)) + assert 1 in cos(4*mpi(pi)) + assert 1 in cos(4*10**50*mpi(pi)) + iv.dps = 15 + assert cos(mpi(2,inf)) == mpi(-1,1) + assert sin(mpi(2,inf)) == mpi(-1,1) + assert cos(mpi(-inf,2)) == mpi(-1,1) + assert sin(mpi(-inf,2)) == mpi(-1,1) + u = tan(mpi(0.5,1)) + assert mpf(u.a).ae(mp.tan(0.5)) + assert mpf(u.b).ae(mp.tan(1)) + v = iv.cot(mpi(0.5,1)) + assert mpf(v.a).ae(mp.cot(1)) + assert mpf(v.b).ae(mp.cot(0.5)) + # Sanity check of evaluation at n*pi and (n+1/2)*pi + for n in range(-5,7,2): + x = iv.cos(n*iv.pi) + assert -1 in x + assert x >= -1 + assert x != -1 + x = iv.sin((n+0.5)*iv.pi) + assert -1 in x + assert x >= -1 + assert x != -1 + for n in range(-6,8,2): + x = iv.cos(n*iv.pi) + assert 1 in x + assert x <= 1 + if n: + assert x != 1 + x = iv.sin((n+0.5)*iv.pi) + assert 1 in x + assert x <= 1 + assert x != 1 + for n in range(-6,7): + x = iv.cos((n+0.5)*iv.pi) + assert x.a < 0 < x.b + x = iv.sin(n*iv.pi) + if n: + assert x.a < 0 < x.b + +def test_interval_complex(): + # TODO: many more tests + iv.dps = 15 + mp.dps = 15 + assert iv.mpc(2,3) == 2+3j + assert iv.mpc(2,3) != 2+4j + assert iv.mpc(2,3) != 1+3j + assert 1+3j in iv.mpc([1,2],[3,4]) + assert 2+5j not in iv.mpc([1,2],[3,4]) + assert iv.mpc(1,2) + 1j == 1+3j + assert iv.mpc([1,2],[2,3]) + 2+3j == iv.mpc([3,4],[5,6]) + assert iv.mpc([2,4],[4,8]) / 2 == iv.mpc([1,2],[2,4]) + assert iv.mpc([1,2],[2,4]) * 2j == iv.mpc([-8,-4],[2,4]) + assert iv.mpc([2,4],[4,8]) / 2j == iv.mpc([2,4],[-2,-1]) + assert iv.exp(2+3j).ae(mp.exp(2+3j)) + assert iv.log(2+3j).ae(mp.log(2+3j)) + assert (iv.mpc(2,3) ** iv.mpc(0.5,2)).ae(mp.mpc(2,3) ** mp.mpc(0.5,2)) + assert 1j in (iv.mpf(-1) ** 0.5) + assert 1j in (iv.mpc(-1) ** 0.5) + assert abs(iv.mpc(0)) == 0 + assert abs(iv.mpc(inf)) == inf + assert abs(iv.mpc(3,4)) == 5 + assert abs(iv.mpc(4)) == 4 + assert abs(iv.mpc(0,4)) == 4 + assert abs(iv.mpc(0,[2,3])) == iv.mpf([2,3]) + assert abs(iv.mpc(0,[-3,2])) == iv.mpf([0,3]) + assert abs(iv.mpc([3,5],[4,12])) == iv.mpf([5,13]) + assert abs(iv.mpc([3,5],[-4,12])) == iv.mpf([3,13]) + assert iv.mpc(2,3) ** 0 == 1 + assert iv.mpc(2,3) ** 1 == (2+3j) + assert iv.mpc(2,3) ** 2 == (2+3j)**2 + assert iv.mpc(2,3) ** 3 == (2+3j)**3 + assert iv.mpc(2,3) ** 4 == (2+3j)**4 + assert iv.mpc(2,3) ** 5 == (2+3j)**5 + assert iv.mpc(2,2) ** (-1) == (2+2j) ** (-1) + assert iv.mpc(2,2) ** (-2) == (2+2j) ** (-2) + assert iv.cos(2).ae(mp.cos(2)) + assert iv.sin(2).ae(mp.sin(2)) + assert iv.cos(2+3j).ae(mp.cos(2+3j)) + assert iv.sin(2+3j).ae(mp.sin(2+3j)) + +def test_interval_complex_arg(): + mp.dps = 15 + iv.dps = 15 + assert iv.arg(3) == 0 + assert iv.arg(0) == 0 + assert iv.arg([0,3]) == 0 + assert iv.arg(-3).ae(pi) + assert iv.arg(2+3j).ae(iv.arg(2+3j)) + z = iv.mpc([-2,-1],[3,4]) + t = iv.arg(z) + assert t.a.ae(mp.arg(-1+4j)) + assert t.b.ae(mp.arg(-2+3j)) + z = iv.mpc([-2,1],[3,4]) + t = iv.arg(z) + assert t.a.ae(mp.arg(1+3j)) + assert t.b.ae(mp.arg(-2+3j)) + z = iv.mpc([1,2],[3,4]) + t = iv.arg(z) + assert t.a.ae(mp.arg(2+3j)) + assert t.b.ae(mp.arg(1+4j)) + z = iv.mpc([1,2],[-2,3]) + t = iv.arg(z) + assert t.a.ae(mp.arg(1-2j)) + assert t.b.ae(mp.arg(1+3j)) + z = iv.mpc([1,2],[-4,-3]) + t = iv.arg(z) + assert t.a.ae(mp.arg(1-4j)) + assert t.b.ae(mp.arg(2-3j)) + z = iv.mpc([-1,2],[-4,-3]) + t = iv.arg(z) + assert t.a.ae(mp.arg(-1-3j)) + assert t.b.ae(mp.arg(2-3j)) + z = iv.mpc([-2,-1],[-4,-3]) + t = iv.arg(z) + assert t.a.ae(mp.arg(-2-3j)) + assert t.b.ae(mp.arg(-1-4j)) + z = iv.mpc([-2,-1],[-3,3]) + t = iv.arg(z) + assert t.a.ae(-mp.pi) + assert t.b.ae(mp.pi) + z = iv.mpc([-2,2],[-3,3]) + t = iv.arg(z) + assert t.a.ae(-mp.pi) + assert t.b.ae(mp.pi) + +def test_interval_ae(): + iv.dps = 15 + x = iv.mpf([1,2]) + assert x.ae(1) is None + assert x.ae(1.5) is None + assert x.ae(2) is None + assert x.ae(2.01) is False + assert x.ae(0.99) is False + x = iv.mpf(3.5) + assert x.ae(3.5) is True + assert x.ae(3.5+1e-15) is True + assert x.ae(3.5-1e-15) is True + assert x.ae(3.501) is False + assert x.ae(3.499) is False + assert x.ae(iv.mpf([3.5,3.501])) is None + assert x.ae(iv.mpf([3.5,4.5+1e-15])) is None + +def test_interval_nstr(): + iv.dps = n = 30 + x = mpi(1, 2) + # FIXME: error_dps should not be necessary + assert iv.nstr(x, n, mode='plusminus', error_dps=6) == '1.5 +- 0.5' + assert iv.nstr(x, n, mode='plusminus', use_spaces=False, error_dps=6) == '1.5+-0.5' + assert iv.nstr(x, n, mode='percent') == '1.5 (33.33%)' + assert iv.nstr(x, n, mode='brackets', use_spaces=False) == '[1.0,2.0]' + assert iv.nstr(x, n, mode='brackets' , brackets=('<', '>')) == '<1.0, 2.0>' + x = mpi('5.2582327113062393041', '5.2582327113062749951') + assert iv.nstr(x, n, mode='diff') == '5.2582327113062[393041, 749951]' + assert iv.nstr(iv.cos(mpi(1)), n, mode='diff', use_spaces=False) == '0.54030230586813971740093660744[2955,3053]' + assert iv.nstr(mpi('1e123', '1e129'), n, mode='diff') == '[1.0e+123, 1.0e+129]' + exp = iv.exp + assert iv.nstr(iv.exp(mpi('5000.1')), n, mode='diff') == '3.2797365856787867069110487[0926, 1191]e+2171' + iv.dps = 15 + +def test_mpi_from_str(): + iv.dps = 15 + assert iv.convert('1.5 +- 0.5') == mpi(mpf('1.0'), mpf('2.0')) + assert mpi(1, 2) in iv.convert('1.5 (33.33333333333333333333333333333%)') + assert iv.convert('[1, 2]') == mpi(1, 2) + assert iv.convert('1[2, 3]') == mpi(12, 13) + assert iv.convert('1.[23,46]e-8') == mpi('1.23e-8', '1.46e-8') + assert iv.convert('12[3.4,5.9]e4') == mpi('123.4e+4', '125.9e4') + +def test_interval_gamma(): + mp.dps = 15 + iv.dps = 15 + # TODO: need many more tests + assert iv.rgamma(0) == 0 + assert iv.fac(0) == 1 + assert iv.fac(1) == 1 + assert iv.fac(2) == 2 + assert iv.fac(3) == 6 + assert iv.gamma(0) == [-inf,inf] + assert iv.gamma(1) == 1 + assert iv.gamma(2) == 1 + assert iv.gamma(3) == 2 + assert -3.5449077018110320546 in iv.gamma(-0.5) + assert iv.loggamma(1) == 0 + assert iv.loggamma(2) == 0 + assert 0.69314718055994530942 in iv.loggamma(3) + # Test tight log-gamma endpoints based on monotonicity + xs = [iv.mpc([2,3],[1,4]), + iv.mpc([2,3],[-4,-1]), + iv.mpc([2,3],[-1,4]), + iv.mpc([2,3],[-4,1]), + iv.mpc([2,3],[-4,4]), + iv.mpc([-3,-2],[2,4]), + iv.mpc([-3,-2],[-4,-2])] + for x in xs: + ys = [mp.loggamma(mp.mpc(x.a,x.c)), + mp.loggamma(mp.mpc(x.b,x.c)), + mp.loggamma(mp.mpc(x.a,x.d)), + mp.loggamma(mp.mpc(x.b,x.d))] + if 0 in x.imag: + ys += [mp.loggamma(x.a), mp.loggamma(x.b)] + min_real = min([y.real for y in ys]) + max_real = max([y.real for y in ys]) + min_imag = min([y.imag for y in ys]) + max_imag = max([y.imag for y in ys]) + z = iv.loggamma(x) + assert z.a.ae(min_real) + assert z.b.ae(max_real) + assert z.c.ae(min_imag) + assert z.d.ae(max_imag) + +def test_interval_conversions(): + mp.dps = 15 + iv.dps = 15 + for a, b in ((-0.0, 0), (0.0, 0.5), (1.0, 1), \ + ('-inf', 20.5), ('-inf', float(sqrt(2)))): + r = mpi(a, b) + assert int(r.b) == int(b) + assert float(r.a) == float(a) + assert float(r.b) == float(b) + assert complex(r.a) == complex(a) + assert complex(r.b) == complex(b) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_levin.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_levin.py new file mode 100644 index 0000000000000000000000000000000000000000..b14855df4de1a45da27080dcd239267842a4ac7a --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_levin.py @@ -0,0 +1,153 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +from mpmath import mp +from mpmath import libmp + +xrange = libmp.backend.xrange + +# Attention: +# These tests run with 15-20 decimal digits precision. For higher precision the +# working precision must be raised. + +def test_levin_0(): + mp.dps = 17 + eps = mp.mpf(mp.eps) + with mp.extraprec(2 * mp.prec): + L = mp.levin(method = "levin", variant = "u") + S, s, n = [], 0, 1 + while 1: + s += mp.one / (n * n) + n += 1 + S.append(s) + v, e = L.update_psum(S) + if e < eps: + break + if n > 1000: raise RuntimeError("iteration limit exceeded") + eps = mp.exp(0.9 * mp.log(eps)) + err = abs(v - mp.pi ** 2 / 6) + assert err < eps + w = mp.nsum(lambda n: 1/(n * n), [1, mp.inf], method = "levin", levin_variant = "u") + err = abs(v - w) + assert err < eps + +def test_levin_1(): + mp.dps = 17 + eps = mp.mpf(mp.eps) + with mp.extraprec(2 * mp.prec): + L = mp.levin(method = "levin", variant = "v") + A, n = [], 1 + while 1: + s = mp.mpf(n) ** (2 + 3j) + n += 1 + A.append(s) + v, e = L.update(A) + if e < eps: + break + if n > 1000: raise RuntimeError("iteration limit exceeded") + eps = mp.exp(0.9 * mp.log(eps)) + err = abs(v - mp.zeta(-2-3j)) + assert err < eps + w = mp.nsum(lambda n: n ** (2 + 3j), [1, mp.inf], method = "levin", levin_variant = "v") + err = abs(v - w) + assert err < eps + +def test_levin_2(): + # [2] A. Sidi - "Pratical Extrapolation Methods" p.373 + mp.dps = 17 + z=mp.mpf(10) + eps = mp.mpf(mp.eps) + with mp.extraprec(2 * mp.prec): + L = mp.levin(method = "sidi", variant = "t") + n = 0 + while 1: + s = (-1)**n * mp.fac(n) * z ** (-n) + v, e = L.step(s) + n += 1 + if e < eps: + break + if n > 1000: raise RuntimeError("iteration limit exceeded") + eps = mp.exp(0.9 * mp.log(eps)) + exact = mp.quad(lambda x: mp.exp(-x)/(1+x/z),[0,mp.inf]) + # there is also a symbolic expression for the integral: + # exact = z * mp.exp(z) * mp.expint(1,z) + err = abs(v - exact) + assert err < eps + w = mp.nsum(lambda n: (-1) ** n * mp.fac(n) * z ** (-n), [0, mp.inf], method = "sidi", levin_variant = "t") + assert err < eps + +def test_levin_3(): + mp.dps = 17 + z=mp.mpf(2) + eps = mp.mpf(mp.eps) + with mp.extraprec(7*mp.prec): # we need copious amount of precision to sum this highly divergent series + L = mp.levin(method = "levin", variant = "t") + n, s = 0, 0 + while 1: + s += (-z)**n * mp.fac(4 * n) / (mp.fac(n) * mp.fac(2 * n) * (4 ** n)) + n += 1 + v, e = L.step_psum(s) + if e < eps: + break + if n > 1000: raise RuntimeError("iteration limit exceeded") + eps = mp.exp(0.8 * mp.log(eps)) + exact = mp.quad(lambda x: mp.exp( -x * x / 2 - z * x ** 4), [0,mp.inf]) * 2 / mp.sqrt(2 * mp.pi) + # there is also a symbolic expression for the integral: + # exact = mp.exp(mp.one / (32 * z)) * mp.besselk(mp.one / 4, mp.one / (32 * z)) / (4 * mp.sqrt(z * mp.pi)) + err = abs(v - exact) + assert err < eps + w = mp.nsum(lambda n: (-z)**n * mp.fac(4 * n) / (mp.fac(n) * mp.fac(2 * n) * (4 ** n)), [0, mp.inf], method = "levin", levin_variant = "t", workprec = 8*mp.prec, steps = [2] + [1 for x in xrange(1000)]) + err = abs(v - w) + assert err < eps + +def test_levin_nsum(): + mp.dps = 17 + + with mp.extraprec(mp.prec): + z = mp.mpf(10) ** (-10) + a = mp.nsum(lambda n: n**(-(1+z)), [1, mp.inf], method = "l") - 1 / z + assert abs(a - mp.euler) < 1e-10 + + eps = mp.exp(0.8 * mp.log(mp.eps)) + + a = mp.nsum(lambda n: (-1)**(n-1) / n, [1, mp.inf], method = "sidi") + assert abs(a - mp.log(2)) < eps + + z = 2 + 1j + f = lambda n: mp.rf(2 / mp.mpf(3), n) * mp.rf(4 / mp.mpf(3), n) * z**n / (mp.rf(1 / mp.mpf(3), n) * mp.fac(n)) + v = mp.nsum(f, [0, mp.inf], method = "levin", steps = [10 for x in xrange(1000)]) + exact = mp.hyp2f1(2 / mp.mpf(3), 4 / mp.mpf(3), 1 / mp.mpf(3), z) + assert abs(exact - v) < eps + +def test_cohen_alt_0(): + mp.dps = 17 + AC = mp.cohen_alt() + S, s, n = [], 0, 1 + while 1: + s += -((-1) ** n) * mp.one / (n * n) + n += 1 + S.append(s) + v, e = AC.update_psum(S) + if e < mp.eps: + break + if n > 1000: raise RuntimeError("iteration limit exceeded") + eps = mp.exp(0.9 * mp.log(mp.eps)) + err = abs(v - mp.pi ** 2 / 12) + assert err < eps + +def test_cohen_alt_1(): + mp.dps = 17 + A = [] + AC = mp.cohen_alt() + n = 1 + while 1: + A.append( mp.loggamma(1 + mp.one / (2 * n - 1))) + A.append(-mp.loggamma(1 + mp.one / (2 * n))) + n += 1 + v, e = AC.update(A) + if e < mp.eps: + break + if n > 1000: raise RuntimeError("iteration limit exceeded") + v = mp.exp(v) + err = abs(v - 1.06215090557106) + assert err < 1e-12 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_linalg.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_linalg.py new file mode 100644 index 0000000000000000000000000000000000000000..14256a79f8953d3e4ef8b296258560d48204f547 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_linalg.py @@ -0,0 +1,332 @@ +# TODO: don't use round + +from __future__ import division + +import pytest +from mpmath import * +xrange = libmp.backend.xrange + +# XXX: these shouldn't be visible(?) +LU_decomp = mp.LU_decomp +L_solve = mp.L_solve +U_solve = mp.U_solve +householder = mp.householder +improve_solution = mp.improve_solution + +A1 = matrix([[3, 1, 6], + [2, 1, 3], + [1, 1, 1]]) +b1 = [2, 7, 4] + +A2 = matrix([[ 2, -1, -1, 2], + [ 6, -2, 3, -1], + [-4, 2, 3, -2], + [ 2, 0, 4, -3]]) +b2 = [3, -3, -2, -1] + +A3 = matrix([[ 1, 0, -1, -1, 0], + [ 0, 1, 1, 0, -1], + [ 4, -5, 2, 0, 0], + [ 0, 0, -2, 9,-12], + [ 0, 5, 0, 0, 12]]) +b3 = [0, 0, 0, 0, 50] + +A4 = matrix([[10.235, -4.56, 0., -0.035, 5.67], + [-2.463, 1.27, 3.97, -8.63, 1.08], + [-6.58, 0.86, -0.257, 9.32, -43.6 ], + [ 9.83, 7.39, -17.25, 0.036, 24.86], + [-9.31, 34.9, 78.56, 1.07, 65.8 ]]) +b4 = [8.95, 20.54, 7.42, 5.60, 58.43] + +A5 = matrix([[ 1, 2, -4], + [-2, -3, 5], + [ 3, 5, -8]]) + +A6 = matrix([[ 1.377360, 2.481400, 5.359190], + [ 2.679280, -1.229560, 25.560210], + [-1.225280+1.e6, 9.910180, -35.049900-1.e6]]) +b6 = [23.500000, -15.760000, 2.340000] + +A7 = matrix([[1, -0.5], + [2, 1], + [-2, 6]]) +b7 = [3, 2, -4] + +A8 = matrix([[1, 2, 3], + [-1, 0, 1], + [-1, -2, -1], + [1, 0, -1]]) +b8 = [1, 2, 3, 4] + +A9 = matrix([[ 4, 2, -2], + [ 2, 5, -4], + [-2, -4, 5.5]]) +b9 = [10, 16, -15.5] + +A10 = matrix([[1.0 + 1.0j, 2.0, 2.0], + [4.0, 5.0, 6.0], + [7.0, 8.0, 9.0]]) +b10 = [1.0, 1.0 + 1.0j, 1.0] + + +def test_LU_decomp(): + A = A3.copy() + b = b3 + A, p = LU_decomp(A) + y = L_solve(A, b, p) + x = U_solve(A, y) + assert p == [2, 1, 2, 3] + assert [round(i, 14) for i in x] == [3.78953107960742, 2.9989094874591098, + -0.081788440567070006, 3.8713195201744801, 2.9171210468920399] + A = A4.copy() + b = b4 + A, p = LU_decomp(A) + y = L_solve(A, b, p) + x = U_solve(A, y) + assert p == [0, 3, 4, 3] + assert [round(i, 14) for i in x] == [2.6383625899619201, 2.6643834462368399, + 0.79208015947958998, -2.5088376454101899, -1.0567657691375001] + A = randmatrix(3) + bak = A.copy() + LU_decomp(A, overwrite=1) + assert A != bak + +def test_inverse(): + for A in [A1, A2, A5]: + inv = inverse(A) + assert mnorm(A*inv - eye(A.rows), 1) < 1.e-14 + +def test_householder(): + mp.dps = 15 + A, b = A8, b8 + H, p, x, r = householder(extend(A, b)) + assert H == matrix( + [[mpf('3.0'), mpf('-2.0'), mpf('-1.0'), 0], + [-1.0,mpf('3.333333333333333'),mpf('-2.9999999999999991'),mpf('2.0')], + [-1.0, mpf('-0.66666666666666674'),mpf('2.8142135623730948'), + mpf('-2.8284271247461898')], + [1.0, mpf('-1.3333333333333333'),mpf('-0.20000000000000018'), + mpf('4.2426406871192857')]]) + assert p == [-2, -2, mpf('-1.4142135623730949')] + assert round(norm(r, 2), 10) == 4.2426406870999998 + + y = [102.102, 58.344, 36.463, 24.310, 17.017, 12.376, 9.282, 7.140, 5.610, + 4.488, 3.6465, 3.003] + + def coeff(n): + # similiar to Hilbert matrix + A = [] + for i in range(1, 13): + A.append([1. / (i + j - 1) for j in range(1, n + 1)]) + return matrix(A) + + residuals = [] + refres = [] + for n in range(2, 7): + A = coeff(n) + H, p, x, r = householder(extend(A, y)) + x = matrix(x) + y = matrix(y) + residuals.append(norm(r, 2)) + refres.append(norm(residual(A, x, y), 2)) + assert [round(res, 10) for res in residuals] == [15.1733888877, + 0.82378073210000002, 0.302645887, 0.0260109244, + 0.00058653999999999998] + assert norm(matrix(residuals) - matrix(refres), inf) < 1.e-13 + + def hilbert_cmplx(n): + # Complexified Hilbert matrix + A = hilbert(2*n,n) + v = randmatrix(2*n, 2, min=-1, max=1) + v = v.apply(lambda x: exp(1J*pi()*x)) + A = diag(v[:,0])*A*diag(v[:n,1]) + return A + + residuals_cmplx = [] + refres_cmplx = [] + for n in range(2, 10): + A = hilbert_cmplx(n) + H, p, x, r = householder(A.copy()) + residuals_cmplx.append(norm(r, 2)) + refres_cmplx.append(norm(residual(A[:,:n-1], x, A[:,n-1]), 2)) + assert norm(matrix(residuals_cmplx) - matrix(refres_cmplx), inf) < 1.e-13 + +def test_factorization(): + A = randmatrix(5) + P, L, U = lu(A) + assert mnorm(P*A - L*U, 1) < 1.e-15 + +def test_solve(): + assert norm(residual(A6, lu_solve(A6, b6), b6), inf) < 1.e-10 + assert norm(residual(A7, lu_solve(A7, b7), b7), inf) < 1.5 + assert norm(residual(A8, lu_solve(A8, b8), b8), inf) <= 3 + 1.e-10 + assert norm(residual(A6, qr_solve(A6, b6)[0], b6), inf) < 1.e-10 + assert norm(residual(A7, qr_solve(A7, b7)[0], b7), inf) < 1.5 + assert norm(residual(A8, qr_solve(A8, b8)[0], b8), 2) <= 4.3 + assert norm(residual(A10, lu_solve(A10, b10), b10), 2) < 1.e-10 + assert norm(residual(A10, qr_solve(A10, b10)[0], b10), 2) < 1.e-10 + +def test_solve_overdet_complex(): + A = matrix([[1, 2j], [3, 4j], [5, 6]]) + b = matrix([1 + j, 2, -j]) + assert norm(residual(A, lu_solve(A, b), b)) < 1.0208 + +def test_singular(): + mp.dps = 15 + A = [[5.6, 1.2], [7./15, .1]] + B = repr(zeros(2)) + b = [1, 2] + for i in ['lu_solve(%s, %s)' % (A, b), 'lu_solve(%s, %s)' % (B, b), + 'qr_solve(%s, %s)' % (A, b), 'qr_solve(%s, %s)' % (B, b)]: + pytest.raises((ZeroDivisionError, ValueError), lambda: eval(i)) + +def test_cholesky(): + assert fp.cholesky(fp.matrix(A9)) == fp.matrix([[2, 0, 0], [1, 2, 0], [-1, -3/2, 3/2]]) + x = fp.cholesky_solve(A9, b9) + assert fp.norm(fp.residual(A9, x, b9), fp.inf) == 0 + +def test_det(): + assert det(A1) == 1 + assert round(det(A2), 14) == 8 + assert round(det(A3)) == 1834 + assert round(det(A4)) == 4443376 + assert det(A5) == 1 + assert round(det(A6)) == 78356463 + assert det(zeros(3)) == 0 + +def test_cond(): + mp.dps = 15 + A = matrix([[1.2969, 0.8648], [0.2161, 0.1441]]) + assert cond(A, lambda x: mnorm(x,1)) == mpf('327065209.73817754') + assert cond(A, lambda x: mnorm(x,inf)) == mpf('327065209.73817754') + assert cond(A, lambda x: mnorm(x,'F')) == mpf('249729266.80008656') + +@extradps(50) +def test_precision(): + A = randmatrix(10, 10) + assert mnorm(inverse(inverse(A)) - A, 1) < 1.e-45 + +def test_interval_matrix(): + mp.dps = 15 + iv.dps = 15 + a = iv.matrix([['0.1','0.3','1.0'],['7.1','5.5','4.8'],['3.2','4.4','5.6']]) + b = iv.matrix(['4','0.6','0.5']) + c = iv.lu_solve(a, b) + assert c[0].delta < 1e-13 + assert c[1].delta < 1e-13 + assert c[2].delta < 1e-13 + assert 5.25823271130625686059275 in c[0] + assert -13.155049396267837541163 in c[1] + assert 7.42069154774972557628979 in c[2] + +def test_LU_cache(): + A = randmatrix(3) + LU = LU_decomp(A) + assert A._LU == LU_decomp(A) + A[0,0] = -1000 + assert A._LU is None + +def test_improve_solution(): + A = randmatrix(5, min=1e-20, max=1e20) + b = randmatrix(5, 1, min=-1000, max=1000) + x1 = lu_solve(A, b) + randmatrix(5, 1, min=-1e-5, max=1.e-5) + x2 = improve_solution(A, x1, b) + assert norm(residual(A, x2, b), 2) < norm(residual(A, x1, b), 2) + +def test_exp_pade(): + for i in range(3): + dps = 15 + extra = 15 + mp.dps = dps + extra + dm = 0 + N = 3 + dg = range(1,N+1) + a = diag(dg) + expa = diag([exp(x) for x in dg]) + # choose a random matrix not close to be singular + # to avoid adding too much extra precision in computing + # m**-1 * M * m + while abs(dm) < 0.01: + m = randmatrix(N) + dm = det(m) + m = m/dm + a1 = m**-1 * a * m + e2 = m**-1 * expa * m + mp.dps = dps + e1 = expm(a1, method='pade') + mp.dps = dps + extra + d = e2 - e1 + #print d + mp.dps = dps + assert norm(d, inf).ae(0) + mp.dps = 15 + +def test_qr(): + mp.dps = 15 # used default value for dps + lowlimit = -9 # lower limit of matrix element value + uplimit = 9 # uppter limit of matrix element value + maxm = 4 # max matrix size + flg = False # toggle to create real vs complex matrix + zero = mpf('0.0') + + for k in xrange(0,10): + exdps = 0 + mode = 'full' + flg = bool(k % 2) + + # generate arbitrary matrix size (2 to maxm) + num1 = nint(maxm*rand()) + num2 = nint(maxm*rand()) + m = int(max(num1, num2)) + n = int(min(num1, num2)) + + # create matrix + A = mp.matrix(m,n) + + # populate matrix values with arbitrary integers + if flg: + flg = False + dtype = 'complex' + for j in xrange(0,n): + for i in xrange(0,m): + val = nint(lowlimit + (uplimit-lowlimit)*rand()) + val2 = nint(lowlimit + (uplimit-lowlimit)*rand()) + A[i,j] = mpc(val, val2) + else: + flg = True + dtype = 'real' + for j in xrange(0,n): + for i in xrange(0,m): + val = nint(lowlimit + (uplimit-lowlimit)*rand()) + A[i,j] = mpf(val) + + # perform A -> QR decomposition + Q, R = qr(A, mode, edps = exdps) + + #print('\n\n A = \n', nstr(A, 4)) + #print('\n Q = \n', nstr(Q, 4)) + #print('\n R = \n', nstr(R, 4)) + #print('\n Q*R = \n', nstr(Q*R, 4)) + + maxnorm = mpf('1.0E-11') + n1 = norm(A - Q * R) + #print '\n Norm of A - Q * R = ', n1 + assert n1 <= maxnorm + + if dtype == 'real': + n1 = norm(eye(m) - Q.T * Q) + #print ' Norm of I - Q.T * Q = ', n1 + assert n1 <= maxnorm + + n1 = norm(eye(m) - Q * Q.T) + #print ' Norm of I - Q * Q.T = ', n1 + assert n1 <= maxnorm + + if dtype == 'complex': + n1 = norm(eye(m) - Q.T * Q.conjugate()) + #print ' Norm of I - Q.T * Q.conjugate() = ', n1 + assert n1 <= maxnorm + + n1 = norm(eye(m) - Q.conjugate() * Q.T) + #print ' Norm of I - Q.conjugate() * Q.T = ', n1 + assert n1 <= maxnorm diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_matrices.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_matrices.py new file mode 100644 index 0000000000000000000000000000000000000000..1547b90664dba66a98a7f026a04a4ed1aa1ed3b4 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_matrices.py @@ -0,0 +1,253 @@ +import pytest +import sys +from mpmath import * + +def test_matrix_basic(): + A1 = matrix(3) + for i in range(3): + A1[i,i] = 1 + assert A1 == eye(3) + assert A1 == matrix(A1) + A2 = matrix(3, 2) + assert not A2._matrix__data + A3 = matrix([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) + assert list(A3) == list(range(1, 10)) + A3[1,1] = 0 + assert not (1, 1) in A3._matrix__data + A4 = matrix([[1, 2, 3], [4, 5, 6]]) + A5 = matrix([[6, -1], [3, 2], [0, -3]]) + assert A4 * A5 == matrix([[12, -6], [39, -12]]) + assert A1 * A3 == A3 * A1 == A3 + pytest.raises(ValueError, lambda: A2*A2) + l = [[10, 20, 30], [40, 0, 60], [70, 80, 90]] + A6 = matrix(l) + assert A6.tolist() == l + assert A6 == eval(repr(A6)) + A6 = fp.matrix(A6) + assert A6 == eval(repr(A6)) + assert A6*1j == eval(repr(A6*1j)) + assert A3 * 10 == 10 * A3 == A6 + assert A2.rows == 3 + assert A2.cols == 2 + A3.rows = 2 + A3.cols = 2 + assert len(A3._matrix__data) == 3 + assert A4 + A4 == 2*A4 + pytest.raises(ValueError, lambda: A4 + A2) + assert sum(A1 - A1) == 0 + A7 = matrix([[1, 2], [3, 4], [5, 6], [7, 8]]) + x = matrix([10, -10]) + assert A7*x == matrix([-10, -10, -10, -10]) + A8 = ones(5) + assert sum((A8 + 1) - (2 - zeros(5))) == 0 + assert (1 + ones(4)) / 2 - 1 == zeros(4) + assert eye(3)**10 == eye(3) + pytest.raises(ValueError, lambda: A7**2) + A9 = randmatrix(3) + A10 = matrix(A9) + A9[0,0] = -100 + assert A9 != A10 + assert nstr(A9) + +def test_matmul(): + """ + Test the PEP465 "@" matrix multiplication syntax. + To avoid syntax errors when importing this file in Python 3.5 and below, we have to use exec() - sorry for that. + """ + # TODO remove exec() wrapper as soon as we drop support for Python <= 3.5 + if sys.hexversion < 0x30500f0: + # we are on Python < 3.5 + pytest.skip("'@' (__matmul__) is only supported in Python 3.5 or newer") + A4 = matrix([[1, 2, 3], [4, 5, 6]]) + A5 = matrix([[6, -1], [3, 2], [0, -3]]) + exec("assert A4 @ A5 == A4 * A5") + +def test_matrix_slices(): + A = matrix([ [1, 2, 3], + [4, 5 ,6], + [7, 8 ,9]]) + V = matrix([1,2,3,4,5]) + + # Get slice + assert A[:,:] == A + assert A[:,1] == matrix([[2],[5],[8]]) + assert A[2,:] == matrix([[7, 8 ,9]]) + assert A[1:3,1:3] == matrix([[5,6],[8,9]]) + assert V[2:4] == matrix([3,4]) + pytest.raises(IndexError, lambda: A[:,1:6]) + + # Assign slice with matrix + A1 = matrix(3) + A1[:,:] = A + assert A1[:,:] == matrix([[1, 2, 3], + [4, 5 ,6], + [7, 8 ,9]]) + A1[0,:] = matrix([[10, 11, 12]]) + assert A1 == matrix([ [10, 11, 12], + [4, 5 ,6], + [7, 8 ,9]]) + A1[:,2] = matrix([[13], [14], [15]]) + assert A1 == matrix([ [10, 11, 13], + [4, 5 ,14], + [7, 8 ,15]]) + A1[:2,:2] = matrix([[16, 17], [18 , 19]]) + assert A1 == matrix([ [16, 17, 13], + [18, 19 ,14], + [7, 8 ,15]]) + V[1:3] = 10 + assert V == matrix([1,10,10,4,5]) + with pytest.raises(ValueError): + A1[2,:] = A[:,1] + + with pytest.raises(IndexError): + A1[2,1:20] = A[:,:] + + # Assign slice with scalar + A1[:,2] = 10 + assert A1 == matrix([ [16, 17, 10], + [18, 19 ,10], + [7, 8 ,10]]) + A1[:,:] = 40 + for x in A1: + assert x == 40 + + +def test_matrix_power(): + A = matrix([[1, 2], [3, 4]]) + assert A**2 == A*A + assert A**3 == A*A*A + assert A**-1 == inverse(A) + assert A**-2 == inverse(A*A) + +def test_matrix_transform(): + A = matrix([[1, 2], [3, 4], [5, 6]]) + assert A.T == A.transpose() == matrix([[1, 3, 5], [2, 4, 6]]) + swap_row(A, 1, 2) + assert A == matrix([[1, 2], [5, 6], [3, 4]]) + l = [1, 2] + swap_row(l, 0, 1) + assert l == [2, 1] + assert extend(eye(3), [1,2,3]) == matrix([[1,0,0,1],[0,1,0,2],[0,0,1,3]]) + +def test_matrix_conjugate(): + A = matrix([[1 + j, 0], [2, j]]) + assert A.conjugate() == matrix([[mpc(1, -1), 0], [2, mpc(0, -1)]]) + assert A.transpose_conj() == A.H == matrix([[mpc(1, -1), 2], + [0, mpc(0, -1)]]) + +def test_matrix_creation(): + assert diag([1, 2, 3]) == matrix([[1, 0, 0], [0, 2, 0], [0, 0, 3]]) + A1 = ones(2, 3) + assert A1.rows == 2 and A1.cols == 3 + for a in A1: + assert a == 1 + A2 = zeros(3, 2) + assert A2.rows == 3 and A2.cols == 2 + for a in A2: + assert a == 0 + assert randmatrix(10) != randmatrix(10) + one = mpf(1) + assert hilbert(3) == matrix([[one, one/2, one/3], + [one/2, one/3, one/4], + [one/3, one/4, one/5]]) + +def test_norms(): + # matrix norms + A = matrix([[1, -2], [-3, -1], [2, 1]]) + assert mnorm(A,1) == 6 + assert mnorm(A,inf) == 4 + assert mnorm(A,'F') == sqrt(20) + # vector norms + assert norm(-3) == 3 + x = [1, -2, 7, -12] + assert norm(x, 1) == 22 + assert round(norm(x, 2), 10) == 14.0712472795 + assert round(norm(x, 10), 10) == 12.0054633727 + assert norm(x, inf) == 12 + +def test_vector(): + x = matrix([0, 1, 2, 3, 4]) + assert x == matrix([[0], [1], [2], [3], [4]]) + assert x[3] == 3 + assert len(x._matrix__data) == 4 + assert list(x) == list(range(5)) + x[0] = -10 + x[4] = 0 + assert x[0] == -10 + assert len(x) == len(x.T) == 5 + assert x.T*x == matrix([[114]]) + +def test_matrix_copy(): + A = ones(6) + B = A.copy() + C = +A + assert A == B + assert A == C + B[0,0] = 0 + assert A != B + C[0,0] = 42 + assert A != C + +def test_matrix_numpy(): + try: + import numpy + except ImportError: + return + l = [[1, 2], [3, 4], [5, 6]] + a = numpy.array(l) + assert matrix(l) == matrix(a) + +def test_interval_matrix_scalar_mult(): + """Multiplication of iv.matrix and any scalar type""" + a = mpi(-1, 1) + b = a + a * 2j + c = mpf(42) + d = c + c * 2j + e = 1.234 + f = fp.convert(e) + g = e + e * 3j + h = fp.convert(g) + M = iv.ones(1) + for x in [a, b, c, d, e, f, g, h]: + assert x * M == iv.matrix([x]) + assert M * x == iv.matrix([x]) + +@pytest.mark.xfail() +def test_interval_matrix_matrix_mult(): + """Multiplication of iv.matrix and other matrix types""" + A = ones(1) + B = fp.ones(1) + M = iv.ones(1) + for X in [A, B, M]: + assert X * M == iv.matrix(X) + assert X * M == X + assert M * X == iv.matrix(X) + assert M * X == X + +def test_matrix_conversion_to_iv(): + # Test that matrices with foreign datatypes are properly converted + for other_type_eye in [eye(3), fp.eye(3), iv.eye(3)]: + A = iv.matrix(other_type_eye) + B = iv.eye(3) + assert type(A[0,0]) == type(B[0,0]) + assert A.tolist() == B.tolist() + +def test_interval_matrix_mult_bug(): + # regression test for interval matrix multiplication: + # result must be nonzero-width and contain the exact result + x = convert('1.00000000000001') # note: this is implicitly rounded to some near mpf float value + A = matrix([[x]]) + B = iv.matrix(A) + C = iv.matrix([[x]]) + assert B == C + B = B * B + C = C * C + assert B == C + assert B[0, 0].delta > 1e-16 + assert B[0, 0].delta < 3e-16 + assert C[0, 0].delta > 1e-16 + assert C[0, 0].delta < 3e-16 + assert mp.mpf('1.00000000000001998401444325291756783368705994138804689654') in B[0, 0] + assert mp.mpf('1.00000000000001998401444325291756783368705994138804689654') in C[0, 0] + # the following caused an error before the bug was fixed + assert iv.matrix(mp.eye(2)) * (iv.ones(2) + mpi(1, 2)) == iv.matrix([[mpi(2, 3), mpi(2, 3)], [mpi(2, 3), mpi(2, 3)]]) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_mpmath.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_mpmath.py new file mode 100644 index 0000000000000000000000000000000000000000..9f1fe36ae9b1b0feca4677eeb90396bfa7ed8f7a --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_mpmath.py @@ -0,0 +1,7 @@ +from mpmath.libmp import * +from mpmath import * + +def test_newstyle_classes(): + for cls in [mp, fp, iv, mpf, mpc]: + for s in cls.__class__.__mro__: + assert isinstance(s, type) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_pickle.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_pickle.py new file mode 100644 index 0000000000000000000000000000000000000000..c3d96e73a53603e0fa3f9525c5c0059725bdffb7 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_pickle.py @@ -0,0 +1,27 @@ +import os +import tempfile +import pickle + +from mpmath import * + +def pickler(obj): + fn = tempfile.mktemp() + + f = open(fn, 'wb') + pickle.dump(obj, f) + f.close() + + f = open(fn, 'rb') + obj2 = pickle.load(f) + f.close() + os.remove(fn) + + return obj2 + +def test_pickle(): + + obj = mpf('0.5') + assert obj == pickler(obj) + + obj = mpc('0.5','0.2') + assert obj == pickler(obj) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_power.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_power.py new file mode 100644 index 0000000000000000000000000000000000000000..7a2447a62c36f9e02df79b9a40a8603f8a69b1d8 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_power.py @@ -0,0 +1,156 @@ +from mpmath import * +from mpmath.libmp import * + +import random + +def test_fractional_pow(): + mp.dps = 15 + assert mpf(16) ** 2.5 == 1024 + assert mpf(64) ** 0.5 == 8 + assert mpf(64) ** -0.5 == 0.125 + assert mpf(16) ** -2.5 == 0.0009765625 + assert (mpf(10) ** 0.5).ae(3.1622776601683791) + assert (mpf(10) ** 2.5).ae(316.2277660168379) + assert (mpf(10) ** -0.5).ae(0.31622776601683794) + assert (mpf(10) ** -2.5).ae(0.0031622776601683794) + assert (mpf(10) ** 0.3).ae(1.9952623149688795) + assert (mpf(10) ** -0.3).ae(0.50118723362727224) + +def test_pow_integer_direction(): + """ + Test that inexact integer powers are rounded in the right + direction. + """ + random.seed(1234) + for prec in [10, 53, 200]: + for i in range(50): + a = random.randint(1<<(prec-1), 1< ab + + +def test_pow_epsilon_rounding(): + """ + Stress test directed rounding for powers with integer exponents. + Basically, we look at the following cases: + + >>> 1.0001 ** -5 # doctest: +SKIP + 0.99950014996500702 + >>> 0.9999 ** -5 # doctest: +SKIP + 1.000500150035007 + >>> (-1.0001) ** -5 # doctest: +SKIP + -0.99950014996500702 + >>> (-0.9999) ** -5 # doctest: +SKIP + -1.000500150035007 + + >>> 1.0001 ** -6 # doctest: +SKIP + 0.99940020994401269 + >>> 0.9999 ** -6 # doctest: +SKIP + 1.0006002100560125 + >>> (-1.0001) ** -6 # doctest: +SKIP + 0.99940020994401269 + >>> (-0.9999) ** -6 # doctest: +SKIP + 1.0006002100560125 + + etc. + + We run the tests with values a very small epsilon away from 1: + small enough that the result is indistinguishable from 1 when + rounded to nearest at the output precision. We check that the + result is not erroneously rounded to 1 in cases where the + rounding should be done strictly away from 1. + """ + + def powr(x, n, r): + return make_mpf(mpf_pow_int(x._mpf_, n, mp.prec, r)) + + for (inprec, outprec) in [(100, 20), (5000, 3000)]: + + mp.prec = inprec + + pos10001 = mpf(1) + mpf(2)**(-inprec+5) + pos09999 = mpf(1) - mpf(2)**(-inprec+5) + neg10001 = -pos10001 + neg09999 = -pos09999 + + mp.prec = outprec + r = round_up + assert powr(pos10001, 5, r) > 1 + assert powr(pos09999, 5, r) == 1 + assert powr(neg10001, 5, r) < -1 + assert powr(neg09999, 5, r) == -1 + assert powr(pos10001, 6, r) > 1 + assert powr(pos09999, 6, r) == 1 + assert powr(neg10001, 6, r) > 1 + assert powr(neg09999, 6, r) == 1 + + assert powr(pos10001, -5, r) == 1 + assert powr(pos09999, -5, r) > 1 + assert powr(neg10001, -5, r) == -1 + assert powr(neg09999, -5, r) < -1 + assert powr(pos10001, -6, r) == 1 + assert powr(pos09999, -6, r) > 1 + assert powr(neg10001, -6, r) == 1 + assert powr(neg09999, -6, r) > 1 + + r = round_down + assert powr(pos10001, 5, r) == 1 + assert powr(pos09999, 5, r) < 1 + assert powr(neg10001, 5, r) == -1 + assert powr(neg09999, 5, r) > -1 + assert powr(pos10001, 6, r) == 1 + assert powr(pos09999, 6, r) < 1 + assert powr(neg10001, 6, r) == 1 + assert powr(neg09999, 6, r) < 1 + + assert powr(pos10001, -5, r) < 1 + assert powr(pos09999, -5, r) == 1 + assert powr(neg10001, -5, r) > -1 + assert powr(neg09999, -5, r) == -1 + assert powr(pos10001, -6, r) < 1 + assert powr(pos09999, -6, r) == 1 + assert powr(neg10001, -6, r) < 1 + assert powr(neg09999, -6, r) == 1 + + r = round_ceiling + assert powr(pos10001, 5, r) > 1 + assert powr(pos09999, 5, r) == 1 + assert powr(neg10001, 5, r) == -1 + assert powr(neg09999, 5, r) > -1 + assert powr(pos10001, 6, r) > 1 + assert powr(pos09999, 6, r) == 1 + assert powr(neg10001, 6, r) > 1 + assert powr(neg09999, 6, r) == 1 + + assert powr(pos10001, -5, r) == 1 + assert powr(pos09999, -5, r) > 1 + assert powr(neg10001, -5, r) > -1 + assert powr(neg09999, -5, r) == -1 + assert powr(pos10001, -6, r) == 1 + assert powr(pos09999, -6, r) > 1 + assert powr(neg10001, -6, r) == 1 + assert powr(neg09999, -6, r) > 1 + + r = round_floor + assert powr(pos10001, 5, r) == 1 + assert powr(pos09999, 5, r) < 1 + assert powr(neg10001, 5, r) < -1 + assert powr(neg09999, 5, r) == -1 + assert powr(pos10001, 6, r) == 1 + assert powr(pos09999, 6, r) < 1 + assert powr(neg10001, 6, r) == 1 + assert powr(neg09999, 6, r) < 1 + + assert powr(pos10001, -5, r) < 1 + assert powr(pos09999, -5, r) == 1 + assert powr(neg10001, -5, r) == -1 + assert powr(neg09999, -5, r) < -1 + assert powr(pos10001, -6, r) < 1 + assert powr(pos09999, -6, r) == 1 + assert powr(neg10001, -6, r) < 1 + assert powr(neg09999, -6, r) == 1 + + mp.dps = 15 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_quad.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_quad.py new file mode 100644 index 0000000000000000000000000000000000000000..fc71c5f5ef9c0ecd876c988e7d033b321f065cdc --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_quad.py @@ -0,0 +1,95 @@ +import pytest +from mpmath import * + +def ae(a, b): + return abs(a-b) < 10**(-mp.dps+5) + +def test_basic_integrals(): + for prec in [15, 30, 100]: + mp.dps = prec + assert ae(quadts(lambda x: x**3 - 3*x**2, [-2, 4]), -12) + assert ae(quadgl(lambda x: x**3 - 3*x**2, [-2, 4]), -12) + assert ae(quadts(sin, [0, pi]), 2) + assert ae(quadts(sin, [0, 2*pi]), 0) + assert ae(quadts(exp, [-inf, -1]), 1/e) + assert ae(quadts(lambda x: exp(-x), [0, inf]), 1) + assert ae(quadts(lambda x: exp(-x*x), [-inf, inf]), sqrt(pi)) + assert ae(quadts(lambda x: 1/(1+x*x), [-1, 1]), pi/2) + assert ae(quadts(lambda x: 1/(1+x*x), [-inf, inf]), pi) + assert ae(quadts(lambda x: 2*sqrt(1-x*x), [-1, 1]), pi) + mp.dps = 15 + +def test_multiple_intervals(): + y,err = quad(lambda x: sign(x), [-0.5, 0.9, 1], maxdegree=2, error=True) + assert abs(y-0.5) < 2*err + +def test_quad_symmetry(): + assert quadts(sin, [-1, 1]) == 0 + assert quadgl(sin, [-1, 1]) == 0 + +def test_quad_infinite_mirror(): + # Check mirrored infinite interval + assert ae(quad(lambda x: exp(-x*x), [inf,-inf]), -sqrt(pi)) + assert ae(quad(lambda x: exp(x), [0,-inf]), -1) + +def test_quadgl_linear(): + assert quadgl(lambda x: x, [0, 1], maxdegree=1).ae(0.5) + +def test_complex_integration(): + assert quadts(lambda x: x, [0, 1+j]).ae(j) + +def test_quadosc(): + mp.dps = 15 + assert quadosc(lambda x: sin(x)/x, [0, inf], period=2*pi).ae(pi/2) + +# Double integrals +def test_double_trivial(): + assert ae(quadts(lambda x, y: x, [0, 1], [0, 1]), 0.5) + assert ae(quadts(lambda x, y: x, [-1, 1], [-1, 1]), 0.0) + +def test_double_1(): + assert ae(quadts(lambda x, y: cos(x+y/2), [-pi/2, pi/2], [0, pi]), 4) + +def test_double_2(): + assert ae(quadts(lambda x, y: (x-1)/((1-x*y)*log(x*y)), [0, 1], [0, 1]), euler) + +def test_double_3(): + assert ae(quadts(lambda x, y: 1/sqrt(1+x*x+y*y), [-1, 1], [-1, 1]), 4*log(2+sqrt(3))-2*pi/3) + +def test_double_4(): + assert ae(quadts(lambda x, y: 1/(1-x*x * y*y), [0, 1], [0, 1]), pi**2 / 8) + +def test_double_5(): + assert ae(quadts(lambda x, y: 1/(1-x*y), [0, 1], [0, 1]), pi**2 / 6) + +def test_double_6(): + assert ae(quadts(lambda x, y: exp(-(x+y)), [0, inf], [0, inf]), 1) + +def test_double_7(): + assert ae(quadts(lambda x, y: exp(-x*x-y*y), [-inf, inf], [-inf, inf]), pi) + + +# Test integrals from "Experimentation in Mathematics" by Borwein, +# Bailey & Girgensohn +def test_expmath_integrals(): + for prec in [15, 30, 50]: + mp.dps = prec + assert ae(quadts(lambda x: x/sinh(x), [0, inf]), pi**2 / 4) + assert ae(quadts(lambda x: log(x)**2 / (1+x**2), [0, inf]), pi**3 / 8) + assert ae(quadts(lambda x: (1+x**2)/(1+x**4), [0, inf]), pi/sqrt(2)) + assert ae(quadts(lambda x: log(x)/cosh(x)**2, [0, inf]), log(pi)-2*log(2)-euler) + assert ae(quadts(lambda x: log(1+x**3)/(1-x+x**2), [0, inf]), 2*pi*log(3)/sqrt(3)) + assert ae(quadts(lambda x: log(x)**2 / (x**2+x+1), [0, 1]), 8*pi**3 / (81*sqrt(3))) + assert ae(quadts(lambda x: log(cos(x))**2, [0, pi/2]), pi/2 * (log(2)**2+pi**2/12)) + assert ae(quadts(lambda x: x**2 / sin(x)**2, [0, pi/2]), pi*log(2)) + assert ae(quadts(lambda x: x**2/sqrt(exp(x)-1), [0, inf]), 4*pi*(log(2)**2 + pi**2/12)) + assert ae(quadts(lambda x: x*exp(-x)*sqrt(1-exp(-2*x)), [0, inf]), pi*(1+2*log(2))/8) + mp.dps = 15 + +# Do not reach full accuracy +@pytest.mark.xfail +def test_expmath_fail(): + assert ae(quadts(lambda x: sqrt(tan(x)), [0, pi/2]), pi*sqrt(2)/2) + assert ae(quadts(lambda x: atan(x)/(x*sqrt(1-x**2)), [0, 1]), pi*log(1+sqrt(2))/2) + assert ae(quadts(lambda x: log(1+x**2)/x**2, [0, 1]), pi/2-log(2)) + assert ae(quadts(lambda x: x**2/((1+x**4)*sqrt(1-x**4)), [0, 1]), pi/8) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_rootfinding.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_rootfinding.py new file mode 100644 index 0000000000000000000000000000000000000000..7c3c06463682eb1fd60efeb75b809bbb932a241c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_rootfinding.py @@ -0,0 +1,91 @@ +import pytest +from mpmath import * +from mpmath.calculus.optimization import Secant, Muller, Bisection, Illinois, \ + Pegasus, Anderson, Ridder, ANewton, Newton, MNewton, MDNewton + +def test_findroot(): + # old tests, assuming secant + mp.dps = 15 + assert findroot(lambda x: 4*x-3, mpf(5)).ae(0.75) + assert findroot(sin, mpf(3)).ae(pi) + assert findroot(sin, (mpf(3), mpf(3.14))).ae(pi) + assert findroot(lambda x: x*x+1, mpc(2+2j)).ae(1j) + # test all solvers with 1 starting point + f = lambda x: cos(x) + for solver in [Newton, Secant, MNewton, Muller, ANewton]: + x = findroot(f, 2., solver=solver) + assert abs(f(x)) < eps + # test all solvers with interval of 2 points + for solver in [Secant, Muller, Bisection, Illinois, Pegasus, Anderson, + Ridder]: + x = findroot(f, (1., 2.), solver=solver) + assert abs(f(x)) < eps + # test types + f = lambda x: (x - 2)**2 + + assert isinstance(findroot(f, 1, tol=1e-10), mpf) + assert isinstance(iv.findroot(f, 1., tol=1e-10), iv.mpf) + assert isinstance(fp.findroot(f, 1, tol=1e-10), float) + assert isinstance(fp.findroot(f, 1+0j, tol=1e-10), complex) + + # issue 401 + with pytest.raises(ValueError): + with workprec(2): + findroot(lambda x: x**2 - 4456178*x + 60372201703370, + mpc(real='5.278e+13', imag='-5.278e+13')) + + # issue 192 + with pytest.raises(ValueError): + findroot(lambda x: -1, 0) + + # issue 387 + with pytest.raises(ValueError): + findroot(lambda p: (1 - p)**30 - 1, 0.9) + +def test_bisection(): + # issue 273 + assert findroot(lambda x: x**2-1,(0,2),solver='bisect') == 1 + +def test_mnewton(): + f = lambda x: polyval([1,3,3,1],x) + x = findroot(f, -0.9, solver='mnewton') + assert abs(f(x)) < eps + +def test_anewton(): + f = lambda x: (x - 2)**100 + x = findroot(f, 1., solver=ANewton) + assert abs(f(x)) < eps + +def test_muller(): + f = lambda x: (2 + x)**3 + 2 + x = findroot(f, 1., solver=Muller) + assert abs(f(x)) < eps + +def test_multiplicity(): + for i in range(1, 5): + assert multiplicity(lambda x: (x - 1)**i, 1) == i + assert multiplicity(lambda x: x**2, 1) == 0 + +def test_multidimensional(): + def f(*x): + return [3*x[0]**2-2*x[1]**2-1, x[0]**2-2*x[0]+x[1]**2+2*x[1]-8] + assert mnorm(jacobian(f, (1,-2)) - matrix([[6,8],[0,-2]]),1) < 1.e-7 + for x, error in MDNewton(mp, f, (1,-2), verbose=0, + norm=lambda x: norm(x, inf)): + pass + assert norm(f(*x), 2) < 1e-14 + # The Chinese mathematician Zhu Shijie was the very first to solve this + # nonlinear system 700 years ago + f1 = lambda x, y: -x + 2*y + f2 = lambda x, y: (x**2 + x*(y**2 - 2) - 4*y) / (x + 4) + f3 = lambda x, y: sqrt(x**2 + y**2) + def f(x, y): + f1x = f1(x, y) + return (f2(x, y) - f1x, f3(x, y) - f1x) + x = findroot(f, (10, 10)) + assert [int(round(i)) for i in x] == [3, 4] + +def test_trivial(): + assert findroot(lambda x: 0, 1) == 1 + assert findroot(lambda x: x, 0) == 0 + #assert findroot(lambda x, y: x + y, (1, -1)) == (1, -1) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_special.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_special.py new file mode 100644 index 0000000000000000000000000000000000000000..30825abd89ada00f937260cb51ef649546be7021 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_special.py @@ -0,0 +1,113 @@ +from mpmath import * + +def test_special(): + assert inf == inf + assert inf != -inf + assert -inf == -inf + assert inf != nan + assert nan != nan + assert isnan(nan) + assert --inf == inf + assert abs(inf) == inf + assert abs(-inf) == inf + assert abs(nan) != abs(nan) + + assert isnan(inf - inf) + assert isnan(inf + (-inf)) + assert isnan(-inf - (-inf)) + + assert isnan(inf + nan) + assert isnan(-inf + nan) + + assert mpf(2) + inf == inf + assert 2 + inf == inf + assert mpf(2) - inf == -inf + assert 2 - inf == -inf + + assert inf > 3 + assert 3 < inf + assert 3 > -inf + assert -inf < 3 + assert inf > mpf(3) + assert mpf(3) < inf + assert mpf(3) > -inf + assert -inf < mpf(3) + + assert not (nan < 3) + assert not (nan > 3) + + assert isnan(inf * 0) + assert isnan(-inf * 0) + assert inf * 3 == inf + assert inf * -3 == -inf + assert -inf * 3 == -inf + assert -inf * -3 == inf + assert inf * inf == inf + assert -inf * -inf == inf + + assert isnan(nan / 3) + assert inf / -3 == -inf + assert inf / 3 == inf + assert 3 / inf == 0 + assert -3 / inf == 0 + assert 0 / inf == 0 + assert isnan(inf / inf) + assert isnan(inf / -inf) + assert isnan(inf / nan) + + assert mpf('inf') == mpf('+inf') == inf + assert mpf('-inf') == -inf + assert isnan(mpf('nan')) + + assert isinf(inf) + assert isinf(-inf) + assert not isinf(mpf(0)) + assert not isinf(nan) + +def test_special_powers(): + assert inf**3 == inf + assert isnan(inf**0) + assert inf**-3 == 0 + assert (-inf)**2 == inf + assert (-inf)**3 == -inf + assert isnan((-inf)**0) + assert (-inf)**-2 == 0 + assert (-inf)**-3 == 0 + assert isnan(nan**5) + assert isnan(nan**0) + +def test_functions_special(): + assert exp(inf) == inf + assert exp(-inf) == 0 + assert isnan(exp(nan)) + assert log(inf) == inf + assert isnan(log(nan)) + assert isnan(sin(inf)) + assert isnan(sin(nan)) + assert atan(inf).ae(pi/2) + assert atan(-inf).ae(-pi/2) + assert isnan(sqrt(nan)) + assert sqrt(inf) == inf + +def test_convert_special(): + float_inf = 1e300 * 1e300 + float_ninf = -float_inf + float_nan = float_inf/float_ninf + assert mpf(3) * float_inf == inf + assert mpf(3) * float_ninf == -inf + assert isnan(mpf(3) * float_nan) + assert not (mpf(3) < float_nan) + assert not (mpf(3) > float_nan) + assert not (mpf(3) <= float_nan) + assert not (mpf(3) >= float_nan) + assert float(mpf('1e1000')) == float_inf + assert float(mpf('-1e1000')) == float_ninf + assert float(mpf('1e100000000000000000')) == float_inf + assert float(mpf('-1e100000000000000000')) == float_ninf + assert float(mpf('1e-100000000000000000')) == 0.0 + +def test_div_bug(): + assert isnan(nan/1) + assert isnan(nan/2) + assert inf/2 == inf + assert (-inf)/2 == -inf diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_str.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_str.py new file mode 100644 index 0000000000000000000000000000000000000000..569244f252c057ec1029b7efbd8b0ffbfbc47522 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_str.py @@ -0,0 +1,14 @@ +from mpmath import nstr, matrix, inf + +def test_nstr(): + m = matrix([[0.75, 0.190940654, -0.0299195971], + [0.190940654, 0.65625, 0.205663228], + [-0.0299195971, 0.205663228, 0.64453125e-20]]) + assert nstr(m, 4, min_fixed=-inf) == \ + '''[ 0.75 0.1909 -0.02992] +[ 0.1909 0.6563 0.2057] +[-0.02992 0.2057 0.000000000000000000006445]''' + assert nstr(m, 4) == \ + '''[ 0.75 0.1909 -0.02992] +[ 0.1909 0.6563 0.2057] +[-0.02992 0.2057 6.445e-21]''' diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_summation.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_summation.py new file mode 100644 index 0000000000000000000000000000000000000000..04ffd29f994e1e6310678eec292c0e03f2d6c725 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_summation.py @@ -0,0 +1,53 @@ +from mpmath import * + +def test_sumem(): + mp.dps = 15 + assert sumem(lambda k: 1/k**2.5, [50, 100]).ae(0.0012524505324784962) + assert sumem(lambda k: k**4 + 3*k + 1, [10, 100]).ae(2050333103) + +def test_nsum(): + mp.dps = 15 + assert nsum(lambda x: x**2, [1, 3]) == 14 + assert nsum(lambda k: 1/factorial(k), [0, inf]).ae(e) + assert nsum(lambda k: (-1)**(k+1) / k, [1, inf]).ae(log(2)) + assert nsum(lambda k: (-1)**(k+1) / k**2, [1, inf]).ae(pi**2 / 12) + assert nsum(lambda k: (-1)**k / log(k), [2, inf]).ae(0.9242998972229388) + assert nsum(lambda k: 1/k**2, [1, inf]).ae(pi**2 / 6) + assert nsum(lambda k: 2**k/fac(k), [0, inf]).ae(exp(2)) + assert nsum(lambda k: 1/k**2, [4, inf], method='e').ae(0.2838229557371153) + assert abs(fp.nsum(lambda k: 1/k**4, [1, fp.inf]) - 1.082323233711138) < 1e-5 + assert abs(fp.nsum(lambda k: 1/k**4, [1, fp.inf], method='e') - 1.082323233711138) < 1e-4 + +def test_nprod(): + mp.dps = 15 + assert nprod(lambda k: exp(1/k**2), [1,inf], method='r').ae(exp(pi**2/6)) + assert nprod(lambda x: x**2, [1, 3]) == 36 + +def test_fsum(): + mp.dps = 15 + assert fsum([]) == 0 + assert fsum([-4]) == -4 + assert fsum([2,3]) == 5 + assert fsum([1e-100,1]) == 1 + assert fsum([1,1e-100]) == 1 + assert fsum([1e100,1]) == 1e100 + assert fsum([1,1e100]) == 1e100 + assert fsum([1e-100,0]) == 1e-100 + assert fsum([1e-100,1e100,1e-100]) == 1e100 + assert fsum([2,1+1j,1]) == 4+1j + assert fsum([2,inf,3]) == inf + assert fsum([2,-1], absolute=1) == 3 + assert fsum([2,-1], squared=1) == 5 + assert fsum([1,1+j], squared=1) == 1+2j + assert fsum([1,3+4j], absolute=1) == 6 + assert fsum([1,2+3j], absolute=1, squared=1) == 14 + assert isnan(fsum([inf,-inf])) + assert fsum([inf,-inf], absolute=1) == inf + assert fsum([inf,-inf], squared=1) == inf + assert fsum([inf,-inf], absolute=1, squared=1) == inf + assert iv.fsum([1,mpi(2,3)]) == mpi(3,4) + +def test_fprod(): + mp.dps = 15 + assert fprod([]) == 1 + assert fprod([2,3]) == 6 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_visualization.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_visualization.py new file mode 100644 index 0000000000000000000000000000000000000000..81ffd05194322f00e4c75dc02bc862b383468bff --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/test_visualization.py @@ -0,0 +1,32 @@ +""" +Limited tests of the visualization module. Right now it just makes +sure that passing custom Axes works. + +""" + +from mpmath import mp, fp + +def test_axes(): + try: + import matplotlib + version = matplotlib.__version__.split("-")[0] + version = version.split(".")[:2] + if [int(_) for _ in version] < [0,99]: + raise ImportError + import pylab + except ImportError: + print("\nSkipping test (pylab not available or too old version)\n") + return + fig = pylab.figure() + axes = fig.add_subplot(111) + for ctx in [mp, fp]: + ctx.plot(lambda x: x**2, [0, 3], axes=axes) + assert axes.get_xlabel() == 'x' + assert axes.get_ylabel() == 'f(x)' + + fig = pylab.figure() + axes = fig.add_subplot(111) + for ctx in [mp, fp]: + ctx.cplot(lambda z: z, [-2, 2], [-10, 10], axes=axes) + assert axes.get_xlabel() == 'Re(z)' + assert axes.get_ylabel() == 'Im(z)' diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/torture.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/torture.py new file mode 100644 index 0000000000000000000000000000000000000000..845d5c6d7d017e51e1ed9a8fe3106cfa32fd967f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/mpmath/tests/torture.py @@ -0,0 +1,224 @@ +""" +Torture tests for asymptotics and high precision evaluation of +special functions. + +(Other torture tests may also be placed here.) + +Running this file (gmpy recommended!) takes several CPU minutes. +With Python 2.6+, multiprocessing is used automatically to run tests +in parallel if many cores are available. (A single test may take between +a second and several minutes; possibly more.) + +The idea: + +* We evaluate functions at positive, negative, imaginary, 45- and 135-degree + complex values with magnitudes between 10^-20 to 10^20, at precisions between + 5 and 150 digits (we can go even higher for fast functions). + +* Comparing the result from two different precision levels provides + a strong consistency check (particularly for functions that use + different algorithms at different precision levels). + +* That the computation finishes at all (without failure), within reasonable + time, provides a check that evaluation works at all: that the code runs, + that it doesn't get stuck in an infinite loop, and that it doesn't use + some extremely slowly algorithm where it could use a faster one. + +TODO: + +* Speed up those functions that take long to finish! +* Generalize to test more cases; more options. +* Implement a timeout mechanism. +* Some functions are notably absent, including the following: + * inverse trigonometric functions (some become inaccurate for complex arguments) + * ci, si (not implemented properly for large complex arguments) + * zeta functions (need to modify test not to try too large imaginary values) + * and others... + +""" + + +import sys, os +from timeit import default_timer as clock + +if "-nogmpy" in sys.argv: + sys.argv.remove('-nogmpy') + os.environ['MPMATH_NOGMPY'] = 'Y' + +filt = '' +if not sys.argv[-1].endswith(".py"): + filt = sys.argv[-1] + +from mpmath import * +from mpmath.libmp.backend import exec_ + +def test_asymp(f, maxdps=150, verbose=False, huge_range=False): + dps = [5,15,25,50,90,150,500,1500,5000,10000] + dps = [p for p in dps if p <= maxdps] + def check(x,y,p,inpt): + if abs(x-y)/abs(y) < workprec(20)(power)(10, -p+1): + return + print() + print("Error!") + print("Input:", inpt) + print("dps =", p) + print("Result 1:", x) + print("Result 2:", y) + print("Absolute error:", abs(x-y)) + print("Relative error:", abs(x-y)/abs(y)) + raise AssertionError + exponents = range(-20,20) + if huge_range: + exponents += [-1000, -100, -50, 50, 100, 1000] + for n in exponents: + if verbose: + sys.stdout.write(". ") + mp.dps = 25 + xpos = mpf(10)**n / 1.1287 + xneg = -xpos + ximag = xpos*j + xcomplex1 = xpos*(1+j) + xcomplex2 = xpos*(-1+j) + for i in range(len(dps)): + if verbose: + print("Testing dps = %s" % dps[i]) + mp.dps = dps[i] + new = f(xpos), f(xneg), f(ximag), f(xcomplex1), f(xcomplex2) + if i != 0: + p = dps[i-1] + check(prev[0], new[0], p, xpos) + check(prev[1], new[1], p, xneg) + check(prev[2], new[2], p, ximag) + check(prev[3], new[3], p, xcomplex1) + check(prev[4], new[4], p, xcomplex2) + prev = new + if verbose: + print() + +a1, a2, a3, a4, a5 = 1.5, -2.25, 3.125, 4, 2 + +def test_bernoulli_huge(): + p, q = bernfrac(9000) + assert p % 10**10 == 9636701091 + assert q == 4091851784687571609141381951327092757255270 + mp.dps = 15 + assert str(bernoulli(10**100)) == '-2.58183325604736e+987675256497386331227838638980680030172857347883537824464410652557820800494271520411283004120790908623' + mp.dps = 50 + assert str(bernoulli(10**100)) == '-2.5818332560473632073252488656039475548106223822913e+987675256497386331227838638980680030172857347883537824464410652557820800494271520411283004120790908623' + mp.dps = 15 + +cases = """\ +test_bernoulli_huge() +test_asymp(lambda z: +pi, maxdps=10000) +test_asymp(lambda z: +e, maxdps=10000) +test_asymp(lambda z: +ln2, maxdps=10000) +test_asymp(lambda z: +ln10, maxdps=10000) +test_asymp(lambda z: +phi, maxdps=10000) +test_asymp(lambda z: +catalan, maxdps=5000) +test_asymp(lambda z: +euler, maxdps=5000) +test_asymp(lambda z: +glaisher, maxdps=1000) +test_asymp(lambda z: +khinchin, maxdps=1000) +test_asymp(lambda z: +twinprime, maxdps=150) +test_asymp(lambda z: stieltjes(2), maxdps=150) +test_asymp(lambda z: +mertens, maxdps=150) +test_asymp(lambda z: +apery, maxdps=5000) +test_asymp(sqrt, maxdps=10000, huge_range=True) +test_asymp(cbrt, maxdps=5000, huge_range=True) +test_asymp(lambda z: root(z,4), maxdps=5000, huge_range=True) +test_asymp(lambda z: root(z,-5), maxdps=5000, huge_range=True) +test_asymp(exp, maxdps=5000, huge_range=True) +test_asymp(expm1, maxdps=1500) +test_asymp(ln, maxdps=5000, huge_range=True) +test_asymp(cosh, maxdps=5000) +test_asymp(sinh, maxdps=5000) +test_asymp(tanh, maxdps=1500) +test_asymp(sin, maxdps=5000, huge_range=True) +test_asymp(cos, maxdps=5000, huge_range=True) +test_asymp(tan, maxdps=1500) +test_asymp(agm, maxdps=1500, huge_range=True) +test_asymp(ellipk, maxdps=1500) +test_asymp(ellipe, maxdps=1500) +test_asymp(lambertw, huge_range=True) +test_asymp(lambda z: lambertw(z,-1)) +test_asymp(lambda z: lambertw(z,1)) +test_asymp(lambda z: lambertw(z,4)) +test_asymp(gamma) +test_asymp(loggamma) # huge_range=True ? +test_asymp(ei) +test_asymp(e1) +test_asymp(li, huge_range=True) +test_asymp(ci) +test_asymp(si) +test_asymp(chi) +test_asymp(shi) +test_asymp(erf) +test_asymp(erfc) +test_asymp(erfi) +test_asymp(lambda z: besselj(2, z)) +test_asymp(lambda z: bessely(2, z)) +test_asymp(lambda z: besseli(2, z)) +test_asymp(lambda z: besselk(2, z)) +test_asymp(lambda z: besselj(-2.25, z)) +test_asymp(lambda z: bessely(-2.25, z)) +test_asymp(lambda z: besseli(-2.25, z)) +test_asymp(lambda z: besselk(-2.25, z)) +test_asymp(airyai) +test_asymp(airybi) +test_asymp(lambda z: hyp0f1(a1, z)) +test_asymp(lambda z: hyp1f1(a1, a2, z)) +test_asymp(lambda z: hyp1f2(a1, a2, a3, z)) +test_asymp(lambda z: hyp2f0(a1, a2, z)) +test_asymp(lambda z: hyperu(a1, a2, z)) +test_asymp(lambda z: hyp2f1(a1, a2, a3, z)) +test_asymp(lambda z: hyp2f2(a1, a2, a3, a4, z)) +test_asymp(lambda z: hyp2f3(a1, a2, a3, a4, a5, z)) +test_asymp(lambda z: coulombf(a1, a2, z)) +test_asymp(lambda z: coulombg(a1, a2, z)) +test_asymp(lambda z: polylog(2,z)) +test_asymp(lambda z: polylog(3,z)) +test_asymp(lambda z: polylog(-2,z)) +test_asymp(lambda z: expint(4, z)) +test_asymp(lambda z: expint(-4, z)) +test_asymp(lambda z: expint(2.25, z)) +test_asymp(lambda z: gammainc(2.5, z, 5)) +test_asymp(lambda z: gammainc(2.5, 5, z)) +test_asymp(lambda z: hermite(3, z)) +test_asymp(lambda z: hermite(2.5, z)) +test_asymp(lambda z: legendre(3, z)) +test_asymp(lambda z: legendre(4, z)) +test_asymp(lambda z: legendre(2.5, z)) +test_asymp(lambda z: legenp(a1, a2, z)) +test_asymp(lambda z: legenq(a1, a2, z), maxdps=90) # abnormally slow +test_asymp(lambda z: jtheta(1, z, 0.5)) +test_asymp(lambda z: jtheta(2, z, 0.5)) +test_asymp(lambda z: jtheta(3, z, 0.5)) +test_asymp(lambda z: jtheta(4, z, 0.5)) +test_asymp(lambda z: jtheta(1, z, 0.5, 1)) +test_asymp(lambda z: jtheta(2, z, 0.5, 1)) +test_asymp(lambda z: jtheta(3, z, 0.5, 1)) +test_asymp(lambda z: jtheta(4, z, 0.5, 1)) +test_asymp(barnesg, maxdps=90) +""" + +def testit(line): + if filt in line: + print(line) + t1 = clock() + exec_(line, globals(), locals()) + t2 = clock() + elapsed = t2-t1 + print("Time:", elapsed, "for", line, "(OK)") + +if __name__ == '__main__': + try: + from multiprocessing import Pool + mapf = Pool(None).map + print("Running tests with multiprocessing") + except ImportError: + print("Not using multiprocessing") + mapf = map + t1 = clock() + tasks = cases.splitlines() + mapf(testit, tasks) + t2 = clock() + print("Cumulative wall time:", t2-t1) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx-3.6.1.dist-info/licenses/LICENSE.txt b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx-3.6.1.dist-info/licenses/LICENSE.txt new file mode 100644 index 0000000000000000000000000000000000000000..0bf9a8f3f4922a0b70aac95ac7aab9574975342b --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx-3.6.1.dist-info/licenses/LICENSE.txt @@ -0,0 +1,37 @@ +NetworkX is distributed with the 3-clause BSD license. + +:: + + Copyright (c) 2004-2025, NetworkX Developers + Aric Hagberg + Dan Schult + Pieter Swart + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the NetworkX Developers nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/components/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/components/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f9ae2caba856daba534037f4a6f967abfad49552 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/components/__init__.py @@ -0,0 +1,6 @@ +from .connected import * +from .strongly_connected import * +from .weakly_connected import * +from .attracting import * +from .biconnected import * +from .semiconnected import * diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/components/attracting.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/components/attracting.py new file mode 100644 index 0000000000000000000000000000000000000000..3d77cd93d70efab5f29c77c7d135f4730e4c3a4a --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/components/attracting.py @@ -0,0 +1,115 @@ +"""Attracting components.""" + +import networkx as nx +from networkx.utils.decorators import not_implemented_for + +__all__ = [ + "number_attracting_components", + "attracting_components", + "is_attracting_component", +] + + +@not_implemented_for("undirected") +@nx._dispatchable +def attracting_components(G): + """Generates the attracting components in `G`. + + An attracting component in a directed graph `G` is a strongly connected + component with the property that a random walker on the graph will never + leave the component, once it enters the component. + + The nodes in attracting components can also be thought of as recurrent + nodes. If a random walker enters the attractor containing the node, then + the node will be visited infinitely often. + + To obtain induced subgraphs on each component use: + ``(G.subgraph(c).copy() for c in attracting_components(G))`` + + Parameters + ---------- + G : DiGraph, MultiDiGraph + The graph to be analyzed. + + Returns + ------- + attractors : generator of sets + A generator of sets of nodes, one for each attracting component of G. + + Raises + ------ + NetworkXNotImplemented + If the input graph is undirected. + + See Also + -------- + number_attracting_components + is_attracting_component + + """ + scc = list(nx.strongly_connected_components(G)) + cG = nx.condensation(G, scc) + for n in cG: + if cG.out_degree(n) == 0: + yield scc[n] + + +@not_implemented_for("undirected") +@nx._dispatchable +def number_attracting_components(G): + """Returns the number of attracting components in `G`. + + Parameters + ---------- + G : DiGraph, MultiDiGraph + The graph to be analyzed. + + Returns + ------- + n : int + The number of attracting components in G. + + Raises + ------ + NetworkXNotImplemented + If the input graph is undirected. + + See Also + -------- + attracting_components + is_attracting_component + + """ + return sum(1 for ac in attracting_components(G)) + + +@not_implemented_for("undirected") +@nx._dispatchable +def is_attracting_component(G): + """Returns True if `G` consists of a single attracting component. + + Parameters + ---------- + G : DiGraph, MultiDiGraph + The graph to be analyzed. + + Returns + ------- + attracting : bool + True if `G` has a single attracting component. Otherwise, False. + + Raises + ------ + NetworkXNotImplemented + If the input graph is undirected. + + See Also + -------- + attracting_components + number_attracting_components + + """ + ac = list(attracting_components(G)) + if len(ac) == 1: + return len(ac[0]) == len(G) + return False diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/components/biconnected.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/components/biconnected.py new file mode 100644 index 0000000000000000000000000000000000000000..fd0f3865bb18e9c9eb37d768c7fd3caceb1cde86 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/components/biconnected.py @@ -0,0 +1,394 @@ +"""Biconnected components and articulation points.""" + +from itertools import chain + +import networkx as nx +from networkx.utils.decorators import not_implemented_for + +__all__ = [ + "biconnected_components", + "biconnected_component_edges", + "is_biconnected", + "articulation_points", +] + + +@not_implemented_for("directed") +@nx._dispatchable +def is_biconnected(G): + """Returns True if the graph is biconnected, False otherwise. + + A graph is biconnected if, and only if, it cannot be disconnected by + removing only one node (and all edges incident on that node). If + removing a node increases the number of disconnected components + in the graph, that node is called an articulation point, or cut + vertex. A biconnected graph has no articulation points. + + Parameters + ---------- + G : NetworkX Graph + An undirected graph. + + Returns + ------- + biconnected : bool + True if the graph is biconnected, False otherwise. + + Raises + ------ + NetworkXNotImplemented + If the input graph is not undirected. + + Examples + -------- + >>> G = nx.path_graph(4) + >>> print(nx.is_biconnected(G)) + False + >>> G.add_edge(0, 3) + >>> print(nx.is_biconnected(G)) + True + + See Also + -------- + biconnected_components + articulation_points + biconnected_component_edges + is_strongly_connected + is_weakly_connected + is_connected + is_semiconnected + + Notes + ----- + The algorithm to find articulation points and biconnected + components is implemented using a non-recursive depth-first-search + (DFS) that keeps track of the highest level that back edges reach + in the DFS tree. A node `n` is an articulation point if, and only + if, there exists a subtree rooted at `n` such that there is no + back edge from any successor of `n` that links to a predecessor of + `n` in the DFS tree. By keeping track of all the edges traversed + by the DFS we can obtain the biconnected components because all + edges of a bicomponent will be traversed consecutively between + articulation points. + + References + ---------- + .. [1] Hopcroft, J.; Tarjan, R. (1973). + "Efficient algorithms for graph manipulation". + Communications of the ACM 16: 372–378. doi:10.1145/362248.362272 + + """ + bccs = biconnected_components(G) + try: + bcc = next(bccs) + except StopIteration: + # No bicomponents (empty graph?) + return False + try: + next(bccs) + except StopIteration: + # Only one bicomponent + return len(bcc) == len(G) + else: + # Multiple bicomponents + return False + + +@not_implemented_for("directed") +@nx._dispatchable +def biconnected_component_edges(G): + """Returns a generator of lists of edges, one list for each biconnected + component of the input graph. + + Biconnected components are maximal subgraphs such that the removal of a + node (and all edges incident on that node) will not disconnect the + subgraph. Note that nodes may be part of more than one biconnected + component. Those nodes are articulation points, or cut vertices. + However, each edge belongs to one, and only one, biconnected component. + + Notice that by convention a dyad is considered a biconnected component. + + Parameters + ---------- + G : NetworkX Graph + An undirected graph. + + Returns + ------- + edges : generator of lists + Generator of lists of edges, one list for each bicomponent. + + Raises + ------ + NetworkXNotImplemented + If the input graph is not undirected. + + Examples + -------- + >>> G = nx.barbell_graph(4, 2) + >>> print(nx.is_biconnected(G)) + False + >>> bicomponents_edges = list(nx.biconnected_component_edges(G)) + >>> len(bicomponents_edges) + 5 + >>> G.add_edge(2, 8) + >>> print(nx.is_biconnected(G)) + True + >>> bicomponents_edges = list(nx.biconnected_component_edges(G)) + >>> len(bicomponents_edges) + 1 + + See Also + -------- + is_biconnected, + biconnected_components, + articulation_points, + + Notes + ----- + The algorithm to find articulation points and biconnected + components is implemented using a non-recursive depth-first-search + (DFS) that keeps track of the highest level that back edges reach + in the DFS tree. A node `n` is an articulation point if, and only + if, there exists a subtree rooted at `n` such that there is no + back edge from any successor of `n` that links to a predecessor of + `n` in the DFS tree. By keeping track of all the edges traversed + by the DFS we can obtain the biconnected components because all + edges of a bicomponent will be traversed consecutively between + articulation points. + + References + ---------- + .. [1] Hopcroft, J.; Tarjan, R. (1973). + "Efficient algorithms for graph manipulation". + Communications of the ACM 16: 372–378. doi:10.1145/362248.362272 + + """ + yield from _biconnected_dfs(G, components=True) + + +@not_implemented_for("directed") +@nx._dispatchable +def biconnected_components(G): + """Returns a generator of sets of nodes, one set for each biconnected + component of the graph + + Biconnected components are maximal subgraphs such that the removal of a + node (and all edges incident on that node) will not disconnect the + subgraph. Note that nodes may be part of more than one biconnected + component. Those nodes are articulation points, or cut vertices. The + removal of articulation points will increase the number of connected + components of the graph. + + Notice that by convention a dyad is considered a biconnected component. + + Parameters + ---------- + G : NetworkX Graph + An undirected graph. + + Returns + ------- + nodes : generator + Generator of sets of nodes, one set for each biconnected component. + + Raises + ------ + NetworkXNotImplemented + If the input graph is not undirected. + + Examples + -------- + >>> G = nx.lollipop_graph(5, 1) + >>> print(nx.is_biconnected(G)) + False + >>> bicomponents = list(nx.biconnected_components(G)) + >>> len(bicomponents) + 2 + >>> G.add_edge(0, 5) + >>> print(nx.is_biconnected(G)) + True + >>> bicomponents = list(nx.biconnected_components(G)) + >>> len(bicomponents) + 1 + + You can generate a sorted list of biconnected components, largest + first, using sort. + + >>> G.remove_edge(0, 5) + >>> [len(c) for c in sorted(nx.biconnected_components(G), key=len, reverse=True)] + [5, 2] + + If you only want the largest connected component, it's more + efficient to use max instead of sort. + + >>> Gc = max(nx.biconnected_components(G), key=len) + + To create the components as subgraphs use: + ``(G.subgraph(c).copy() for c in biconnected_components(G))`` + + See Also + -------- + is_biconnected + articulation_points + biconnected_component_edges + k_components : this function is a special case where k=2 + bridge_components : similar to this function, but is defined using + 2-edge-connectivity instead of 2-node-connectivity. + + Notes + ----- + The algorithm to find articulation points and biconnected + components is implemented using a non-recursive depth-first-search + (DFS) that keeps track of the highest level that back edges reach + in the DFS tree. A node `n` is an articulation point if, and only + if, there exists a subtree rooted at `n` such that there is no + back edge from any successor of `n` that links to a predecessor of + `n` in the DFS tree. By keeping track of all the edges traversed + by the DFS we can obtain the biconnected components because all + edges of a bicomponent will be traversed consecutively between + articulation points. + + References + ---------- + .. [1] Hopcroft, J.; Tarjan, R. (1973). + "Efficient algorithms for graph manipulation". + Communications of the ACM 16: 372–378. doi:10.1145/362248.362272 + + """ + for comp in _biconnected_dfs(G, components=True): + yield set(chain.from_iterable(comp)) + + +@not_implemented_for("directed") +@nx._dispatchable +def articulation_points(G): + """Yield the articulation points, or cut vertices, of a graph. + + An articulation point or cut vertex is any node whose removal (along with + all its incident edges) increases the number of connected components of + a graph. An undirected connected graph without articulation points is + biconnected. Articulation points belong to more than one biconnected + component of a graph. + + Notice that by convention a dyad is considered a biconnected component. + + Parameters + ---------- + G : NetworkX Graph + An undirected graph. + + Yields + ------ + node + An articulation point in the graph. + + Raises + ------ + NetworkXNotImplemented + If the input graph is not undirected. + + Examples + -------- + + >>> G = nx.barbell_graph(4, 2) + >>> print(nx.is_biconnected(G)) + False + >>> len(list(nx.articulation_points(G))) + 4 + >>> G.add_edge(2, 8) + >>> print(nx.is_biconnected(G)) + True + >>> len(list(nx.articulation_points(G))) + 0 + + See Also + -------- + is_biconnected + biconnected_components + biconnected_component_edges + + Notes + ----- + The algorithm to find articulation points and biconnected + components is implemented using a non-recursive depth-first-search + (DFS) that keeps track of the highest level that back edges reach + in the DFS tree. A node `n` is an articulation point if, and only + if, there exists a subtree rooted at `n` such that there is no + back edge from any successor of `n` that links to a predecessor of + `n` in the DFS tree. By keeping track of all the edges traversed + by the DFS we can obtain the biconnected components because all + edges of a bicomponent will be traversed consecutively between + articulation points. + + References + ---------- + .. [1] Hopcroft, J.; Tarjan, R. (1973). + "Efficient algorithms for graph manipulation". + Communications of the ACM 16: 372–378. doi:10.1145/362248.362272 + + """ + seen = set() + for articulation in _biconnected_dfs(G, components=False): + if articulation not in seen: + seen.add(articulation) + yield articulation + + +@not_implemented_for("directed") +def _biconnected_dfs(G, components=True): + # depth-first search algorithm to generate articulation points + # and biconnected components + visited = set() + for start in G: + if start in visited: + continue + discovery = {start: 0} # time of first discovery of node during search + low = {start: 0} + root_children = 0 + visited.add(start) + edge_stack = [] + stack = [(start, start, iter(G[start]))] + edge_index = {} + while stack: + grandparent, parent, children = stack[-1] + try: + child = next(children) + if grandparent == child: + continue + if child in visited: + if discovery[child] <= discovery[parent]: # back edge + low[parent] = min(low[parent], discovery[child]) + if components: + edge_index[parent, child] = len(edge_stack) + edge_stack.append((parent, child)) + else: + low[child] = discovery[child] = len(discovery) + visited.add(child) + stack.append((parent, child, iter(G[child]))) + if components: + edge_index[parent, child] = len(edge_stack) + edge_stack.append((parent, child)) + + except StopIteration: + stack.pop() + if len(stack) > 1: + if low[parent] >= discovery[grandparent]: + if components: + ind = edge_index[grandparent, parent] + yield edge_stack[ind:] + del edge_stack[ind:] + + else: + yield grandparent + low[grandparent] = min(low[parent], low[grandparent]) + elif stack: # length 1 so grandparent is root + root_children += 1 + if components: + ind = edge_index[grandparent, parent] + yield edge_stack[ind:] + del edge_stack[ind:] + if not components: + # root node is articulation point if it has more than 1 child + if root_children > 1: + yield start diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/components/connected.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/components/connected.py new file mode 100644 index 0000000000000000000000000000000000000000..0dd009a8220b08ce47c71afe8239958786f09209 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/components/connected.py @@ -0,0 +1,282 @@ +"""Connected components.""" + +import networkx as nx +from networkx.utils.decorators import not_implemented_for + +from ...utils import arbitrary_element + +__all__ = [ + "number_connected_components", + "connected_components", + "is_connected", + "node_connected_component", +] + + +@not_implemented_for("directed") +@nx._dispatchable +def connected_components(G): + """Generate connected components. + + The connected components of an undirected graph partition the graph into + disjoint sets of nodes. Each of these sets induces a subgraph of graph + `G` that is connected and not part of any larger connected subgraph. + + A graph is connected (:func:`is_connected`) if, for every pair of distinct + nodes, there is a path between them. If there is a pair of nodes for + which such path does not exist, the graph is not connected (also referred + to as "disconnected"). + + A graph consisting of a single node and no edges is connected. + Connectivity is undefined for the null graph (graph with no nodes). + + Parameters + ---------- + G : NetworkX graph + An undirected graph + + Yields + ------ + comp : set + A set of nodes in one connected component of the graph. + + Raises + ------ + NetworkXNotImplemented + If G is directed. + + Examples + -------- + Generate a sorted list of connected components, largest first. + + >>> G = nx.path_graph(4) + >>> nx.add_path(G, [10, 11, 12]) + >>> [len(c) for c in sorted(nx.connected_components(G), key=len, reverse=True)] + [4, 3] + + If you only want the largest connected component, it's more + efficient to use max instead of sort. + + >>> largest_cc = max(nx.connected_components(G), key=len) + + To create the induced subgraph of each component use: + + >>> S = [G.subgraph(c).copy() for c in nx.connected_components(G)] + + See Also + -------- + number_connected_components + is_connected + number_weakly_connected_components + number_strongly_connected_components + + Notes + ----- + This function is for undirected graphs only. For directed graphs, use + :func:`strongly_connected_components` or + :func:`weakly_connected_components`. + + The algorithm is based on a Breadth-First Search (BFS) traversal and its + time complexity is $O(n + m)$, where $n$ is the number of nodes and $m$ the + number of edges in the graph. + + """ + seen = set() + n = len(G) # must be outside the loop to avoid performance hit with graph views + for v in G: + if v not in seen: + c = _plain_bfs(G, n - len(seen), v) + seen.update(c) + yield c + + +@not_implemented_for("directed") +@nx._dispatchable +def number_connected_components(G): + """Returns the number of connected components. + + The connected components of an undirected graph partition the graph into + disjoint sets of nodes. Each of these sets induces a subgraph of graph + `G` that is connected and not part of any larger connected subgraph. + + A graph is connected (:func:`is_connected`) if, for every pair of distinct + nodes, there is a path between them. If there is a pair of nodes for + which such path does not exist, the graph is not connected (also referred + to as "disconnected"). + + A graph consisting of a single node and no edges is connected. + Connectivity is undefined for the null graph (graph with no nodes). + + Parameters + ---------- + G : NetworkX graph + An undirected graph. + + Returns + ------- + n : integer + Number of connected components + + Raises + ------ + NetworkXNotImplemented + If G is directed. + + Examples + -------- + >>> G = nx.Graph([(0, 1), (1, 2), (5, 6), (3, 4)]) + >>> nx.number_connected_components(G) + 3 + + See Also + -------- + connected_components + is_connected + number_weakly_connected_components + number_strongly_connected_components + + Notes + ----- + This function is for undirected graphs only. For directed graphs, use + :func:`number_strongly_connected_components` or + :func:`number_weakly_connected_components`. + + The algorithm is based on a Breadth-First Search (BFS) traversal and its + time complexity is $O(n + m)$, where $n$ is the number of nodes and $m$ the + number of edges in the graph. + + """ + return sum(1 for _ in connected_components(G)) + + +@not_implemented_for("directed") +@nx._dispatchable +def is_connected(G): + """Returns True if the graph is connected, False otherwise. + + A graph is connected if, for every pair of distinct nodes, there is a + path between them. If there is a pair of nodes for which such path does + not exist, the graph is not connected (also referred to as "disconnected"). + + A graph consisting of a single node and no edges is connected. + Connectivity is undefined for the null graph (graph with no nodes). + + Parameters + ---------- + G : NetworkX Graph + An undirected graph. + + Returns + ------- + connected : bool + True if the graph is connected, False otherwise. + + Raises + ------ + NetworkXNotImplemented + If G is directed. + + Examples + -------- + >>> G = nx.path_graph(4) + >>> print(nx.is_connected(G)) + True + + See Also + -------- + is_strongly_connected + is_weakly_connected + is_semiconnected + is_biconnected + connected_components + + Notes + ----- + This function is for undirected graphs only. For directed graphs, use + :func:`is_strongly_connected` or :func:`is_weakly_connected`. + + The algorithm is based on a Breadth-First Search (BFS) traversal and its + time complexity is $O(n + m)$, where $n$ is the number of nodes and $m$ the + number of edges in the graph. + + """ + n = len(G) + if n == 0: + raise nx.NetworkXPointlessConcept( + "Connectivity is undefined for the null graph." + ) + return len(next(connected_components(G))) == n + + +@not_implemented_for("directed") +@nx._dispatchable +def node_connected_component(G, n): + """Returns the set of nodes in the component of graph containing node n. + + A connected component is a set of nodes that induces a subgraph of graph + `G` that is connected and not part of any larger connected subgraph. + + A graph is connected (:func:`is_connected`) if, for every pair of distinct + nodes, there is a path between them. If there is a pair of nodes for + which such path does not exist, the graph is not connected (also referred + to as "disconnected"). + + A graph consisting of a single node and no edges is connected. + Connectivity is undefined for the null graph (graph with no nodes). + + Parameters + ---------- + G : NetworkX Graph + An undirected graph. + + n : node label + A node in G + + Returns + ------- + comp : set + A set of nodes in the component of G containing node n. + + Raises + ------ + NetworkXNotImplemented + If G is directed. + + Examples + -------- + >>> G = nx.Graph([(0, 1), (1, 2), (5, 6), (3, 4)]) + >>> nx.node_connected_component(G, 0) # nodes of component that contains node 0 + {0, 1, 2} + + See Also + -------- + connected_components + + Notes + ----- + This function is for undirected graphs only. + + The algorithm is based on a Breadth-First Search (BFS) traversal and its + time complexity is $O(n + m)$, where $n$ is the number of nodes and $m$ the + number of edges in the graph. + + """ + return _plain_bfs(G, len(G), n) + + +def _plain_bfs(G, n, source): + """A fast BFS node generator""" + adj = G._adj + seen = {source} + nextlevel = [source] + while nextlevel: + thislevel = nextlevel + nextlevel = [] + for v in thislevel: + for w in adj[v]: + if w not in seen: + seen.add(w) + nextlevel.append(w) + if len(seen) == n: + return seen + return seen diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/components/strongly_connected.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/components/strongly_connected.py new file mode 100644 index 0000000000000000000000000000000000000000..a69a6c8801fe896bd0bf6813637072480e1a28d5 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/components/strongly_connected.py @@ -0,0 +1,359 @@ +"""Strongly connected components.""" + +import networkx as nx +from networkx.utils.decorators import not_implemented_for + +__all__ = [ + "number_strongly_connected_components", + "strongly_connected_components", + "is_strongly_connected", + "kosaraju_strongly_connected_components", + "condensation", +] + + +@not_implemented_for("undirected") +@nx._dispatchable +def strongly_connected_components(G): + """Generate nodes in strongly connected components of graph. + + Parameters + ---------- + G : NetworkX Graph + A directed graph. + + Returns + ------- + comp : generator of sets + A generator of sets of nodes, one for each strongly connected + component of G. + + Raises + ------ + NetworkXNotImplemented + If G is undirected. + + Examples + -------- + Generate a sorted list of strongly connected components, largest first. + + >>> G = nx.cycle_graph(4, create_using=nx.DiGraph()) + >>> nx.add_cycle(G, [10, 11, 12]) + >>> [ + ... len(c) + ... for c in sorted(nx.strongly_connected_components(G), key=len, reverse=True) + ... ] + [4, 3] + + If you only want the largest component, it's more efficient to + use max instead of sort. + + >>> largest = max(nx.strongly_connected_components(G), key=len) + + See Also + -------- + connected_components + weakly_connected_components + kosaraju_strongly_connected_components + + Notes + ----- + Uses Tarjan's algorithm[1]_ with Nuutila's modifications[2]_. + Nonrecursive version of algorithm. + + References + ---------- + .. [1] Depth-first search and linear graph algorithms, R. Tarjan + SIAM Journal of Computing 1(2):146-160, (1972). + + .. [2] On finding the strongly connected components in a directed graph. + E. Nuutila and E. Soisalon-Soinen + Information Processing Letters 49(1): 9-14, (1994).. + + """ + preorder = {} + lowlink = {} + scc_found = set() + scc_queue = [] + i = 0 # Preorder counter + neighbors = {v: iter(G._adj[v]) for v in G} + for source in G: + if source not in scc_found: + queue = [source] + while queue: + v = queue[-1] + if v not in preorder: + i = i + 1 + preorder[v] = i + done = True + for w in neighbors[v]: + if w not in preorder: + queue.append(w) + done = False + break + if done: + lowlink[v] = preorder[v] + for w in G._adj[v]: + if w not in scc_found: + if preorder[w] > preorder[v]: + lowlink[v] = min([lowlink[v], lowlink[w]]) + else: + lowlink[v] = min([lowlink[v], preorder[w]]) + queue.pop() + if lowlink[v] == preorder[v]: + scc = {v} + while scc_queue and preorder[scc_queue[-1]] > preorder[v]: + k = scc_queue.pop() + scc.add(k) + scc_found.update(scc) + yield scc + else: + scc_queue.append(v) + + +@not_implemented_for("undirected") +@nx._dispatchable +def kosaraju_strongly_connected_components(G, source=None): + """Generate nodes in strongly connected components of graph. + + Parameters + ---------- + G : NetworkX Graph + A directed graph. + + source : node, optional (default=None) + Specify a node from which to start the depth-first search. + If not provided, the algorithm will start from an arbitrary node. + + Yields + ------ + set + A set of all nodes in a strongly connected component of `G`. + + Raises + ------ + NetworkXNotImplemented + If `G` is undirected. + + NetworkXError + If `source` is not a node in `G`. + + Examples + -------- + Generate a list of strongly connected components of a graph: + + >>> G = nx.cycle_graph(4, create_using=nx.DiGraph()) + >>> nx.add_cycle(G, [10, 11, 12]) + >>> sorted(nx.kosaraju_strongly_connected_components(G), key=len, reverse=True) + [{0, 1, 2, 3}, {10, 11, 12}] + + If you only want the largest component, it's more efficient to + use `max()` instead of `sorted()`. + + >>> max(nx.kosaraju_strongly_connected_components(G), key=len) + {0, 1, 2, 3} + + See Also + -------- + strongly_connected_components + + Notes + ----- + Uses Kosaraju's algorithm. + """ + post = list(nx.dfs_postorder_nodes(G.reverse(copy=False), source=source)) + n = len(post) + seen = set() + while post and len(seen) < n: + r = post.pop() + if r in seen: + continue + new = {r} + seen.add(r) + stack = [r] + while stack and len(seen) < n: + v = stack.pop() + for w in G._adj[v]: + if w not in seen: + new.add(w) + seen.add(w) + stack.append(w) + yield new + + +@not_implemented_for("undirected") +@nx._dispatchable +def number_strongly_connected_components(G): + """Returns number of strongly connected components in graph. + + Parameters + ---------- + G : NetworkX graph + A directed graph. + + Returns + ------- + n : integer + Number of strongly connected components + + Raises + ------ + NetworkXNotImplemented + If G is undirected. + + Examples + -------- + >>> G = nx.DiGraph( + ... [(0, 1), (1, 2), (2, 0), (2, 3), (4, 5), (3, 4), (5, 6), (6, 3), (6, 7)] + ... ) + >>> nx.number_strongly_connected_components(G) + 3 + + See Also + -------- + strongly_connected_components + number_connected_components + number_weakly_connected_components + + Notes + ----- + For directed graphs only. + """ + return sum(1 for scc in strongly_connected_components(G)) + + +@not_implemented_for("undirected") +@nx._dispatchable +def is_strongly_connected(G): + """Test directed graph for strong connectivity. + + A directed graph is strongly connected if and only if every vertex in + the graph is reachable from every other vertex. + + Parameters + ---------- + G : NetworkX Graph + A directed graph. + + Returns + ------- + connected : bool + True if the graph is strongly connected, False otherwise. + + Examples + -------- + >>> G = nx.DiGraph([(0, 1), (1, 2), (2, 3), (3, 0), (2, 4), (4, 2)]) + >>> nx.is_strongly_connected(G) + True + >>> G.remove_edge(2, 3) + >>> nx.is_strongly_connected(G) + False + + Raises + ------ + NetworkXNotImplemented + If G is undirected. + + See Also + -------- + is_weakly_connected + is_semiconnected + is_connected + is_biconnected + strongly_connected_components + + Notes + ----- + For directed graphs only. + """ + if len(G) == 0: + raise nx.NetworkXPointlessConcept( + """Connectivity is undefined for the null graph.""" + ) + + return len(next(strongly_connected_components(G))) == len(G) + + +@not_implemented_for("undirected") +@nx._dispatchable(returns_graph=True) +def condensation(G, scc=None): + """Returns the condensation of G. + + The condensation of G is the graph with each of the strongly connected + components contracted into a single node. + + Parameters + ---------- + G : NetworkX DiGraph + A directed graph. + + scc: list or generator (optional, default=None) + Strongly connected components. If provided, the elements in + `scc` must partition the nodes in `G`. If not provided, it will be + calculated as scc=nx.strongly_connected_components(G). + + Returns + ------- + C : NetworkX DiGraph + The condensation graph C of G. The node labels are integers + corresponding to the index of the component in the list of + strongly connected components of G. C has a graph attribute named + 'mapping' with a dictionary mapping the original nodes to the + nodes in C to which they belong. Each node in C also has a node + attribute 'members' with the set of original nodes in G that + form the SCC that the node in C represents. + + Raises + ------ + NetworkXNotImplemented + If G is undirected. + + Examples + -------- + Contracting two sets of strongly connected nodes into two distinct SCC + using the barbell graph. + + >>> G = nx.barbell_graph(4, 0) + >>> G.remove_edge(3, 4) + >>> G = nx.DiGraph(G) + >>> H = nx.condensation(G) + >>> H.nodes.data() + NodeDataView({0: {'members': {0, 1, 2, 3}}, 1: {'members': {4, 5, 6, 7}}}) + >>> H.graph["mapping"] + {0: 0, 1: 0, 2: 0, 3: 0, 4: 1, 5: 1, 6: 1, 7: 1} + + Contracting a complete graph into one single SCC. + + >>> G = nx.complete_graph(7, create_using=nx.DiGraph) + >>> H = nx.condensation(G) + >>> H.nodes + NodeView((0,)) + >>> H.nodes.data() + NodeDataView({0: {'members': {0, 1, 2, 3, 4, 5, 6}}}) + + Notes + ----- + After contracting all strongly connected components to a single node, + the resulting graph is a directed acyclic graph. + + """ + if scc is None: + scc = nx.strongly_connected_components(G) + mapping = {} + members = {} + C = nx.DiGraph() + # Add mapping dict as graph attribute + C.graph["mapping"] = mapping + if len(G) == 0: + return C + for i, component in enumerate(scc): + members[i] = component + mapping.update((n, i) for n in component) + number_of_components = i + 1 + C.add_nodes_from(range(number_of_components)) + C.add_edges_from( + (mapping[u], mapping[v]) for u, v in G.edges() if mapping[u] != mapping[v] + ) + # Add a list of members (ie original nodes) to each node (ie scc) in C. + nx.set_node_attributes(C, members, "members") + return C diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/components/weakly_connected.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/components/weakly_connected.py new file mode 100644 index 0000000000000000000000000000000000000000..fdb07ff7de5d93b1b01889217d5d8698bbf7fa32 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/components/weakly_connected.py @@ -0,0 +1,196 @@ +"""Weakly connected components.""" + +import networkx as nx +from networkx.utils.decorators import not_implemented_for + +__all__ = [ + "number_weakly_connected_components", + "weakly_connected_components", + "is_weakly_connected", +] + + +@not_implemented_for("undirected") +@nx._dispatchable +def weakly_connected_components(G): + """Generate weakly connected components of G. + + Parameters + ---------- + G : NetworkX graph + A directed graph + + Returns + ------- + comp : generator of sets + A generator of sets of nodes, one for each weakly connected + component of G. + + Raises + ------ + NetworkXNotImplemented + If G is undirected. + + Examples + -------- + Generate a sorted list of weakly connected components, largest first. + + >>> G = nx.path_graph(4, create_using=nx.DiGraph()) + >>> nx.add_path(G, [10, 11, 12]) + >>> [ + ... len(c) + ... for c in sorted(nx.weakly_connected_components(G), key=len, reverse=True) + ... ] + [4, 3] + + If you only want the largest component, it's more efficient to + use max instead of sort: + + >>> largest_cc = max(nx.weakly_connected_components(G), key=len) + + See Also + -------- + connected_components + strongly_connected_components + + Notes + ----- + For directed graphs only. + + """ + seen = set() + n = len(G) # must be outside the loop to avoid performance hit with graph views + for v in G: + if v not in seen: + c = _plain_bfs(G, n - len(seen), v) + seen.update(c) + yield c + + +@not_implemented_for("undirected") +@nx._dispatchable +def number_weakly_connected_components(G): + """Returns the number of weakly connected components in G. + + Parameters + ---------- + G : NetworkX graph + A directed graph. + + Returns + ------- + n : integer + Number of weakly connected components + + Raises + ------ + NetworkXNotImplemented + If G is undirected. + + Examples + -------- + >>> G = nx.DiGraph([(0, 1), (2, 1), (3, 4)]) + >>> nx.number_weakly_connected_components(G) + 2 + + See Also + -------- + weakly_connected_components + number_connected_components + number_strongly_connected_components + + Notes + ----- + For directed graphs only. + + """ + return sum(1 for _ in weakly_connected_components(G)) + + +@not_implemented_for("undirected") +@nx._dispatchable +def is_weakly_connected(G): + """Test directed graph for weak connectivity. + + A directed graph is weakly connected if and only if the graph + is connected when the direction of the edge between nodes is ignored. + + Note that if a graph is strongly connected (i.e. the graph is connected + even when we account for directionality), it is by definition weakly + connected as well. + + Parameters + ---------- + G : NetworkX Graph + A directed graph. + + Returns + ------- + connected : bool + True if the graph is weakly connected, False otherwise. + + Raises + ------ + NetworkXNotImplemented + If G is undirected. + + Examples + -------- + >>> G = nx.DiGraph([(0, 1), (2, 1)]) + >>> G.add_node(3) + >>> nx.is_weakly_connected(G) # node 3 is not connected to the graph + False + >>> G.add_edge(2, 3) + >>> nx.is_weakly_connected(G) + True + + See Also + -------- + is_strongly_connected + is_semiconnected + is_connected + is_biconnected + weakly_connected_components + + Notes + ----- + For directed graphs only. + + """ + n = len(G) + if n == 0: + raise nx.NetworkXPointlessConcept( + """Connectivity is undefined for the null graph.""" + ) + + return len(next(weakly_connected_components(G))) == n + + +def _plain_bfs(G, n, source): + """A fast BFS node generator + + The direction of the edge between nodes is ignored. + + For directed graphs only. + + """ + Gsucc = G._succ + Gpred = G._pred + seen = {source} + nextlevel = [source] + + while nextlevel: + thislevel = nextlevel + nextlevel = [] + for v in thislevel: + for w in Gsucc[v]: + if w not in seen: + seen.add(w) + nextlevel.append(w) + for w in Gpred[v]: + if w not in seen: + seen.add(w) + nextlevel.append(w) + if len(seen) == n: + return seen + return seen diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/flow/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/flow/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c5d19abed99501086359c87670edc31a680fe36c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/flow/__init__.py @@ -0,0 +1,11 @@ +from .maxflow import * +from .mincost import * +from .boykovkolmogorov import * +from .dinitz_alg import * +from .edmondskarp import * +from .gomory_hu import * +from .preflowpush import * +from .shortestaugmentingpath import * +from .capacityscaling import * +from .networksimplex import * +from .utils import build_flow_dict, build_residual_network diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/flow/capacityscaling.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/flow/capacityscaling.py new file mode 100644 index 0000000000000000000000000000000000000000..bf68565c5486bb7b60e7ddcf6089e448bc6ddef1 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/flow/capacityscaling.py @@ -0,0 +1,407 @@ +""" +Capacity scaling minimum cost flow algorithm. +""" + +__all__ = ["capacity_scaling"] + +from itertools import chain +from math import log + +import networkx as nx + +from ...utils import BinaryHeap, arbitrary_element, not_implemented_for + + +def _detect_unboundedness(R): + """Detect infinite-capacity negative cycles.""" + G = nx.DiGraph() + G.add_nodes_from(R) + + # Value simulating infinity. + inf = R.graph["inf"] + # True infinity. + f_inf = float("inf") + for u in R: + for v, e in R[u].items(): + # Compute the minimum weight of infinite-capacity (u, v) edges. + w = f_inf + for k, e in e.items(): + if e["capacity"] == inf: + w = min(w, e["weight"]) + if w != f_inf: + G.add_edge(u, v, weight=w) + + if nx.negative_edge_cycle(G): + raise nx.NetworkXUnbounded( + "Negative cost cycle of infinite capacity found. " + "Min cost flow may be unbounded below." + ) + + +@not_implemented_for("undirected") +def _build_residual_network(G, demand, capacity, weight): + """Build a residual network and initialize a zero flow.""" + if sum(G.nodes[u].get(demand, 0) for u in G) != 0: + raise nx.NetworkXUnfeasible("Sum of the demands should be 0.") + + R = nx.MultiDiGraph() + R.add_nodes_from( + (u, {"excess": -G.nodes[u].get(demand, 0), "potential": 0}) for u in G + ) + + inf = float("inf") + # Detect selfloops with infinite capacities and negative weights. + for u, v, e in nx.selfloop_edges(G, data=True): + if e.get(weight, 0) < 0 and e.get(capacity, inf) == inf: + raise nx.NetworkXUnbounded( + "Negative cost cycle of infinite capacity found. " + "Min cost flow may be unbounded below." + ) + + # Extract edges with positive capacities. Self loops excluded. + if G.is_multigraph(): + edge_list = [ + (u, v, k, e) + for u, v, k, e in G.edges(data=True, keys=True) + if u != v and e.get(capacity, inf) > 0 + ] + else: + edge_list = [ + (u, v, 0, e) + for u, v, e in G.edges(data=True) + if u != v and e.get(capacity, inf) > 0 + ] + # Simulate infinity with the larger of the sum of absolute node imbalances + # the sum of finite edge capacities or any positive value if both sums are + # zero. This allows the infinite-capacity edges to be distinguished for + # unboundedness detection and directly participate in residual capacity + # calculation. + inf = ( + max( + sum(abs(R.nodes[u]["excess"]) for u in R), + 2 + * sum( + e[capacity] + for u, v, k, e in edge_list + if capacity in e and e[capacity] != inf + ), + ) + or 1 + ) + for u, v, k, e in edge_list: + r = min(e.get(capacity, inf), inf) + w = e.get(weight, 0) + # Add both (u, v) and (v, u) into the residual network marked with the + # original key. (key[1] == True) indicates the (u, v) is in the + # original network. + R.add_edge(u, v, key=(k, True), capacity=r, weight=w, flow=0) + R.add_edge(v, u, key=(k, False), capacity=0, weight=-w, flow=0) + + # Record the value simulating infinity. + R.graph["inf"] = inf + + _detect_unboundedness(R) + + return R + + +def _build_flow_dict(G, R, capacity, weight): + """Build a flow dictionary from a residual network.""" + inf = float("inf") + flow_dict = {} + if G.is_multigraph(): + for u in G: + flow_dict[u] = {} + for v, es in G[u].items(): + flow_dict[u][v] = { + # Always saturate negative selfloops. + k: ( + 0 + if ( + u != v or e.get(capacity, inf) <= 0 or e.get(weight, 0) >= 0 + ) + else e[capacity] + ) + for k, e in es.items() + } + for v, es in R[u].items(): + if v in flow_dict[u]: + flow_dict[u][v].update( + (k[0], e["flow"]) for k, e in es.items() if e["flow"] > 0 + ) + else: + for u in G: + flow_dict[u] = { + # Always saturate negative selfloops. + v: ( + 0 + if (u != v or e.get(capacity, inf) <= 0 or e.get(weight, 0) >= 0) + else e[capacity] + ) + for v, e in G[u].items() + } + flow_dict[u].update( + (v, e["flow"]) + for v, es in R[u].items() + for e in es.values() + if e["flow"] > 0 + ) + return flow_dict + + +@nx._dispatchable( + node_attrs="demand", edge_attrs={"capacity": float("inf"), "weight": 0} +) +def capacity_scaling( + G, demand="demand", capacity="capacity", weight="weight", heap=BinaryHeap +): + r"""Find a minimum cost flow satisfying all demands in digraph G. + + This is a capacity scaling successive shortest augmenting path algorithm. + + G is a digraph with edge costs and capacities and in which nodes + have demand, i.e., they want to send or receive some amount of + flow. A negative demand means that the node wants to send flow, a + positive demand means that the node want to receive flow. A flow on + the digraph G satisfies all demand if the net flow into each node + is equal to the demand of that node. + + Parameters + ---------- + G : NetworkX graph + DiGraph or MultiDiGraph on which a minimum cost flow satisfying all + demands is to be found. + + demand : string + Nodes of the graph G are expected to have an attribute demand + that indicates how much flow a node wants to send (negative + demand) or receive (positive demand). Note that the sum of the + demands should be 0 otherwise the problem in not feasible. If + this attribute is not present, a node is considered to have 0 + demand. Default value: 'demand'. + + capacity : string + Edges of the graph G are expected to have an attribute capacity + that indicates how much flow the edge can support. If this + attribute is not present, the edge is considered to have + infinite capacity. Default value: 'capacity'. + + weight : string + Edges of the graph G are expected to have an attribute weight + that indicates the cost incurred by sending one unit of flow on + that edge. If not present, the weight is considered to be 0. + Default value: 'weight'. + + heap : class + Type of heap to be used in the algorithm. It should be a subclass of + :class:`MinHeap` or implement a compatible interface. + + If a stock heap implementation is to be used, :class:`BinaryHeap` is + recommended over :class:`PairingHeap` for Python implementations without + optimized attribute accesses (e.g., CPython) despite a slower + asymptotic running time. For Python implementations with optimized + attribute accesses (e.g., PyPy), :class:`PairingHeap` provides better + performance. Default value: :class:`BinaryHeap`. + + Returns + ------- + flowCost : integer + Cost of a minimum cost flow satisfying all demands. + + flowDict : dictionary + If G is a digraph, a dict-of-dicts keyed by nodes such that + flowDict[u][v] is the flow on edge (u, v). + If G is a MultiDiGraph, a dict-of-dicts-of-dicts keyed by nodes + so that flowDict[u][v][key] is the flow on edge (u, v, key). + + Raises + ------ + NetworkXError + This exception is raised if the input graph is not directed, + not connected. + + NetworkXUnfeasible + This exception is raised in the following situations: + + * The sum of the demands is not zero. Then, there is no + flow satisfying all demands. + * There is no flow satisfying all demand. + + NetworkXUnbounded + This exception is raised if the digraph G has a cycle of + negative cost and infinite capacity. Then, the cost of a flow + satisfying all demands is unbounded below. + + Notes + ----- + This algorithm does not work if edge weights are floating-point numbers. + + See also + -------- + :meth:`network_simplex` + + Examples + -------- + A simple example of a min cost flow problem. + + >>> G = nx.DiGraph() + >>> G.add_node("a", demand=-5) + >>> G.add_node("d", demand=5) + >>> G.add_edge("a", "b", weight=3, capacity=4) + >>> G.add_edge("a", "c", weight=6, capacity=10) + >>> G.add_edge("b", "d", weight=1, capacity=9) + >>> G.add_edge("c", "d", weight=2, capacity=5) + >>> flowCost, flowDict = nx.capacity_scaling(G) + >>> flowCost + 24 + >>> flowDict + {'a': {'b': 4, 'c': 1}, 'd': {}, 'b': {'d': 4}, 'c': {'d': 1}} + + It is possible to change the name of the attributes used for the + algorithm. + + >>> G = nx.DiGraph() + >>> G.add_node("p", spam=-4) + >>> G.add_node("q", spam=2) + >>> G.add_node("a", spam=-2) + >>> G.add_node("d", spam=-1) + >>> G.add_node("t", spam=2) + >>> G.add_node("w", spam=3) + >>> G.add_edge("p", "q", cost=7, vacancies=5) + >>> G.add_edge("p", "a", cost=1, vacancies=4) + >>> G.add_edge("q", "d", cost=2, vacancies=3) + >>> G.add_edge("t", "q", cost=1, vacancies=2) + >>> G.add_edge("a", "t", cost=2, vacancies=4) + >>> G.add_edge("d", "w", cost=3, vacancies=4) + >>> G.add_edge("t", "w", cost=4, vacancies=1) + >>> flowCost, flowDict = nx.capacity_scaling( + ... G, demand="spam", capacity="vacancies", weight="cost" + ... ) + >>> flowCost + 37 + >>> flowDict + {'p': {'q': 2, 'a': 2}, 'q': {'d': 1}, 'a': {'t': 4}, 'd': {'w': 2}, 't': {'q': 1, 'w': 1}, 'w': {}} + """ + R = _build_residual_network(G, demand, capacity, weight) + + inf = float("inf") + # Account cost of negative selfloops. + flow_cost = sum( + 0 + if e.get(capacity, inf) <= 0 or e.get(weight, 0) >= 0 + else e[capacity] * e[weight] + for u, v, e in nx.selfloop_edges(G, data=True) + ) + + # Determine the maximum edge capacity. + wmax = max(chain([-inf], (e["capacity"] for u, v, e in R.edges(data=True)))) + if wmax == -inf: + # Residual network has no edges. + return flow_cost, _build_flow_dict(G, R, capacity, weight) + + R_nodes = R.nodes + R_succ = R.succ + + delta = 2 ** int(log(wmax, 2)) + while delta >= 1: + # Saturate Δ-residual edges with negative reduced costs to achieve + # Δ-optimality. + for u in R: + p_u = R_nodes[u]["potential"] + for v, es in R_succ[u].items(): + for k, e in es.items(): + flow = e["capacity"] - e["flow"] + if e["weight"] - p_u + R_nodes[v]["potential"] < 0: + flow = e["capacity"] - e["flow"] + if flow >= delta: + e["flow"] += flow + R_succ[v][u][(k[0], not k[1])]["flow"] -= flow + R_nodes[u]["excess"] -= flow + R_nodes[v]["excess"] += flow + # Determine the Δ-active nodes. + S = set() + T = set() + S_add = S.add + S_remove = S.remove + T_add = T.add + T_remove = T.remove + for u in R: + excess = R_nodes[u]["excess"] + if excess >= delta: + S_add(u) + elif excess <= -delta: + T_add(u) + # Repeatedly augment flow from S to T along shortest paths until + # Δ-feasibility is achieved. + while S and T: + s = arbitrary_element(S) + t = None + # Search for a shortest path in terms of reduce costs from s to + # any t in T in the Δ-residual network. + d = {} + pred = {s: None} + h = heap() + h_insert = h.insert + h_get = h.get + h_insert(s, 0) + while h: + u, d_u = h.pop() + d[u] = d_u + if u in T: + # Path found. + t = u + break + p_u = R_nodes[u]["potential"] + for v, es in R_succ[u].items(): + if v in d: + continue + wmin = inf + # Find the minimum-weighted (u, v) Δ-residual edge. + for k, e in es.items(): + if e["capacity"] - e["flow"] >= delta: + w = e["weight"] + if w < wmin: + wmin = w + kmin = k + emin = e + if wmin == inf: + continue + # Update the distance label of v. + d_v = d_u + wmin - p_u + R_nodes[v]["potential"] + if h_insert(v, d_v): + pred[v] = (u, kmin, emin) + if t is not None: + # Augment Δ units of flow from s to t. + while u != s: + v = u + u, k, e = pred[v] + e["flow"] += delta + R_succ[v][u][(k[0], not k[1])]["flow"] -= delta + # Account node excess and deficit. + R_nodes[s]["excess"] -= delta + R_nodes[t]["excess"] += delta + if R_nodes[s]["excess"] < delta: + S_remove(s) + if R_nodes[t]["excess"] > -delta: + T_remove(t) + # Update node potentials. + d_t = d[t] + for u, d_u in d.items(): + R_nodes[u]["potential"] -= d_u - d_t + else: + # Path not found. + S_remove(s) + delta //= 2 + + if any(R.nodes[u]["excess"] != 0 for u in R): + raise nx.NetworkXUnfeasible("No flow satisfying all demands.") + + # Calculate the flow cost. + for u in R: + for v, es in R_succ[u].items(): + for e in es.values(): + flow = e["flow"] + if flow > 0: + flow_cost += flow * e["weight"] + + return flow_cost, _build_flow_dict(G, R, capacity, weight) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/flow/gomory_hu.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/flow/gomory_hu.py new file mode 100644 index 0000000000000000000000000000000000000000..69913da904547b3a9fe682467b69e696e9c8e0dc --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/flow/gomory_hu.py @@ -0,0 +1,178 @@ +""" +Gomory-Hu tree of undirected Graphs. +""" + +import networkx as nx +from networkx.utils import not_implemented_for + +from .edmondskarp import edmonds_karp +from .utils import build_residual_network + +default_flow_func = edmonds_karp + +__all__ = ["gomory_hu_tree"] + + +@not_implemented_for("directed") +@nx._dispatchable(edge_attrs={"capacity": float("inf")}, returns_graph=True) +def gomory_hu_tree(G, capacity="capacity", flow_func=None): + r"""Returns the Gomory-Hu tree of an undirected graph G. + + A Gomory-Hu tree of an undirected graph with capacities is a + weighted tree that represents the minimum s-t cuts for all s-t + pairs in the graph. + + It only requires `n-1` minimum cut computations instead of the + obvious `n(n-1)/2`. The tree represents all s-t cuts as the + minimum cut value among any pair of nodes is the minimum edge + weight in the shortest path between the two nodes in the + Gomory-Hu tree. + + The Gomory-Hu tree also has the property that removing the + edge with the minimum weight in the shortest path between + any two nodes leaves two connected components that form + a partition of the nodes in G that defines the minimum s-t + cut. + + See Examples section below for details. + + Parameters + ---------- + G : NetworkX graph + Undirected graph + + capacity : string + Edges of the graph G are expected to have an attribute capacity + that indicates how much flow the edge can support. If this + attribute is not present, the edge is considered to have + infinite capacity. Default value: 'capacity'. + + flow_func : function + Function to perform the underlying flow computations. Default value + :func:`edmonds_karp`. This function performs better in sparse graphs + with right tailed degree distributions. + :func:`shortest_augmenting_path` will perform better in denser + graphs. + + Returns + ------- + Tree : NetworkX graph + A NetworkX graph representing the Gomory-Hu tree of the input graph. + + Raises + ------ + NetworkXNotImplemented + Raised if the input graph is directed. + + NetworkXError + Raised if the input graph is an empty Graph. + + Examples + -------- + >>> G = nx.karate_club_graph() + >>> nx.set_edge_attributes(G, 1, "capacity") + >>> T = nx.gomory_hu_tree(G) + >>> # The value of the minimum cut between any pair + ... # of nodes in G is the minimum edge weight in the + ... # shortest path between the two nodes in the + ... # Gomory-Hu tree. + ... def minimum_edge_weight_in_shortest_path(T, u, v): + ... path = nx.shortest_path(T, u, v, weight="weight") + ... return min((T[u][v]["weight"], (u, v)) for (u, v) in zip(path, path[1:])) + >>> u, v = 0, 33 + >>> cut_value, edge = minimum_edge_weight_in_shortest_path(T, u, v) + >>> cut_value + 10 + >>> nx.minimum_cut_value(G, u, v) + 10 + >>> # The Gomory-Hu tree also has the property that removing the + ... # edge with the minimum weight in the shortest path between + ... # any two nodes leaves two connected components that form + ... # a partition of the nodes in G that defines the minimum s-t + ... # cut. + ... cut_value, edge = minimum_edge_weight_in_shortest_path(T, u, v) + >>> T.remove_edge(*edge) + >>> U, V = list(nx.connected_components(T)) + >>> # Thus U and V form a partition that defines a minimum cut + ... # between u and v in G. You can compute the edge cut set, + ... # that is, the set of edges that if removed from G will + ... # disconnect u from v in G, with this information: + ... cutset = set() + >>> for x, nbrs in ((n, G[n]) for n in U): + ... cutset.update((x, y) for y in nbrs if y in V) + >>> # Because we have set the capacities of all edges to 1 + ... # the cutset contains ten edges + ... len(cutset) + 10 + >>> # You can use any maximum flow algorithm for the underlying + ... # flow computations using the argument flow_func + ... from networkx.algorithms import flow + >>> T = nx.gomory_hu_tree(G, flow_func=flow.boykov_kolmogorov) + >>> cut_value, edge = minimum_edge_weight_in_shortest_path(T, u, v) + >>> cut_value + 10 + >>> nx.minimum_cut_value(G, u, v, flow_func=flow.boykov_kolmogorov) + 10 + + Notes + ----- + This implementation is based on Gusfield approach [1]_ to compute + Gomory-Hu trees, which does not require node contractions and has + the same computational complexity than the original method. + + See also + -------- + :func:`minimum_cut` + :func:`maximum_flow` + + References + ---------- + .. [1] Gusfield D: Very simple methods for all pairs network flow analysis. + SIAM J Comput 19(1):143-155, 1990. + + """ + if flow_func is None: + flow_func = default_flow_func + + if len(G) == 0: # empty graph + msg = "Empty Graph does not have a Gomory-Hu tree representation" + raise nx.NetworkXError(msg) + + # Start the tree as a star graph with an arbitrary node at the center + tree = {} + labels = {} + iter_nodes = iter(G) + root = next(iter_nodes) + for n in iter_nodes: + tree[n] = root + + # Reuse residual network + R = build_residual_network(G, capacity) + + # For all the leaves in the star graph tree (that is n-1 nodes). + for source in tree: + # Find neighbor in the tree + target = tree[source] + # compute minimum cut + cut_value, partition = nx.minimum_cut( + G, source, target, capacity=capacity, flow_func=flow_func, residual=R + ) + labels[(source, target)] = cut_value + # Update the tree + # Source will always be in partition[0] and target in partition[1] + for node in partition[0]: + if node != source and node in tree and tree[node] == target: + tree[node] = source + labels[node, source] = labels.get((node, target), cut_value) + # + if target != root and tree[target] in partition[0]: + labels[source, tree[target]] = labels[target, tree[target]] + labels[target, source] = cut_value + tree[source] = tree[target] + tree[target] = source + + # Build the tree + T = nx.Graph() + T.add_nodes_from(G) + T.add_weighted_edges_from(((u, v, labels[u, v]) for u, v in tree.items())) + return T diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/flow/networksimplex.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/flow/networksimplex.py new file mode 100644 index 0000000000000000000000000000000000000000..5baa9766c39c3ac2e02e396879461668cc62bfc7 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/flow/networksimplex.py @@ -0,0 +1,662 @@ +""" +Minimum cost flow algorithms on directed connected graphs. +""" + +__all__ = ["network_simplex"] + +from itertools import chain, islice, repeat +from math import ceil, sqrt + +import networkx as nx +from networkx.utils import not_implemented_for + + +class _DataEssentialsAndFunctions: + def __init__( + self, G, multigraph, demand="demand", capacity="capacity", weight="weight" + ): + # Number all nodes and edges and hereafter reference them using ONLY their numbers + self.node_list = list(G) # nodes + self.node_indices = {u: i for i, u in enumerate(self.node_list)} # node indices + self.node_demands = [ + G.nodes[u].get(demand, 0) for u in self.node_list + ] # node demands + + self.edge_sources = [] # edge sources + self.edge_targets = [] # edge targets + if multigraph: + self.edge_keys = [] # edge keys + self.edge_indices = {} # edge indices + self.edge_capacities = [] # edge capacities + self.edge_weights = [] # edge weights + + if not multigraph: + edges = G.edges(data=True) + else: + edges = G.edges(data=True, keys=True) + + inf = float("inf") + edges = (e for e in edges if e[0] != e[1] and e[-1].get(capacity, inf) != 0) + for i, e in enumerate(edges): + self.edge_sources.append(self.node_indices[e[0]]) + self.edge_targets.append(self.node_indices[e[1]]) + if multigraph: + self.edge_keys.append(e[2]) + self.edge_indices[e[:-1]] = i + self.edge_capacities.append(e[-1].get(capacity, inf)) + self.edge_weights.append(e[-1].get(weight, 0)) + + # spanning tree specific data to be initialized + + self.edge_count = None # number of edges + self.edge_flow = None # edge flows + self.node_potentials = None # node potentials + self.parent = None # parent nodes + self.parent_edge = None # edges to parents + self.subtree_size = None # subtree sizes + self.next_node_dft = None # next nodes in depth-first thread + self.prev_node_dft = None # previous nodes in depth-first thread + self.last_descendent_dft = None # last descendants in depth-first thread + self._spanning_tree_initialized = ( + False # False until initialize_spanning_tree() is called + ) + + def initialize_spanning_tree(self, n, faux_inf): + self.edge_count = len(self.edge_indices) # number of edges + self.edge_flow = list( + chain(repeat(0, self.edge_count), (abs(d) for d in self.node_demands)) + ) # edge flows + self.node_potentials = [ + faux_inf if d <= 0 else -faux_inf for d in self.node_demands + ] # node potentials + self.parent = list(chain(repeat(-1, n), [None])) # parent nodes + self.parent_edge = list( + range(self.edge_count, self.edge_count + n) + ) # edges to parents + self.subtree_size = list(chain(repeat(1, n), [n + 1])) # subtree sizes + self.next_node_dft = list( + chain(range(1, n), [-1, 0]) + ) # next nodes in depth-first thread + self.prev_node_dft = list(range(-1, n)) # previous nodes in depth-first thread + self.last_descendent_dft = list( + chain(range(n), [n - 1]) + ) # last descendants in depth-first thread + self._spanning_tree_initialized = True # True only if all the assignments pass + + def find_apex(self, p, q): + """ + Find the lowest common ancestor of nodes p and q in the spanning tree. + """ + size_p = self.subtree_size[p] + size_q = self.subtree_size[q] + while True: + while size_p < size_q: + p = self.parent[p] + size_p = self.subtree_size[p] + while size_p > size_q: + q = self.parent[q] + size_q = self.subtree_size[q] + if size_p == size_q: + if p != q: + p = self.parent[p] + size_p = self.subtree_size[p] + q = self.parent[q] + size_q = self.subtree_size[q] + else: + return p + + def trace_path(self, p, w): + """ + Returns the nodes and edges on the path from node p to its ancestor w. + """ + Wn = [p] + We = [] + while p != w: + We.append(self.parent_edge[p]) + p = self.parent[p] + Wn.append(p) + return Wn, We + + def find_cycle(self, i, p, q): + """ + Returns the nodes and edges on the cycle containing edge i == (p, q) + when the latter is added to the spanning tree. + + The cycle is oriented in the direction from p to q. + """ + w = self.find_apex(p, q) + Wn, We = self.trace_path(p, w) + Wn.reverse() + We.reverse() + if We != [i]: + We.append(i) + WnR, WeR = self.trace_path(q, w) + del WnR[-1] + Wn += WnR + We += WeR + return Wn, We + + def augment_flow(self, Wn, We, f): + """ + Augment f units of flow along a cycle represented by Wn and We. + """ + for i, p in zip(We, Wn): + if self.edge_sources[i] == p: + self.edge_flow[i] += f + else: + self.edge_flow[i] -= f + + def trace_subtree(self, p): + """ + Yield the nodes in the subtree rooted at a node p. + """ + yield p + l = self.last_descendent_dft[p] + while p != l: + p = self.next_node_dft[p] + yield p + + def remove_edge(self, s, t): + """ + Remove an edge (s, t) where parent[t] == s from the spanning tree. + """ + size_t = self.subtree_size[t] + prev_t = self.prev_node_dft[t] + last_t = self.last_descendent_dft[t] + next_last_t = self.next_node_dft[last_t] + # Remove (s, t). + self.parent[t] = None + self.parent_edge[t] = None + # Remove the subtree rooted at t from the depth-first thread. + self.next_node_dft[prev_t] = next_last_t + self.prev_node_dft[next_last_t] = prev_t + self.next_node_dft[last_t] = t + self.prev_node_dft[t] = last_t + # Update the subtree sizes and last descendants of the (old) ancestors + # of t. + while s is not None: + self.subtree_size[s] -= size_t + if self.last_descendent_dft[s] == last_t: + self.last_descendent_dft[s] = prev_t + s = self.parent[s] + + def make_root(self, q): + """ + Make a node q the root of its containing subtree. + """ + ancestors = [] + while q is not None: + ancestors.append(q) + q = self.parent[q] + ancestors.reverse() + for p, q in zip(ancestors, islice(ancestors, 1, None)): + size_p = self.subtree_size[p] + last_p = self.last_descendent_dft[p] + prev_q = self.prev_node_dft[q] + last_q = self.last_descendent_dft[q] + next_last_q = self.next_node_dft[last_q] + # Make p a child of q. + self.parent[p] = q + self.parent[q] = None + self.parent_edge[p] = self.parent_edge[q] + self.parent_edge[q] = None + self.subtree_size[p] = size_p - self.subtree_size[q] + self.subtree_size[q] = size_p + # Remove the subtree rooted at q from the depth-first thread. + self.next_node_dft[prev_q] = next_last_q + self.prev_node_dft[next_last_q] = prev_q + self.next_node_dft[last_q] = q + self.prev_node_dft[q] = last_q + if last_p == last_q: + self.last_descendent_dft[p] = prev_q + last_p = prev_q + # Add the remaining parts of the subtree rooted at p as a subtree + # of q in the depth-first thread. + self.prev_node_dft[p] = last_q + self.next_node_dft[last_q] = p + self.next_node_dft[last_p] = q + self.prev_node_dft[q] = last_p + self.last_descendent_dft[q] = last_p + + def add_edge(self, i, p, q): + """ + Add an edge (p, q) to the spanning tree where q is the root of a subtree. + """ + last_p = self.last_descendent_dft[p] + next_last_p = self.next_node_dft[last_p] + size_q = self.subtree_size[q] + last_q = self.last_descendent_dft[q] + # Make q a child of p. + self.parent[q] = p + self.parent_edge[q] = i + # Insert the subtree rooted at q into the depth-first thread. + self.next_node_dft[last_p] = q + self.prev_node_dft[q] = last_p + self.prev_node_dft[next_last_p] = last_q + self.next_node_dft[last_q] = next_last_p + # Update the subtree sizes and last descendants of the (new) ancestors + # of q. + while p is not None: + self.subtree_size[p] += size_q + if self.last_descendent_dft[p] == last_p: + self.last_descendent_dft[p] = last_q + p = self.parent[p] + + def update_potentials(self, i, p, q): + """ + Update the potentials of the nodes in the subtree rooted at a node + q connected to its parent p by an edge i. + """ + if q == self.edge_targets[i]: + d = self.node_potentials[p] - self.edge_weights[i] - self.node_potentials[q] + else: + d = self.node_potentials[p] + self.edge_weights[i] - self.node_potentials[q] + for q in self.trace_subtree(q): + self.node_potentials[q] += d + + def reduced_cost(self, i): + """Returns the reduced cost of an edge i.""" + c = ( + self.edge_weights[i] + - self.node_potentials[self.edge_sources[i]] + + self.node_potentials[self.edge_targets[i]] + ) + return c if self.edge_flow[i] == 0 else -c + + def find_entering_edges(self): + """Yield entering edges until none can be found.""" + if self.edge_count == 0: + return + + # Entering edges are found by combining Dantzig's rule and Bland's + # rule. The edges are cyclically grouped into blocks of size B. Within + # each block, Dantzig's rule is applied to find an entering edge. The + # blocks to search is determined following Bland's rule. + B = int(ceil(sqrt(self.edge_count))) # pivot block size + M = (self.edge_count + B - 1) // B # number of blocks needed to cover all edges + m = 0 # number of consecutive blocks without eligible + # entering edges + f = 0 # first edge in block + while m < M: + # Determine the next block of edges. + l = f + B + if l <= self.edge_count: + edges = range(f, l) + else: + l -= self.edge_count + edges = chain(range(f, self.edge_count), range(l)) + f = l + # Find the first edge with the lowest reduced cost. + i = min(edges, key=self.reduced_cost) + c = self.reduced_cost(i) + if c >= 0: + # No entering edge found in the current block. + m += 1 + else: + # Entering edge found. + if self.edge_flow[i] == 0: + p = self.edge_sources[i] + q = self.edge_targets[i] + else: + p = self.edge_targets[i] + q = self.edge_sources[i] + yield i, p, q + m = 0 + # All edges have nonnegative reduced costs. The current flow is + # optimal. + + def residual_capacity(self, i, p): + """Returns the residual capacity of an edge i in the direction away + from its endpoint p. + """ + return ( + self.edge_capacities[i] - self.edge_flow[i] + if self.edge_sources[i] == p + else self.edge_flow[i] + ) + + def find_leaving_edge(self, Wn, We): + """Returns the leaving edge in a cycle represented by Wn and We.""" + j, s = min( + zip(reversed(We), reversed(Wn)), + key=lambda i_p: self.residual_capacity(*i_p), + ) + t = self.edge_targets[j] if self.edge_sources[j] == s else self.edge_sources[j] + return j, s, t + + +@not_implemented_for("undirected") +@nx._dispatchable( + node_attrs="demand", edge_attrs={"capacity": float("inf"), "weight": 0} +) +def network_simplex(G, demand="demand", capacity="capacity", weight="weight"): + r"""Find a minimum cost flow satisfying all demands in digraph G. + + This is a primal network simplex algorithm that uses the leaving + arc rule to prevent cycling. + + G is a digraph with edge costs and capacities and in which nodes + have demand, i.e., they want to send or receive some amount of + flow. A negative demand means that the node wants to send flow, a + positive demand means that the node want to receive flow. A flow on + the digraph G satisfies all demand if the net flow into each node + is equal to the demand of that node. + + Parameters + ---------- + G : NetworkX graph + DiGraph on which a minimum cost flow satisfying all demands is + to be found. + + demand : string + Nodes of the graph G are expected to have an attribute demand + that indicates how much flow a node wants to send (negative + demand) or receive (positive demand). Note that the sum of the + demands should be 0 otherwise the problem in not feasible. If + this attribute is not present, a node is considered to have 0 + demand. Default value: 'demand'. + + capacity : string + Edges of the graph G are expected to have an attribute capacity + that indicates how much flow the edge can support. If this + attribute is not present, the edge is considered to have + infinite capacity. Default value: 'capacity'. + + weight : string + Edges of the graph G are expected to have an attribute weight + that indicates the cost incurred by sending one unit of flow on + that edge. If not present, the weight is considered to be 0. + Default value: 'weight'. + + Returns + ------- + flowCost : integer, float + Cost of a minimum cost flow satisfying all demands. + + flowDict : dictionary + Dictionary of dictionaries keyed by nodes such that + flowDict[u][v] is the flow edge (u, v). + + Raises + ------ + NetworkXError + This exception is raised if the input graph is not directed or + not connected. + + NetworkXUnfeasible + This exception is raised in the following situations: + + * The sum of the demands is not zero. Then, there is no + flow satisfying all demands. + * There is no flow satisfying all demand. + + NetworkXUnbounded + This exception is raised if the digraph G has a cycle of + negative cost and infinite capacity. Then, the cost of a flow + satisfying all demands is unbounded below. + + Notes + ----- + This algorithm is not guaranteed to work if edge weights or demands + are floating point numbers (overflows and roundoff errors can + cause problems). As a workaround you can use integer numbers by + multiplying the relevant edge attributes by a convenient + constant factor (eg 100). + + See also + -------- + cost_of_flow, max_flow_min_cost, min_cost_flow, min_cost_flow_cost + + Examples + -------- + A simple example of a min cost flow problem. + + >>> G = nx.DiGraph() + >>> G.add_node("a", demand=-5) + >>> G.add_node("d", demand=5) + >>> G.add_edge("a", "b", weight=3, capacity=4) + >>> G.add_edge("a", "c", weight=6, capacity=10) + >>> G.add_edge("b", "d", weight=1, capacity=9) + >>> G.add_edge("c", "d", weight=2, capacity=5) + >>> flowCost, flowDict = nx.network_simplex(G) + >>> flowCost + 24 + >>> flowDict + {'a': {'b': 4, 'c': 1}, 'd': {}, 'b': {'d': 4}, 'c': {'d': 1}} + + The mincost flow algorithm can also be used to solve shortest path + problems. To find the shortest path between two nodes u and v, + give all edges an infinite capacity, give node u a demand of -1 and + node v a demand a 1. Then run the network simplex. The value of a + min cost flow will be the distance between u and v and edges + carrying positive flow will indicate the path. + + >>> G = nx.DiGraph() + >>> G.add_weighted_edges_from( + ... [ + ... ("s", "u", 10), + ... ("s", "x", 5), + ... ("u", "v", 1), + ... ("u", "x", 2), + ... ("v", "y", 1), + ... ("x", "u", 3), + ... ("x", "v", 5), + ... ("x", "y", 2), + ... ("y", "s", 7), + ... ("y", "v", 6), + ... ] + ... ) + >>> G.add_node("s", demand=-1) + >>> G.add_node("v", demand=1) + >>> flowCost, flowDict = nx.network_simplex(G) + >>> flowCost == nx.shortest_path_length(G, "s", "v", weight="weight") + True + >>> sorted([(u, v) for u in flowDict for v in flowDict[u] if flowDict[u][v] > 0]) + [('s', 'x'), ('u', 'v'), ('x', 'u')] + >>> nx.shortest_path(G, "s", "v", weight="weight") + ['s', 'x', 'u', 'v'] + + It is possible to change the name of the attributes used for the + algorithm. + + >>> G = nx.DiGraph() + >>> G.add_node("p", spam=-4) + >>> G.add_node("q", spam=2) + >>> G.add_node("a", spam=-2) + >>> G.add_node("d", spam=-1) + >>> G.add_node("t", spam=2) + >>> G.add_node("w", spam=3) + >>> G.add_edge("p", "q", cost=7, vacancies=5) + >>> G.add_edge("p", "a", cost=1, vacancies=4) + >>> G.add_edge("q", "d", cost=2, vacancies=3) + >>> G.add_edge("t", "q", cost=1, vacancies=2) + >>> G.add_edge("a", "t", cost=2, vacancies=4) + >>> G.add_edge("d", "w", cost=3, vacancies=4) + >>> G.add_edge("t", "w", cost=4, vacancies=1) + >>> flowCost, flowDict = nx.network_simplex( + ... G, demand="spam", capacity="vacancies", weight="cost" + ... ) + >>> flowCost + 37 + >>> flowDict + {'p': {'q': 2, 'a': 2}, 'q': {'d': 1}, 'a': {'t': 4}, 'd': {'w': 2}, 't': {'q': 1, 'w': 1}, 'w': {}} + + References + ---------- + .. [1] Z. Kiraly, P. Kovacs. + Efficient implementation of minimum-cost flow algorithms. + Acta Universitatis Sapientiae, Informatica 4(1):67--118. 2012. + .. [2] R. Barr, F. Glover, D. Klingman. + Enhancement of spanning tree labeling procedures for network + optimization. + INFOR 17(1):16--34. 1979. + """ + ########################################################################### + # Problem essentials extraction and sanity check + ########################################################################### + + if len(G) == 0: + raise nx.NetworkXError("graph has no nodes") + + multigraph = G.is_multigraph() + + # extracting data essential to problem + DEAF = _DataEssentialsAndFunctions( + G, multigraph, demand=demand, capacity=capacity, weight=weight + ) + + ########################################################################### + # Quick Error Detection + ########################################################################### + + inf = float("inf") + for u, d in zip(DEAF.node_list, DEAF.node_demands): + if abs(d) == inf: + raise nx.NetworkXError(f"node {u!r} has infinite demand") + for e, w in zip(DEAF.edge_indices, DEAF.edge_weights): + if abs(w) == inf: + raise nx.NetworkXError(f"edge {e!r} has infinite weight") + if not multigraph: + edges = nx.selfloop_edges(G, data=True) + else: + edges = nx.selfloop_edges(G, data=True, keys=True) + for e in edges: + if abs(e[-1].get(weight, 0)) == inf: + raise nx.NetworkXError(f"edge {e[:-1]!r} has infinite weight") + + ########################################################################### + # Quick Infeasibility Detection + ########################################################################### + + if sum(DEAF.node_demands) != 0: + raise nx.NetworkXUnfeasible("total node demand is not zero") + for e, c in zip(DEAF.edge_indices, DEAF.edge_capacities): + if c < 0: + raise nx.NetworkXUnfeasible(f"edge {e!r} has negative capacity") + if not multigraph: + edges = nx.selfloop_edges(G, data=True) + else: + edges = nx.selfloop_edges(G, data=True, keys=True) + for e in edges: + if e[-1].get(capacity, inf) < 0: + raise nx.NetworkXUnfeasible(f"edge {e[:-1]!r} has negative capacity") + + ########################################################################### + # Initialization + ########################################################################### + + # Add a dummy node -1 and connect all existing nodes to it with infinite- + # capacity dummy edges. Node -1 will serve as the root of the + # spanning tree of the network simplex method. The new edges will used to + # trivially satisfy the node demands and create an initial strongly + # feasible spanning tree. + for i, d in enumerate(DEAF.node_demands): + # Must be greater-than here. Zero-demand nodes must have + # edges pointing towards the root to ensure strong feasibility. + if d > 0: + DEAF.edge_sources.append(-1) + DEAF.edge_targets.append(i) + else: + DEAF.edge_sources.append(i) + DEAF.edge_targets.append(-1) + faux_inf = ( + 3 + * max( + sum(c for c in DEAF.edge_capacities if c < inf), + sum(abs(w) for w in DEAF.edge_weights), + sum(abs(d) for d in DEAF.node_demands), + ) + or 1 + ) + + n = len(DEAF.node_list) # number of nodes + DEAF.edge_weights.extend(repeat(faux_inf, n)) + DEAF.edge_capacities.extend(repeat(faux_inf, n)) + + # Construct the initial spanning tree. + DEAF.initialize_spanning_tree(n, faux_inf) + + ########################################################################### + # Pivot loop + ########################################################################### + + for i, p, q in DEAF.find_entering_edges(): + Wn, We = DEAF.find_cycle(i, p, q) + j, s, t = DEAF.find_leaving_edge(Wn, We) + DEAF.augment_flow(Wn, We, DEAF.residual_capacity(j, s)) + # Do nothing more if the entering edge is the same as the leaving edge. + if i != j: + if DEAF.parent[t] != s: + # Ensure that s is the parent of t. + s, t = t, s + if We.index(i) > We.index(j): + # Ensure that q is in the subtree rooted at t. + p, q = q, p + DEAF.remove_edge(s, t) + DEAF.make_root(q) + DEAF.add_edge(i, p, q) + DEAF.update_potentials(i, p, q) + + ########################################################################### + # Infeasibility and unboundedness detection + ########################################################################### + + if any(DEAF.edge_flow[i] != 0 for i in range(-n, 0)): + raise nx.NetworkXUnfeasible("no flow satisfies all node demands") + + if any(DEAF.edge_flow[i] * 2 >= faux_inf for i in range(DEAF.edge_count)) or any( + e[-1].get(capacity, inf) == inf and e[-1].get(weight, 0) < 0 + for e in nx.selfloop_edges(G, data=True) + ): + raise nx.NetworkXUnbounded("negative cycle with infinite capacity found") + + ########################################################################### + # Flow cost calculation and flow dict construction + ########################################################################### + + del DEAF.edge_flow[DEAF.edge_count :] + flow_cost = sum(w * x for w, x in zip(DEAF.edge_weights, DEAF.edge_flow)) + flow_dict = {n: {} for n in DEAF.node_list} + + def add_entry(e): + """Add a flow dict entry.""" + d = flow_dict[e[0]] + for k in e[1:-2]: + try: + d = d[k] + except KeyError: + t = {} + d[k] = t + d = t + d[e[-2]] = e[-1] + + DEAF.edge_sources = ( + DEAF.node_list[s] for s in DEAF.edge_sources + ) # Use original nodes. + DEAF.edge_targets = ( + DEAF.node_list[t] for t in DEAF.edge_targets + ) # Use original nodes. + if not multigraph: + for e in zip(DEAF.edge_sources, DEAF.edge_targets, DEAF.edge_flow): + add_entry(e) + edges = G.edges(data=True) + else: + for e in zip( + DEAF.edge_sources, DEAF.edge_targets, DEAF.edge_keys, DEAF.edge_flow + ): + add_entry(e) + edges = G.edges(data=True, keys=True) + for e in edges: + if e[0] != e[1]: + if e[-1].get(capacity, inf) == 0: + add_entry(e[:-1] + (0,)) + else: + w = e[-1].get(weight, 0) + if w >= 0: + add_entry(e[:-1] + (0,)) + else: + c = e[-1][capacity] + flow_cost += w * c + add_entry(e[:-1] + (c,)) + + return flow_cost, flow_dict diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/flow/utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/flow/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..4d63b5b64ca5f501cbf1098f9d280ecffd4a8f1b --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/flow/utils.py @@ -0,0 +1,194 @@ +""" +Utility classes and functions for network flow algorithms. +""" + +from collections import deque + +import networkx as nx + +__all__ = [ + "CurrentEdge", + "Level", + "GlobalRelabelThreshold", + "build_residual_network", + "detect_unboundedness", + "build_flow_dict", +] + + +class CurrentEdge: + """Mechanism for iterating over out-edges incident to a node in a circular + manner. StopIteration exception is raised when wraparound occurs. + """ + + __slots__ = ("_edges", "_it", "_curr") + + def __init__(self, edges): + self._edges = edges + if self._edges: + self._rewind() + + def get(self): + return self._curr + + def move_to_next(self): + try: + self._curr = next(self._it) + except StopIteration: + self._rewind() + raise + + def _rewind(self): + self._it = iter(self._edges.items()) + self._curr = next(self._it) + + def __eq__(self, other): + return (getattr(self, "_curr", None), self._edges) == ( + (getattr(other, "_curr", None), other._edges) + ) + + +class Level: + """Active and inactive nodes in a level.""" + + __slots__ = ("active", "inactive") + + def __init__(self): + self.active = set() + self.inactive = set() + + +class GlobalRelabelThreshold: + """Measurement of work before the global relabeling heuristic should be + applied. + """ + + def __init__(self, n, m, freq): + self._threshold = (n + m) / freq if freq else float("inf") + self._work = 0 + + def add_work(self, work): + self._work += work + + def is_reached(self): + return self._work >= self._threshold + + def clear_work(self): + self._work = 0 + + +@nx._dispatchable(edge_attrs={"capacity": float("inf")}, returns_graph=True) +def build_residual_network(G, capacity): + """Build a residual network and initialize a zero flow. + + The residual network :samp:`R` from an input graph :samp:`G` has the + same nodes as :samp:`G`. :samp:`R` is a DiGraph that contains a pair + of edges :samp:`(u, v)` and :samp:`(v, u)` iff :samp:`(u, v)` is not a + self-loop, and at least one of :samp:`(u, v)` and :samp:`(v, u)` exists + in :samp:`G`. + + For each edge :samp:`(u, v)` in :samp:`R`, :samp:`R[u][v]['capacity']` + is equal to the capacity of :samp:`(u, v)` in :samp:`G` if it exists + in :samp:`G` or zero otherwise. If the capacity is infinite, + :samp:`R[u][v]['capacity']` will have a high arbitrary finite value + that does not affect the solution of the problem. This value is stored in + :samp:`R.graph['inf']`. For each edge :samp:`(u, v)` in :samp:`R`, + :samp:`R[u][v]['flow']` represents the flow function of :samp:`(u, v)` and + satisfies :samp:`R[u][v]['flow'] == -R[v][u]['flow']`. + + The flow value, defined as the total flow into :samp:`t`, the sink, is + stored in :samp:`R.graph['flow_value']`. If :samp:`cutoff` is not + specified, reachability to :samp:`t` using only edges :samp:`(u, v)` such + that :samp:`R[u][v]['flow'] < R[u][v]['capacity']` induces a minimum + :samp:`s`-:samp:`t` cut. + + """ + if G.is_multigraph(): + raise nx.NetworkXError("MultiGraph and MultiDiGraph not supported (yet).") + + R = nx.DiGraph() + R.__networkx_cache__ = None # Disable caching + R.add_nodes_from(G) + + inf = float("inf") + # Extract edges with positive capacities. Self loops excluded. + edge_list = [ + (u, v, attr) + for u, v, attr in G.edges(data=True) + if u != v and attr.get(capacity, inf) > 0 + ] + # Simulate infinity with three times the sum of the finite edge capacities + # or any positive value if the sum is zero. This allows the + # infinite-capacity edges to be distinguished for unboundedness detection + # and directly participate in residual capacity calculation. If the maximum + # flow is finite, these edges cannot appear in the minimum cut and thus + # guarantee correctness. Since the residual capacity of an + # infinite-capacity edge is always at least 2/3 of inf, while that of an + # finite-capacity edge is at most 1/3 of inf, if an operation moves more + # than 1/3 of inf units of flow to t, there must be an infinite-capacity + # s-t path in G. + inf = ( + 3 + * sum( + attr[capacity] + for u, v, attr in edge_list + if capacity in attr and attr[capacity] != inf + ) + or 1 + ) + if G.is_directed(): + for u, v, attr in edge_list: + r = min(attr.get(capacity, inf), inf) + if not R.has_edge(u, v): + # Both (u, v) and (v, u) must be present in the residual + # network. + R.add_edge(u, v, capacity=r) + R.add_edge(v, u, capacity=0) + else: + # The edge (u, v) was added when (v, u) was visited. + R[u][v]["capacity"] = r + else: + for u, v, attr in edge_list: + # Add a pair of edges with equal residual capacities. + r = min(attr.get(capacity, inf), inf) + R.add_edge(u, v, capacity=r) + R.add_edge(v, u, capacity=r) + + # Record the value simulating infinity. + R.graph["inf"] = inf + + return R + + +@nx._dispatchable( + graphs="R", + preserve_edge_attrs={"R": {"capacity": float("inf")}}, + preserve_graph_attrs=True, +) +def detect_unboundedness(R, s, t): + """Detect an infinite-capacity s-t path in R.""" + q = deque([s]) + seen = {s} + inf = R.graph["inf"] + while q: + u = q.popleft() + for v, attr in R[u].items(): + if attr["capacity"] == inf and v not in seen: + if v == t: + raise nx.NetworkXUnbounded( + "Infinite capacity path, flow unbounded above." + ) + seen.add(v) + q.append(v) + + +@nx._dispatchable(graphs={"G": 0, "R": 1}, preserve_edge_attrs={"R": {"flow": None}}) +def build_flow_dict(G, R): + """Build a flow dictionary from a residual network.""" + flow_dict = {} + for u in G: + flow_dict[u] = {v: 0 for v in G[u]} + flow_dict[u].update( + (v, attr["flow"]) for v, attr in R[u].items() if attr["flow"] > 0 + ) + return flow_dict diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/link_analysis/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/link_analysis/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6009f000814753ab436278e2d2cc38e961e80f3f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/link_analysis/__init__.py @@ -0,0 +1,2 @@ +from networkx.algorithms.link_analysis.hits_alg import * +from networkx.algorithms.link_analysis.pagerank_alg import * diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/link_analysis/hits_alg.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/link_analysis/hits_alg.py new file mode 100644 index 0000000000000000000000000000000000000000..3a292f9b2b755d0711f6f8d760f3036736864a74 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/link_analysis/hits_alg.py @@ -0,0 +1,337 @@ +"""Hubs and authorities analysis of graph structure.""" + +import networkx as nx + +__all__ = ["hits"] + + +@nx._dispatchable(preserve_edge_attrs={"G": {"weight": 1}}) +def hits(G, max_iter=100, tol=1.0e-8, nstart=None, normalized=True): + """Returns HITS hubs and authorities values for nodes. + + The HITS algorithm computes two numbers for a node. + Authorities estimates the node value based on the incoming links. + Hubs estimates the node value based on outgoing links. + + Parameters + ---------- + G : graph + A NetworkX graph + + max_iter : integer, optional + Maximum number of iterations in power method. + + tol : float, optional + Error tolerance used to check convergence in power method iteration. + + nstart : dictionary, optional + Starting value of each node for power method iteration. + + normalized : bool (default=True) + Normalize results by the sum of all of the values. + + Returns + ------- + (hubs,authorities) : two-tuple of dictionaries + Two dictionaries keyed by node containing the hub and authority + values. + + Raises + ------ + PowerIterationFailedConvergence + If the algorithm fails to converge to the specified tolerance + within the specified number of iterations of the power iteration + method. + + Examples + -------- + >>> G = nx.path_graph(4) + >>> h, a = nx.hits(G) + + Notes + ----- + The eigenvector calculation is done by the power iteration method + and has no guarantee of convergence. The iteration will stop + after max_iter iterations or an error tolerance of + number_of_nodes(G)*tol has been reached. + + The HITS algorithm was designed for directed graphs but this + algorithm does not check if the input graph is directed and will + execute on undirected graphs. + + References + ---------- + .. [1] A. Langville and C. Meyer, + "A survey of eigenvector methods of web information retrieval." + https://epubs.siam.org/doi/epdf/10.1137/S0036144503424786 + .. [2] Jon Kleinberg, + Authoritative sources in a hyperlinked environment + Journal of the ACM 46 (5): 604-32, 1999. + https://www.cs.cornell.edu/home/kleinber/auth.pdf + doi:10.1145/324133.324140. + """ + import numpy as np + import scipy as sp + + if len(G) == 0: + return {}, {} + A = nx.adjacency_matrix(G, nodelist=list(G), dtype=float) + + if nstart is not None: + nstart = np.array(list(nstart.values())) + if max_iter <= 0: + raise nx.PowerIterationFailedConvergence(max_iter) + try: + _, _, vt = sp.sparse.linalg.svds(A, k=1, v0=nstart, maxiter=max_iter, tol=tol) + except sp.sparse.linalg.ArpackNoConvergence as exc: + raise nx.PowerIterationFailedConvergence(max_iter) from exc + + a = vt.flatten().real + h = A @ a + if normalized: + h /= h.sum() + a /= a.sum() + hubs = dict(zip(G, map(float, h))) + authorities = dict(zip(G, map(float, a))) + return hubs, authorities + + +def _hits_python(G, max_iter=100, tol=1.0e-8, nstart=None, normalized=True): + if isinstance(G, nx.MultiGraph | nx.MultiDiGraph): + raise Exception("hits() not defined for graphs with multiedges.") + if len(G) == 0: + return {}, {} + # choose fixed starting vector if not given + if nstart is None: + h = dict.fromkeys(G, 1.0 / G.number_of_nodes()) + else: + h = nstart + # normalize starting vector + s = 1.0 / sum(h.values()) + for k in h: + h[k] *= s + for _ in range(max_iter): # power iteration: make up to max_iter iterations + hlast = h + h = dict.fromkeys(hlast.keys(), 0) + a = dict.fromkeys(hlast.keys(), 0) + # this "matrix multiply" looks odd because it is + # doing a left multiply a^T=hlast^T*G + for n in h: + for nbr in G[n]: + a[nbr] += hlast[n] * G[n][nbr].get("weight", 1) + # now multiply h=Ga + for n in h: + for nbr in G[n]: + h[n] += a[nbr] * G[n][nbr].get("weight", 1) + # normalize vector + s = 1.0 / max(h.values()) + for n in h: + h[n] *= s + # normalize vector + s = 1.0 / max(a.values()) + for n in a: + a[n] *= s + # check convergence, l1 norm + err = sum(abs(h[n] - hlast[n]) for n in h) + if err < tol: + break + else: + raise nx.PowerIterationFailedConvergence(max_iter) + if normalized: + s = 1.0 / sum(a.values()) + for n in a: + a[n] *= s + s = 1.0 / sum(h.values()) + for n in h: + h[n] *= s + return h, a + + +def _hits_numpy(G, normalized=True): + """Returns HITS hubs and authorities values for nodes. + + The HITS algorithm computes two numbers for a node. + Authorities estimates the node value based on the incoming links. + Hubs estimates the node value based on outgoing links. + + Parameters + ---------- + G : graph + A NetworkX graph + + normalized : bool (default=True) + Normalize results by the sum of all of the values. + + Returns + ------- + (hubs,authorities) : two-tuple of dictionaries + Two dictionaries keyed by node containing the hub and authority + values. + + Examples + -------- + >>> G = nx.path_graph(4) + + The `hubs` and `authorities` are given by the eigenvectors corresponding to the + maximum eigenvalues of the hubs_matrix and the authority_matrix, respectively. + + The ``hubs`` and ``authority`` matrices are computed from the adjacency + matrix: + + >>> adj_ary = nx.to_numpy_array(G) + >>> hubs_matrix = adj_ary @ adj_ary.T + >>> authority_matrix = adj_ary.T @ adj_ary + + `_hits_numpy` maps the eigenvector corresponding to the maximum eigenvalue + of the respective matrices to the nodes in `G`: + + >>> from networkx.algorithms.link_analysis.hits_alg import _hits_numpy + >>> hubs, authority = _hits_numpy(G) + + Notes + ----- + The eigenvector calculation uses NumPy's interface to LAPACK. + + The HITS algorithm was designed for directed graphs but this + algorithm does not check if the input graph is directed and will + execute on undirected graphs. + + References + ---------- + .. [1] A. Langville and C. Meyer, + "A survey of eigenvector methods of web information retrieval." + http://citeseer.ist.psu.edu/713792.html + .. [2] Jon Kleinberg, + Authoritative sources in a hyperlinked environment + Journal of the ACM 46 (5): 604-32, 1999. + doi:10.1145/324133.324140. + http://www.cs.cornell.edu/home/kleinber/auth.pdf. + """ + import numpy as np + + if len(G) == 0: + return {}, {} + adj_ary = nx.to_numpy_array(G) + # Hub matrix + H = adj_ary @ adj_ary.T + e, ev = np.linalg.eig(H) + h = ev[:, np.argmax(e)] # eigenvector corresponding to the maximum eigenvalue + # Authority matrix + A = adj_ary.T @ adj_ary + e, ev = np.linalg.eig(A) + a = ev[:, np.argmax(e)] # eigenvector corresponding to the maximum eigenvalue + if normalized: + h /= h.sum() + a /= a.sum() + else: + h /= h.max() + a /= a.max() + hubs = dict(zip(G, map(float, h))) + authorities = dict(zip(G, map(float, a))) + return hubs, authorities + + +def _hits_scipy(G, max_iter=100, tol=1.0e-6, nstart=None, normalized=True): + """Returns HITS hubs and authorities values for nodes. + + + The HITS algorithm computes two numbers for a node. + Authorities estimates the node value based on the incoming links. + Hubs estimates the node value based on outgoing links. + + Parameters + ---------- + G : graph + A NetworkX graph + + max_iter : integer, optional + Maximum number of iterations in power method. + + tol : float, optional + Error tolerance used to check convergence in power method iteration. + + nstart : dictionary, optional + Starting value of each node for power method iteration. + + normalized : bool (default=True) + Normalize results by the sum of all of the values. + + Returns + ------- + (hubs,authorities) : two-tuple of dictionaries + Two dictionaries keyed by node containing the hub and authority + values. + + Examples + -------- + >>> from networkx.algorithms.link_analysis.hits_alg import _hits_scipy + >>> G = nx.path_graph(4) + >>> h, a = _hits_scipy(G) + + Notes + ----- + This implementation uses SciPy sparse matrices. + + The eigenvector calculation is done by the power iteration method + and has no guarantee of convergence. The iteration will stop + after max_iter iterations or an error tolerance of + number_of_nodes(G)*tol has been reached. + + The HITS algorithm was designed for directed graphs but this + algorithm does not check if the input graph is directed and will + execute on undirected graphs. + + Raises + ------ + PowerIterationFailedConvergence + If the algorithm fails to converge to the specified tolerance + within the specified number of iterations of the power iteration + method. + + References + ---------- + .. [1] A. Langville and C. Meyer, + "A survey of eigenvector methods of web information retrieval." + http://citeseer.ist.psu.edu/713792.html + .. [2] Jon Kleinberg, + Authoritative sources in a hyperlinked environment + Journal of the ACM 46 (5): 604-632, 1999. + doi:10.1145/324133.324140. + http://www.cs.cornell.edu/home/kleinber/auth.pdf. + """ + import numpy as np + + if len(G) == 0: + return {}, {} + A = nx.to_scipy_sparse_array(G, nodelist=list(G)) + (n, _) = A.shape # should be square + ATA = A.T @ A # authority matrix + # choose fixed starting vector if not given + if nstart is None: + x = np.ones((n, 1)) / n + else: + x = np.array([nstart.get(n, 0) for n in list(G)], dtype=float) + x /= x.sum() + + # power iteration on authority matrix + i = 0 + while True: + xlast = x + x = ATA @ x + x /= x.max() + # check convergence, l1 norm + err = np.absolute(x - xlast).sum() + if err < tol: + break + if i > max_iter: + raise nx.PowerIterationFailedConvergence(max_iter) + i += 1 + + a = x.flatten() + h = A @ a + if normalized: + h /= h.sum() + a /= a.sum() + hubs = dict(zip(G, map(float, h))) + authorities = dict(zip(G, map(float, a))) + return hubs, authorities diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/link_analysis/pagerank_alg.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/link_analysis/pagerank_alg.py new file mode 100644 index 0000000000000000000000000000000000000000..220d17350113e0e11b80f638cf29bfb5afdc7759 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/link_analysis/pagerank_alg.py @@ -0,0 +1,498 @@ +"""PageRank analysis of graph structure.""" + +import networkx as nx + +__all__ = ["pagerank", "google_matrix"] + + +@nx._dispatchable(edge_attrs="weight") +def pagerank( + G, + alpha=0.85, + personalization=None, + max_iter=100, + tol=1.0e-6, + nstart=None, + weight="weight", + dangling=None, +): + """Returns the PageRank of the nodes in the graph. + + PageRank computes a ranking of the nodes in the graph G based on + the structure of the incoming links. It was originally designed as + an algorithm to rank web pages. + + Parameters + ---------- + G : graph + A NetworkX graph. Undirected graphs will be converted to a directed + graph with two directed edges for each undirected edge. + + alpha : float, optional + Damping parameter for PageRank, default=0.85. + + personalization: dict, optional + The "personalization vector" consisting of a dictionary with a + key some subset of graph nodes and personalization value each of those. + At least one personalization value must be non-zero. + If not specified, a nodes personalization value will be zero. + By default, a uniform distribution is used. + + max_iter : integer, optional + Maximum number of iterations in power method eigenvalue solver. + + tol : float, optional + Error tolerance used to check convergence in power method solver. + The iteration will stop after a tolerance of ``len(G) * tol`` is reached. + + nstart : dictionary, optional + Starting value of PageRank iteration for each node. + + weight : key, optional + Edge data key to use as weight. If None weights are set to 1. + + dangling: dict, optional + The outedges to be assigned to any "dangling" nodes, i.e., nodes without + any outedges. The dict key is the node the outedge points to and the dict + value is the weight of that outedge. By default, dangling nodes are given + outedges according to the personalization vector (uniform if not + specified). This must be selected to result in an irreducible transition + matrix (see notes under google_matrix). It may be common to have the + dangling dict to be the same as the personalization dict. + + + Returns + ------- + pagerank : dictionary + Dictionary of nodes with PageRank as value + + Examples + -------- + >>> G = nx.DiGraph(nx.path_graph(4)) + >>> pr = nx.pagerank(G, alpha=0.9) + + Notes + ----- + The eigenvector calculation is done by the power iteration method + and has no guarantee of convergence. The iteration will stop after + an error tolerance of ``len(G) * tol`` has been reached. If the + number of iterations exceed `max_iter`, a + :exc:`networkx.exception.PowerIterationFailedConvergence` exception + is raised. + + The PageRank algorithm was designed for directed graphs but this + algorithm does not check if the input graph is directed and will + execute on undirected graphs by converting each edge in the + directed graph to two edges. + + See Also + -------- + google_matrix + :func:`~networkx.algorithms.bipartite.link_analysis.birank` + + Raises + ------ + PowerIterationFailedConvergence + If the algorithm fails to converge to the specified tolerance + within the specified number of iterations of the power iteration + method. + + References + ---------- + .. [1] A. Langville and C. Meyer, + "A survey of eigenvector methods of web information retrieval." + http://citeseer.ist.psu.edu/713792.html + .. [2] Page, Lawrence; Brin, Sergey; Motwani, Rajeev and Winograd, Terry, + The PageRank citation ranking: Bringing order to the Web. 1999 + http://dbpubs.stanford.edu:8090/pub/showDoc.Fulltext?lang=en&doc=1999-66&format=pdf + + """ + return _pagerank_scipy( + G, alpha, personalization, max_iter, tol, nstart, weight, dangling + ) + + +def _pagerank_python( + G, + alpha=0.85, + personalization=None, + max_iter=100, + tol=1.0e-6, + nstart=None, + weight="weight", + dangling=None, +): + if len(G) == 0: + return {} + + D = G.to_directed() + + # Create a copy in (right) stochastic form + W = nx.stochastic_graph(D, weight=weight) + N = W.number_of_nodes() + + # Choose fixed starting vector if not given + if nstart is None: + x = dict.fromkeys(W, 1.0 / N) + else: + # Normalized nstart vector + s = sum(nstart.values()) + x = {k: v / s for k, v in nstart.items()} + + if personalization is None: + # Assign uniform personalization vector if not given + p = dict.fromkeys(W, 1.0 / N) + else: + s = sum(personalization.values()) + p = {k: v / s for k, v in personalization.items()} + + if dangling is None: + # Use personalization vector if dangling vector not specified + dangling_weights = p + else: + s = sum(dangling.values()) + dangling_weights = {k: v / s for k, v in dangling.items()} + dangling_nodes = [n for n in W if W.out_degree(n, weight=weight) == 0.0] + + # power iteration: make up to max_iter iterations + for _ in range(max_iter): + xlast = x + x = dict.fromkeys(xlast.keys(), 0) + danglesum = alpha * sum(xlast[n] for n in dangling_nodes) + for n in x: + # this matrix multiply looks odd because it is + # doing a left multiply x^T=xlast^T*W + for _, nbr, wt in W.edges(n, data=weight): + x[nbr] += alpha * xlast[n] * wt + x[n] += danglesum * dangling_weights.get(n, 0) + (1.0 - alpha) * p.get(n, 0) + # check convergence, l1 norm + err = sum(abs(x[n] - xlast[n]) for n in x) + if err < N * tol: + return x + raise nx.PowerIterationFailedConvergence(max_iter) + + +@nx._dispatchable(edge_attrs="weight") +def google_matrix( + G, alpha=0.85, personalization=None, nodelist=None, weight="weight", dangling=None +): + """Returns the Google matrix of the graph. + + Parameters + ---------- + G : graph + A NetworkX graph. Undirected graphs will be converted to a directed + graph with two directed edges for each undirected edge. + + alpha : float + The damping factor. + + personalization: dict, optional + The "personalization vector" consisting of a dictionary with a + key some subset of graph nodes and personalization value each of those. + At least one personalization value must be non-zero. + If not specified, a nodes personalization value will be zero. + By default, a uniform distribution is used. + + nodelist : list, optional + The rows and columns are ordered according to the nodes in nodelist. + If nodelist is None, then the ordering is produced by G.nodes(). + + weight : key, optional + Edge data key to use as weight. If None weights are set to 1. + + dangling: dict, optional + The outedges to be assigned to any "dangling" nodes, i.e., nodes without + any outedges. The dict key is the node the outedge points to and the dict + value is the weight of that outedge. By default, dangling nodes are given + outedges according to the personalization vector (uniform if not + specified) This must be selected to result in an irreducible transition + matrix (see notes below). It may be common to have the dangling dict to + be the same as the personalization dict. + + Returns + ------- + A : 2D NumPy ndarray + Google matrix of the graph + + Notes + ----- + The array returned represents the transition matrix that describes the + Markov chain used in PageRank. For PageRank to converge to a unique + solution (i.e., a unique stationary distribution in a Markov chain), the + transition matrix must be irreducible. In other words, it must be that + there exists a path between every pair of nodes in the graph, or else there + is the potential of "rank sinks." + + This implementation works with Multi(Di)Graphs. For multigraphs the + weight between two nodes is set to be the sum of all edge weights + between those nodes. + + See Also + -------- + pagerank + """ + import numpy as np + + if nodelist is None: + nodelist = list(G) + + A = nx.to_numpy_array(G, nodelist=nodelist, weight=weight) + N = len(G) + if N == 0: + return A + + # Personalization vector + if personalization is None: + p = np.repeat(1.0 / N, N) + else: + p = np.array([personalization.get(n, 0) for n in nodelist], dtype=float) + if p.sum() == 0: + raise ZeroDivisionError + p /= p.sum() + + # Dangling nodes + if dangling is None: + dangling_weights = p + else: + # Convert the dangling dictionary into an array in nodelist order + dangling_weights = np.array([dangling.get(n, 0) for n in nodelist], dtype=float) + dangling_weights /= dangling_weights.sum() + dangling_nodes = np.where(A.sum(axis=1) == 0)[0] + + # Assign dangling_weights to any dangling nodes (nodes with no out links) + A[dangling_nodes] = dangling_weights + + A /= A.sum(axis=1)[:, np.newaxis] # Normalize rows to sum to 1 + + return alpha * A + (1 - alpha) * p + + +def _pagerank_numpy( + G, alpha=0.85, personalization=None, weight="weight", dangling=None +): + """Returns the PageRank of the nodes in the graph. + + PageRank computes a ranking of the nodes in the graph G based on + the structure of the incoming links. It was originally designed as + an algorithm to rank web pages. + + Parameters + ---------- + G : graph + A NetworkX graph. Undirected graphs will be converted to a directed + graph with two directed edges for each undirected edge. + + alpha : float, optional + Damping parameter for PageRank, default=0.85. + + personalization: dict, optional + The "personalization vector" consisting of a dictionary with a + key some subset of graph nodes and personalization value each of those. + At least one personalization value must be non-zero. + If not specified, a nodes personalization value will be zero. + By default, a uniform distribution is used. + + weight : key, optional + Edge data key to use as weight. If None weights are set to 1. + + dangling: dict, optional + The outedges to be assigned to any "dangling" nodes, i.e., nodes without + any outedges. The dict key is the node the outedge points to and the dict + value is the weight of that outedge. By default, dangling nodes are given + outedges according to the personalization vector (uniform if not + specified) This must be selected to result in an irreducible transition + matrix (see notes under google_matrix). It may be common to have the + dangling dict to be the same as the personalization dict. + + Returns + ------- + pagerank : dictionary + Dictionary of nodes with PageRank as value. + + Examples + -------- + >>> from networkx.algorithms.link_analysis.pagerank_alg import _pagerank_numpy + >>> G = nx.DiGraph(nx.path_graph(4)) + >>> pr = _pagerank_numpy(G, alpha=0.9) + + Notes + ----- + The eigenvector calculation uses NumPy's interface to the LAPACK + eigenvalue solvers. This will be the fastest and most accurate + for small graphs. + + This implementation works with Multi(Di)Graphs. For multigraphs the + weight between two nodes is set to be the sum of all edge weights + between those nodes. + + See Also + -------- + pagerank, google_matrix + + References + ---------- + .. [1] A. Langville and C. Meyer, + "A survey of eigenvector methods of web information retrieval." + http://citeseer.ist.psu.edu/713792.html + .. [2] Page, Lawrence; Brin, Sergey; Motwani, Rajeev and Winograd, Terry, + The PageRank citation ranking: Bringing order to the Web. 1999 + http://dbpubs.stanford.edu:8090/pub/showDoc.Fulltext?lang=en&doc=1999-66&format=pdf + """ + import numpy as np + + if len(G) == 0: + return {} + M = google_matrix( + G, alpha, personalization=personalization, weight=weight, dangling=dangling + ) + # use numpy LAPACK solver + eigenvalues, eigenvectors = np.linalg.eig(M.T) + ind = np.argmax(eigenvalues) + # eigenvector of largest eigenvalue is at ind, normalized + largest = np.array(eigenvectors[:, ind]).flatten().real + norm = largest.sum() + return dict(zip(G, map(float, largest / norm))) + + +def _pagerank_scipy( + G, + alpha=0.85, + personalization=None, + max_iter=100, + tol=1.0e-6, + nstart=None, + weight="weight", + dangling=None, +): + """Returns the PageRank of the nodes in the graph. + + PageRank computes a ranking of the nodes in the graph G based on + the structure of the incoming links. It was originally designed as + an algorithm to rank web pages. + + Parameters + ---------- + G : graph + A NetworkX graph. Undirected graphs will be converted to a directed + graph with two directed edges for each undirected edge. + + alpha : float, optional + Damping parameter for PageRank, default=0.85. + + personalization: dict, optional + The "personalization vector" consisting of a dictionary with a + key some subset of graph nodes and personalization value each of those. + At least one personalization value must be non-zero. + If not specified, a nodes personalization value will be zero. + By default, a uniform distribution is used. + + max_iter : integer, optional + Maximum number of iterations in power method eigenvalue solver. + + tol : float, optional + Error tolerance used to check convergence in power method solver. + The iteration will stop after a tolerance of ``len(G) * tol`` is reached. + + nstart : dictionary, optional + Starting value of PageRank iteration for each node. + + weight : key, optional + Edge data key to use as weight. If None weights are set to 1. + + dangling: dict, optional + The outedges to be assigned to any "dangling" nodes, i.e., nodes without + any outedges. The dict key is the node the outedge points to and the dict + value is the weight of that outedge. By default, dangling nodes are given + outedges according to the personalization vector (uniform if not + specified) This must be selected to result in an irreducible transition + matrix (see notes under google_matrix). It may be common to have the + dangling dict to be the same as the personalization dict. + + Returns + ------- + pagerank : dictionary + Dictionary of nodes with PageRank as value + + Examples + -------- + >>> from networkx.algorithms.link_analysis.pagerank_alg import _pagerank_scipy + >>> G = nx.DiGraph(nx.path_graph(4)) + >>> pr = _pagerank_scipy(G, alpha=0.9) + + Notes + ----- + The eigenvector calculation uses power iteration with a SciPy + sparse matrix representation. + + This implementation works with Multi(Di)Graphs. For multigraphs the + weight between two nodes is set to be the sum of all edge weights + between those nodes. + + See Also + -------- + pagerank + + Raises + ------ + PowerIterationFailedConvergence + If the algorithm fails to converge to the specified tolerance + within the specified number of iterations of the power iteration + method. + + References + ---------- + .. [1] A. Langville and C. Meyer, + "A survey of eigenvector methods of web information retrieval." + http://citeseer.ist.psu.edu/713792.html + .. [2] Page, Lawrence; Brin, Sergey; Motwani, Rajeev and Winograd, Terry, + The PageRank citation ranking: Bringing order to the Web. 1999 + http://dbpubs.stanford.edu:8090/pub/showDoc.Fulltext?lang=en&doc=1999-66&format=pdf + """ + import numpy as np + import scipy as sp + + N = len(G) + if N == 0: + return {} + + nodelist = list(G) + A = nx.to_scipy_sparse_array(G, nodelist=nodelist, weight=weight, dtype=float) + S = A.sum(axis=1) + S[S != 0] = 1.0 / S[S != 0] + Q = sp.sparse.dia_array((S.T, 0), shape=A.shape).tocsr() + A = Q @ A + + # initial vector + if nstart is None: + x = np.repeat(1.0 / N, N) + else: + x = np.array([nstart.get(n, 0) for n in nodelist], dtype=float) + x /= x.sum() + + # Personalization vector + if personalization is None: + p = np.repeat(1.0 / N, N) + else: + p = np.array([personalization.get(n, 0) for n in nodelist], dtype=float) + if p.sum() == 0: + raise ZeroDivisionError + p /= p.sum() + # Dangling nodes + if dangling is None: + dangling_weights = p + else: + # Convert the dangling dictionary into an array in nodelist order + dangling_weights = np.array([dangling.get(n, 0) for n in nodelist], dtype=float) + dangling_weights /= dangling_weights.sum() + is_dangling = np.where(S == 0)[0] + + # power iteration: make up to max_iter iterations + for _ in range(max_iter): + xlast = x + x = alpha * (x @ A + sum(x[is_dangling]) * dangling_weights) + (1 - alpha) * p + # check convergence, l1 norm + err = np.absolute(x - xlast).sum() + if err < N * tol: + return dict(zip(nodelist, map(float, x))) + raise nx.PowerIterationFailedConvergence(max_iter) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/shortest_paths/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/shortest_paths/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..eb0d91cecc902f6390cb8309c017cb1558f7753f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/shortest_paths/__init__.py @@ -0,0 +1,5 @@ +from networkx.algorithms.shortest_paths.generic import * +from networkx.algorithms.shortest_paths.unweighted import * +from networkx.algorithms.shortest_paths.weighted import * +from networkx.algorithms.shortest_paths.astar import * +from networkx.algorithms.shortest_paths.dense import * diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/shortest_paths/astar.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/shortest_paths/astar.py new file mode 100644 index 0000000000000000000000000000000000000000..118229716cdfd5fe3a45619010437ec0df502d3d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/shortest_paths/astar.py @@ -0,0 +1,239 @@ +"""Shortest paths and path lengths using the A* ("A star") algorithm.""" + +from heapq import heappop, heappush +from itertools import count + +import networkx as nx +from networkx.algorithms.shortest_paths.weighted import _weight_function + +__all__ = ["astar_path", "astar_path_length"] + + +@nx._dispatchable(edge_attrs="weight", preserve_node_attrs="heuristic") +def astar_path(G, source, target, heuristic=None, weight="weight", *, cutoff=None): + """Returns a list of nodes in a shortest path between source and target + using the A* ("A-star") algorithm. + + There may be more than one shortest path. This returns only one. + + Parameters + ---------- + G : NetworkX graph + + source : node + Starting node for path + + target : node + Ending node for path + + heuristic : function + A function to evaluate the estimate of the distance + from the a node to the target. The function takes + two nodes arguments and must return a number. + If the heuristic is inadmissible (if it might + overestimate the cost of reaching the goal from a node), + the result may not be a shortest path. + The algorithm does not support updating heuristic + values for the same node due to caching the first + heuristic calculation per node. + + weight : string or function + If this is a string, then edge weights will be accessed via the + edge attribute with this key (that is, the weight of the edge + joining `u` to `v` will be ``G.edges[u, v][weight]``). If no + such edge attribute exists, the weight of the edge is assumed to + be one. + If this is a function, the weight of an edge is the value + returned by the function. The function must accept exactly three + positional arguments: the two endpoints of an edge and the + dictionary of edge attributes for that edge. The function must + return a number or None to indicate a hidden edge. + + cutoff : float, optional + If this is provided, the search will be bounded to this value. I.e. if + the evaluation function surpasses this value for a node n, the node will not + be expanded further and will be ignored. More formally, let h'(n) be the + heuristic function, and g(n) be the cost of reaching n from the source node. Then, + if g(n) + h'(n) > cutoff, the node will not be explored further. + Note that if the heuristic is inadmissible, it is possible that paths + are ignored even though they satisfy the cutoff. + + Raises + ------ + NetworkXNoPath + If no path exists between source and target. + + Examples + -------- + >>> G = nx.path_graph(5) + >>> print(nx.astar_path(G, 0, 4)) + [0, 1, 2, 3, 4] + >>> G = nx.grid_graph(dim=[3, 3]) # nodes are two-tuples (x,y) + >>> nx.set_edge_attributes(G, {e: e[1][0] * 2 for e in G.edges()}, "cost") + >>> def dist(a, b): + ... (x1, y1) = a + ... (x2, y2) = b + ... return ((x1 - x2) ** 2 + (y1 - y2) ** 2) ** 0.5 + >>> print(nx.astar_path(G, (0, 0), (2, 2), heuristic=dist, weight="cost")) + [(0, 0), (0, 1), (0, 2), (1, 2), (2, 2)] + + Notes + ----- + Edge weight attributes must be numerical. + Distances are calculated as sums of weighted edges traversed. + + The weight function can be used to hide edges by returning None. + So ``weight = lambda u, v, d: 1 if d['color']=="red" else None`` + will find the shortest red path. + + See Also + -------- + shortest_path, dijkstra_path + + """ + if source not in G: + raise nx.NodeNotFound(f"Source {source} is not in G") + + if target not in G: + raise nx.NodeNotFound(f"Target {target} is not in G") + + if heuristic is None: + # The default heuristic is h=0 - same as Dijkstra's algorithm + def heuristic(u, v): + return 0 + + weight = _weight_function(G, weight) + + G_succ = G._adj # For speed-up (and works for both directed and undirected graphs) + + # The queue stores priority, node, cost to reach, and parent. + # Uses Python heapq to keep in priority order. + # Add a counter to the queue to prevent the underlying heap from + # attempting to compare the nodes themselves. The hash breaks ties in the + # priority and is guaranteed unique for all nodes in the graph. + c = count() + queue = [(0, next(c), source, 0, None)] + + # Maps enqueued nodes to distance of discovered paths and the + # computed heuristics to target. We avoid computing the heuristics + # more than once and inserting the node into the queue too many times. + enqueued = {} + # Maps explored nodes to parent closest to the source. + explored = {} + + while queue: + # Pop the smallest item from queue. + _, __, curnode, dist, parent = heappop(queue) + + if curnode == target: + path = [curnode] + node = parent + while node is not None: + path.append(node) + node = explored[node] + path.reverse() + return path + + if curnode in explored: + # Do not override the parent of starting node + if explored[curnode] is None: + continue + + # Skip bad paths that were enqueued before finding a better one + qcost, h = enqueued[curnode] + if qcost < dist: + continue + + explored[curnode] = parent + + for neighbor, w in G_succ[curnode].items(): + cost = weight(curnode, neighbor, w) + if cost is None: + continue + ncost = dist + cost + if neighbor in enqueued: + qcost, h = enqueued[neighbor] + # if qcost <= ncost, a less costly path from the + # neighbor to the source was already determined. + # Therefore, we won't attempt to push this neighbor + # to the queue + if qcost <= ncost: + continue + else: + h = heuristic(neighbor, target) + + if cutoff and ncost + h > cutoff: + continue + + enqueued[neighbor] = ncost, h + heappush(queue, (ncost + h, next(c), neighbor, ncost, curnode)) + + raise nx.NetworkXNoPath(f"Node {target} not reachable from {source}") + + +@nx._dispatchable(edge_attrs="weight", preserve_node_attrs="heuristic") +def astar_path_length( + G, source, target, heuristic=None, weight="weight", *, cutoff=None +): + """Returns the length of the shortest path between source and target using + the A* ("A-star") algorithm. + + Parameters + ---------- + G : NetworkX graph + + source : node + Starting node for path + + target : node + Ending node for path + + heuristic : function + A function to evaluate the estimate of the distance + from the a node to the target. The function takes + two nodes arguments and must return a number. + If the heuristic is inadmissible (if it might + overestimate the cost of reaching the goal from a node), + the result may not be a shortest path. + The algorithm does not support updating heuristic + values for the same node due to caching the first + heuristic calculation per node. + + weight : string or function + If this is a string, then edge weights will be accessed via the + edge attribute with this key (that is, the weight of the edge + joining `u` to `v` will be ``G.edges[u, v][weight]``). If no + such edge attribute exists, the weight of the edge is assumed to + be one. + If this is a function, the weight of an edge is the value + returned by the function. The function must accept exactly three + positional arguments: the two endpoints of an edge and the + dictionary of edge attributes for that edge. The function must + return a number or None to indicate a hidden edge. + + cutoff : float, optional + If this is provided, the search will be bounded to this value. I.e. if + the evaluation function surpasses this value for a node n, the node will not + be expanded further and will be ignored. More formally, let h'(n) be the + heuristic function, and g(n) be the cost of reaching n from the source node. Then, + if g(n) + h'(n) > cutoff, the node will not be explored further. + Note that if the heuristic is inadmissible, it is possible that paths + are ignored even though they satisfy the cutoff. + + Raises + ------ + NetworkXNoPath + If no path exists between source and target. + + See Also + -------- + astar_path + + """ + if source not in G or target not in G: + msg = f"Either source {source} or target {target} is not in G" + raise nx.NodeNotFound(msg) + + weight = _weight_function(G, weight) + path = astar_path(G, source, target, heuristic, weight, cutoff=cutoff) + return sum(weight(u, v, G[u][v]) for u, v in zip(path[:-1], path[1:])) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/shortest_paths/dense.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/shortest_paths/dense.py new file mode 100644 index 0000000000000000000000000000000000000000..d259d8c2710dfeb0c299d8ec8e46677b1f27606d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/shortest_paths/dense.py @@ -0,0 +1,264 @@ +"""Floyd-Warshall algorithm for shortest paths.""" + +import networkx as nx + +__all__ = [ + "floyd_warshall", + "floyd_warshall_predecessor_and_distance", + "reconstruct_path", + "floyd_warshall_numpy", +] + + +@nx._dispatchable(edge_attrs="weight") +def floyd_warshall_numpy(G, nodelist=None, weight="weight"): + """Find all-pairs shortest path lengths using Floyd's algorithm. + + This algorithm for finding shortest paths takes advantage of + matrix representations of a graph and works well for dense + graphs where all-pairs shortest path lengths are desired. + The results are returned as a NumPy array, distance[i, j], + where i and j are the indexes of two nodes in nodelist. + The entry distance[i, j] is the distance along a shortest + path from i to j. If no path exists the distance is Inf. + + Parameters + ---------- + G : NetworkX graph + + nodelist : list, optional (default=G.nodes) + The rows and columns are ordered by the nodes in nodelist. + If nodelist is None then the ordering is produced by G.nodes. + Nodelist should include all nodes in G. + + weight: string, optional (default='weight') + Edge data key corresponding to the edge weight. + + Returns + ------- + distance : 2D numpy.ndarray + A numpy array of shortest path distances between nodes. + If there is no path between two nodes the value is Inf. + + Examples + -------- + >>> G = nx.DiGraph() + >>> G.add_weighted_edges_from( + ... [(0, 1, 5), (1, 2, 2), (2, 3, -3), (1, 3, 10), (3, 2, 8)] + ... ) + >>> nx.floyd_warshall_numpy(G) + array([[ 0., 5., 7., 4.], + [inf, 0., 2., -1.], + [inf, inf, 0., -3.], + [inf, inf, 8., 0.]]) + + Notes + ----- + Floyd's algorithm is appropriate for finding shortest paths in + dense graphs or graphs with negative weights when Dijkstra's + algorithm fails. This algorithm can still fail if there are negative + cycles. It has running time $O(n^3)$ with running space of $O(n^2)$. + + Raises + ------ + NetworkXError + If nodelist is not a list of the nodes in G. + """ + import numpy as np + + if nodelist is not None: + if not (len(nodelist) == len(G) == len(set(nodelist))): + raise nx.NetworkXError( + "nodelist must contain every node in G with no repeats." + "If you wanted a subgraph of G use G.subgraph(nodelist)" + ) + + # To handle cases when an edge has weight=0, we must make sure that + # nonedges are not given the value 0 as well. + A = nx.to_numpy_array( + G, nodelist, multigraph_weight=min, weight=weight, nonedge=np.inf + ) + n, m = A.shape + np.fill_diagonal(A, 0) # diagonal elements should be zero + for i in range(n): + # The second term has the same shape as A due to broadcasting + A = np.minimum(A, A[i, :][np.newaxis, :] + A[:, i][:, np.newaxis]) + return A + + +@nx._dispatchable(edge_attrs="weight") +def floyd_warshall_predecessor_and_distance(G, weight="weight"): + """Find all-pairs shortest path lengths using Floyd's algorithm. + + Parameters + ---------- + G : NetworkX graph + + weight: string, optional (default= 'weight') + Edge data key corresponding to the edge weight. + + Returns + ------- + predecessor,distance : dictionaries + Dictionaries, keyed by source and target, of predecessors and distances + in the shortest path. + + Examples + -------- + >>> G = nx.DiGraph() + >>> G.add_weighted_edges_from( + ... [ + ... ("s", "u", 10), + ... ("s", "x", 5), + ... ("u", "v", 1), + ... ("u", "x", 2), + ... ("v", "y", 1), + ... ("x", "u", 3), + ... ("x", "v", 5), + ... ("x", "y", 2), + ... ("y", "s", 7), + ... ("y", "v", 6), + ... ] + ... ) + >>> predecessors, _ = nx.floyd_warshall_predecessor_and_distance(G) + >>> print(nx.reconstruct_path("s", "v", predecessors)) + ['s', 'x', 'u', 'v'] + + Notes + ----- + Floyd's algorithm is appropriate for finding shortest paths + in dense graphs or graphs with negative weights when Dijkstra's algorithm + fails. This algorithm can still fail if there are negative cycles. + It has running time $O(n^3)$ with running space of $O(n^2)$. + + See Also + -------- + floyd_warshall + floyd_warshall_numpy + all_pairs_shortest_path + all_pairs_shortest_path_length + """ + from collections import defaultdict + + # dictionary-of-dictionaries representation for dist and pred + # use some defaultdict magick here + # for dist the default is the floating point inf value + dist = defaultdict(lambda: defaultdict(lambda: float("inf"))) + for u in G: + dist[u][u] = 0 + pred = defaultdict(dict) + # initialize path distance dictionary to be the adjacency matrix + # also set the distance to self to 0 (zero diagonal) + undirected = not G.is_directed() + for u, v, d in G.edges(data=True): + e_weight = d.get(weight, 1.0) + dist[u][v] = min(e_weight, dist[u][v]) + pred[u][v] = u + if undirected: + dist[v][u] = min(e_weight, dist[v][u]) + pred[v][u] = v + for w in G: + dist_w = dist[w] # save recomputation + for u in G: + dist_u = dist[u] # save recomputation + for v in G: + d = dist_u[w] + dist_w[v] + if dist_u[v] > d: + dist_u[v] = d + pred[u][v] = pred[w][v] + return dict(pred), dict(dist) + + +@nx._dispatchable(graphs=None) +def reconstruct_path(source, target, predecessors): + """Reconstruct a path from source to target using the predecessors + dict as returned by floyd_warshall_predecessor_and_distance + + Parameters + ---------- + source : node + Starting node for path + + target : node + Ending node for path + + predecessors: dictionary + Dictionary, keyed by source and target, of predecessors in the + shortest path, as returned by floyd_warshall_predecessor_and_distance + + Returns + ------- + path : list + A list of nodes containing the shortest path from source to target + + If source and target are the same, an empty list is returned + + Notes + ----- + This function is meant to give more applicability to the + floyd_warshall_predecessor_and_distance function + + See Also + -------- + floyd_warshall_predecessor_and_distance + """ + if source == target: + return [] + prev = predecessors[source] + curr = prev[target] + path = [target, curr] + while curr != source: + curr = prev[curr] + path.append(curr) + return list(reversed(path)) + + +@nx._dispatchable(edge_attrs="weight") +def floyd_warshall(G, weight="weight"): + """Find all-pairs shortest path lengths using Floyd's algorithm. + + Parameters + ---------- + G : NetworkX graph + + weight: string, optional (default= 'weight') + Edge data key corresponding to the edge weight. + + + Returns + ------- + distance : dict + A dictionary, keyed by source and target, of shortest paths distances + between nodes. + + Examples + -------- + >>> from pprint import pprint + >>> G = nx.DiGraph() + >>> G.add_weighted_edges_from( + ... [(0, 1, 5), (1, 2, 2), (2, 3, -3), (1, 3, 10), (3, 2, 8)] + ... ) + >>> fw = nx.floyd_warshall(G, weight="weight") + >>> results = {a: dict(b) for a, b in fw.items()} + >>> pprint(results) + {0: {0: 0, 1: 5, 2: 7, 3: 4}, + 1: {0: inf, 1: 0, 2: 2, 3: -1}, + 2: {0: inf, 1: inf, 2: 0, 3: -3}, + 3: {0: inf, 1: inf, 2: 8, 3: 0}} + + Notes + ----- + Floyd's algorithm is appropriate for finding shortest paths + in dense graphs or graphs with negative weights when Dijkstra's algorithm + fails. This algorithm can still fail if there are negative cycles. + It has running time $O(n^3)$ with running space of $O(n^2)$. + + See Also + -------- + floyd_warshall_predecessor_and_distance + floyd_warshall_numpy + all_pairs_shortest_path + all_pairs_shortest_path_length + """ + # could make this its own function to reduce memory costs + return floyd_warshall_predecessor_and_distance(G, weight=weight)[1] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/shortest_paths/generic.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/shortest_paths/generic.py new file mode 100644 index 0000000000000000000000000000000000000000..1fd0eafdb1b30de7cc501144807f4c77ac8a7a08 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/shortest_paths/generic.py @@ -0,0 +1,716 @@ +""" +Compute the shortest paths and path lengths between nodes in the graph. + +These algorithms work with undirected and directed graphs. + +""" + +import networkx as nx + +__all__ = [ + "shortest_path", + "all_shortest_paths", + "single_source_all_shortest_paths", + "all_pairs_all_shortest_paths", + "shortest_path_length", + "average_shortest_path_length", + "has_path", +] + + +@nx._dispatchable +def has_path(G, source, target): + """Returns *True* if *G* has a path from *source* to *target*. + + Parameters + ---------- + G : NetworkX graph + + source : node + Starting node for path + + target : node + Ending node for path + """ + try: + nx.shortest_path(G, source, target) + except nx.NetworkXNoPath: + return False + return True + + +@nx._dispatchable(edge_attrs="weight") +def shortest_path(G, source=None, target=None, weight=None, method="dijkstra"): + """Compute shortest paths in the graph. + + Parameters + ---------- + G : NetworkX graph + + source : node, optional + Starting node for path. If not specified, compute shortest + paths for each possible starting node. + + target : node, optional + Ending node for path. If not specified, compute shortest + paths to all possible nodes. + + weight : None, string or function, optional (default = None) + If None, every edge has weight/distance/cost 1. + If a string, use this edge attribute as the edge weight. + Any edge attribute not present defaults to 1. + If this is a function, the weight of an edge is the value + returned by the function. The function must accept exactly + three positional arguments: the two endpoints of an edge and + the dictionary of edge attributes for that edge. + The function must return a number. + + method : string, optional (default = 'dijkstra') + The algorithm to use to compute the path. + Supported options: 'dijkstra', 'bellman-ford'. + Other inputs produce a ValueError. + If `weight` is None, unweighted graph methods are used, and this + suggestion is ignored. + + Returns + ------- + path: list or dictionary or iterator + All returned paths include both the source and target in the path. + + If the source and target are both specified, return a single list + of nodes in a shortest path from the source to the target. + + If only the source is specified, return a dictionary keyed by + targets with a list of nodes in a shortest path from the source + to one of the targets. + + If only the target is specified, return a dictionary keyed by + sources with a list of nodes in a shortest path from one of the + sources to the target. + + If neither the source nor target are specified, return an iterator + over (source, dictionary) where dictionary is keyed by target to + list of nodes in a shortest path from the source to the target. + + Raises + ------ + NodeNotFound + If `source` is not in `G`. + + ValueError + If `method` is not among the supported options. + + NetworkXNoPath + If `source` and `target` are specified but no path exists between them. + + Examples + -------- + >>> G = nx.path_graph(5) + >>> print(nx.shortest_path(G, source=0, target=4)) + [0, 1, 2, 3, 4] + >>> p = nx.shortest_path(G, source=0) # target not specified + >>> p[3] # shortest path from source=0 to target=3 + [0, 1, 2, 3] + >>> p = nx.shortest_path(G, target=4) # source not specified + >>> p[1] # shortest path from source=1 to target=4 + [1, 2, 3, 4] + >>> p = dict(nx.shortest_path(G)) # source, target not specified + >>> p[2][4] # shortest path from source=2 to target=4 + [2, 3, 4] + + Notes + ----- + There may be more than one shortest path between a source and target. + This returns only one of them. + + See Also + -------- + all_pairs_shortest_path + all_pairs_dijkstra_path + all_pairs_bellman_ford_path + single_source_shortest_path + single_source_dijkstra_path + single_source_bellman_ford_path + """ + if method not in ("dijkstra", "bellman-ford"): + # so we don't need to check in each branch later + raise ValueError(f"method not supported: {method}") + method = "unweighted" if weight is None else method + if source is None: + if target is None: + # Find paths between all pairs. Iterator of dicts. + if method == "unweighted": + paths = nx.all_pairs_shortest_path(G) + elif method == "dijkstra": + paths = nx.all_pairs_dijkstra_path(G, weight=weight) + else: # method == 'bellman-ford': + paths = nx.all_pairs_bellman_ford_path(G, weight=weight) + else: + # Find paths from all nodes co-accessible to the target. + if G.is_directed(): + G = G.reverse(copy=False) + if method == "unweighted": + paths = nx.single_source_shortest_path(G, target) + elif method == "dijkstra": + paths = nx.single_source_dijkstra_path(G, target, weight=weight) + else: # method == 'bellman-ford': + paths = nx.single_source_bellman_ford_path(G, target, weight=weight) + # Now flip the paths so they go from a source to the target. + for target in paths: + paths[target] = list(reversed(paths[target])) + else: + if target is None: + # Find paths to all nodes accessible from the source. + if method == "unweighted": + paths = nx.single_source_shortest_path(G, source) + elif method == "dijkstra": + paths = nx.single_source_dijkstra_path(G, source, weight=weight) + else: # method == 'bellman-ford': + paths = nx.single_source_bellman_ford_path(G, source, weight=weight) + else: + # Find shortest source-target path. + if method == "unweighted": + paths = nx.bidirectional_shortest_path(G, source, target) + elif method == "dijkstra": + _, paths = nx.bidirectional_dijkstra(G, source, target, weight) + else: # method == 'bellman-ford': + paths = nx.bellman_ford_path(G, source, target, weight) + return paths + + +@nx._dispatchable(edge_attrs="weight") +def shortest_path_length(G, source=None, target=None, weight=None, method="dijkstra"): + """Compute shortest path lengths in the graph. + + Parameters + ---------- + G : NetworkX graph + + source : node, optional + Starting node for path. + If not specified, compute shortest path lengths using all nodes as + source nodes. + + target : node, optional + Ending node for path. + If not specified, compute shortest path lengths using all nodes as + target nodes. + + weight : None, string or function, optional (default = None) + If None, every edge has weight/distance/cost 1. + If a string, use this edge attribute as the edge weight. + Any edge attribute not present defaults to 1. + If this is a function, the weight of an edge is the value + returned by the function. The function must accept exactly + three positional arguments: the two endpoints of an edge and + the dictionary of edge attributes for that edge. + The function must return a number. + + method : string, optional (default = 'dijkstra') + The algorithm to use to compute the path length. + Supported options: 'dijkstra', 'bellman-ford'. + Other inputs produce a ValueError. + If `weight` is None, unweighted graph methods are used, and this + suggestion is ignored. + + Returns + ------- + length: number or iterator + If the source and target are both specified, return the length of + the shortest path from the source to the target. + + If only the source is specified, return a dict keyed by target + to the shortest path length from the source to that target. + + If only the target is specified, return a dict keyed by source + to the shortest path length from that source to the target. + + If neither the source nor target are specified, return an iterator + over (source, dictionary) where dictionary is keyed by target to + shortest path length from source to that target. + + Raises + ------ + NodeNotFound + If `source` is not in `G`. + + NetworkXNoPath + If no path exists between source and target. + + ValueError + If `method` is not among the supported options. + + Examples + -------- + >>> G = nx.path_graph(5) + >>> nx.shortest_path_length(G, source=0, target=4) + 4 + >>> p = nx.shortest_path_length(G, source=0) # target not specified + >>> p[4] + 4 + >>> p = nx.shortest_path_length(G, target=4) # source not specified + >>> p[0] + 4 + >>> p = dict(nx.shortest_path_length(G)) # source,target not specified + >>> p[0][4] + 4 + + Notes + ----- + The length of the path is always 1 less than the number of nodes involved + in the path since the length measures the number of edges followed. + + For digraphs this returns the shortest directed path length. To find path + lengths in the reverse direction use G.reverse(copy=False) first to flip + the edge orientation. + + See Also + -------- + all_pairs_shortest_path_length + all_pairs_dijkstra_path_length + all_pairs_bellman_ford_path_length + single_source_shortest_path_length + single_source_dijkstra_path_length + single_source_bellman_ford_path_length + """ + if method not in ("dijkstra", "bellman-ford"): + # so we don't need to check in each branch later + raise ValueError(f"method not supported: {method}") + method = "unweighted" if weight is None else method + if source is None: + if target is None: + # Find paths between all pairs. + if method == "unweighted": + paths = nx.all_pairs_shortest_path_length(G) + elif method == "dijkstra": + paths = nx.all_pairs_dijkstra_path_length(G, weight=weight) + else: # method == 'bellman-ford': + paths = nx.all_pairs_bellman_ford_path_length(G, weight=weight) + else: + # Find paths from all nodes co-accessible to the target. + if G.is_directed(): + G = G.reverse(copy=False) + if method == "unweighted": + path_length = nx.single_source_shortest_path_length + paths = path_length(G, target) + elif method == "dijkstra": + path_length = nx.single_source_dijkstra_path_length + paths = path_length(G, target, weight=weight) + else: # method == 'bellman-ford': + path_length = nx.single_source_bellman_ford_path_length + paths = path_length(G, target, weight=weight) + else: + if target is None: + # Find paths to all nodes accessible from the source. + if method == "unweighted": + paths = nx.single_source_shortest_path_length(G, source) + elif method == "dijkstra": + path_length = nx.single_source_dijkstra_path_length + paths = path_length(G, source, weight=weight) + else: # method == 'bellman-ford': + path_length = nx.single_source_bellman_ford_path_length + paths = path_length(G, source, weight=weight) + else: + # Find shortest source-target path. + if method == "unweighted": + p = nx.bidirectional_shortest_path(G, source, target) + paths = len(p) - 1 + elif method == "dijkstra": + paths = nx.dijkstra_path_length(G, source, target, weight) + else: # method == 'bellman-ford': + paths = nx.bellman_ford_path_length(G, source, target, weight) + return paths + + +@nx._dispatchable(edge_attrs="weight") +def average_shortest_path_length(G, weight=None, method=None): + r"""Returns the average shortest path length. + + The average shortest path length is + + .. math:: + + a =\sum_{\substack{s,t \in V \\ s\neq t}} \frac{d(s, t)}{n(n-1)} + + where `V` is the set of nodes in `G`, + `d(s, t)` is the shortest path from `s` to `t`, + and `n` is the number of nodes in `G`. + + .. versionchanged:: 3.0 + An exception is raised for directed graphs that are not strongly + connected. + + Parameters + ---------- + G : NetworkX graph + + weight : None, string or function, optional (default = None) + If None, every edge has weight/distance/cost 1. + If a string, use this edge attribute as the edge weight. + Any edge attribute not present defaults to 1. + If this is a function, the weight of an edge is the value + returned by the function. The function must accept exactly + three positional arguments: the two endpoints of an edge and + the dictionary of edge attributes for that edge. + The function must return a number. + + method : string, optional (default = 'unweighted' or 'dijkstra') + The algorithm to use to compute the path lengths. + Supported options are 'unweighted', 'dijkstra', 'bellman-ford', + 'floyd-warshall' and 'floyd-warshall-numpy'. + Other method values produce a ValueError. + The default method is 'unweighted' if `weight` is None, + otherwise the default method is 'dijkstra'. + + Raises + ------ + NetworkXPointlessConcept + If `G` is the null graph (that is, the graph on zero nodes). + + NetworkXError + If `G` is not connected (or not strongly connected, in the case + of a directed graph). + + ValueError + If `method` is not among the supported options. + + Examples + -------- + >>> G = nx.path_graph(5) + >>> nx.average_shortest_path_length(G) + 2.0 + + For disconnected graphs, you can compute the average shortest path + length for each component + + >>> G = nx.Graph([(1, 2), (3, 4)]) + >>> for C in (G.subgraph(c).copy() for c in nx.connected_components(G)): + ... print(nx.average_shortest_path_length(C)) + 1.0 + 1.0 + + """ + single_source_methods = ["unweighted", "dijkstra", "bellman-ford"] + all_pairs_methods = ["floyd-warshall", "floyd-warshall-numpy"] + supported_methods = single_source_methods + all_pairs_methods + + if method is None: + method = "unweighted" if weight is None else "dijkstra" + if method not in supported_methods: + raise ValueError(f"method not supported: {method}") + + n = len(G) + # For the special case of the null graph, raise an exception, since + # there are no paths in the null graph. + if n == 0: + msg = ( + "the null graph has no paths, thus there is no average shortest path length" + ) + raise nx.NetworkXPointlessConcept(msg) + # For the special case of the trivial graph, return zero immediately. + if n == 1: + return 0 + # Shortest path length is undefined if the graph is not strongly connected. + if G.is_directed() and not nx.is_strongly_connected(G): + raise nx.NetworkXError("Graph is not strongly connected.") + # Shortest path length is undefined if the graph is not connected. + if not G.is_directed() and not nx.is_connected(G): + raise nx.NetworkXError("Graph is not connected.") + + # Compute all-pairs shortest paths. + def path_length(v): + if method == "unweighted": + return nx.single_source_shortest_path_length(G, v) + elif method == "dijkstra": + return nx.single_source_dijkstra_path_length(G, v, weight=weight) + elif method == "bellman-ford": + return nx.single_source_bellman_ford_path_length(G, v, weight=weight) + + if method in single_source_methods: + # Sum the distances for each (ordered) pair of source and target node. + s = sum(l for u in G for l in path_length(u).values()) + else: + if method == "floyd-warshall": + all_pairs = nx.floyd_warshall(G, weight=weight) + s = sum(sum(t.values()) for t in all_pairs.values()) + elif method == "floyd-warshall-numpy": + s = float(nx.floyd_warshall_numpy(G, weight=weight).sum()) + return s / (n * (n - 1)) + + +@nx._dispatchable(edge_attrs="weight") +def all_shortest_paths(G, source, target, weight=None, method="dijkstra"): + """Compute all shortest simple paths in the graph. + + Parameters + ---------- + G : NetworkX graph + + source : node + Starting node for path. + + target : node + Ending node for path. + + weight : None, string or function, optional (default = None) + If None, every edge has weight/distance/cost 1. + If a string, use this edge attribute as the edge weight. + Any edge attribute not present defaults to 1. + If this is a function, the weight of an edge is the value + returned by the function. The function must accept exactly + three positional arguments: the two endpoints of an edge and + the dictionary of edge attributes for that edge. + The function must return a number. + + method : string, optional (default = 'dijkstra') + The algorithm to use to compute the path lengths. + Supported options: 'dijkstra', 'bellman-ford'. + Other inputs produce a ValueError. + If `weight` is None, unweighted graph methods are used, and this + suggestion is ignored. + + Returns + ------- + paths : generator of lists + A generator of all paths between source and target. + + Raises + ------ + ValueError + If `method` is not among the supported options. + + NetworkXNoPath + If `target` cannot be reached from `source`. + + Examples + -------- + >>> G = nx.Graph() + >>> nx.add_path(G, [0, 1, 2]) + >>> nx.add_path(G, [0, 10, 2]) + >>> print([p for p in nx.all_shortest_paths(G, source=0, target=2)]) + [[0, 1, 2], [0, 10, 2]] + + Notes + ----- + There may be many shortest paths between the source and target. If G + contains zero-weight cycles, this function will not produce all shortest + paths because doing so would produce infinitely many paths of unbounded + length -- instead, we only produce the shortest simple paths. + + See Also + -------- + shortest_path + single_source_shortest_path + all_pairs_shortest_path + """ + method = "unweighted" if weight is None else method + if method == "unweighted": + pred = nx.predecessor(G, source) + elif method == "dijkstra": + pred, dist = nx.dijkstra_predecessor_and_distance(G, source, weight=weight) + elif method == "bellman-ford": + pred, dist = nx.bellman_ford_predecessor_and_distance(G, source, weight=weight) + else: + raise ValueError(f"method not supported: {method}") + + return _build_paths_from_predecessors({source}, target, pred) + + +@nx._dispatchable(edge_attrs="weight") +def single_source_all_shortest_paths(G, source, weight=None, method="dijkstra"): + """Compute all shortest simple paths from the given source in the graph. + + Parameters + ---------- + G : NetworkX graph + + source : node + Starting node for path. + + weight : None, string or function, optional (default = None) + If None, every edge has weight/distance/cost 1. + If a string, use this edge attribute as the edge weight. + Any edge attribute not present defaults to 1. + If this is a function, the weight of an edge is the value + returned by the function. The function must accept exactly + three positional arguments: the two endpoints of an edge and + the dictionary of edge attributes for that edge. + The function must return a number. + + method : string, optional (default = 'dijkstra') + The algorithm to use to compute the path lengths. + Supported options: 'dijkstra', 'bellman-ford'. + Other inputs produce a ValueError. + If `weight` is None, unweighted graph methods are used, and this + suggestion is ignored. + + Returns + ------- + paths : generator of dictionary + A generator of all paths between source and all nodes in the graph. + + Raises + ------ + ValueError + If `method` is not among the supported options. + + Examples + -------- + >>> G = nx.Graph() + >>> nx.add_path(G, [0, 1, 2, 3, 0]) + >>> dict(nx.single_source_all_shortest_paths(G, source=0)) + {0: [[0]], 1: [[0, 1]], 3: [[0, 3]], 2: [[0, 1, 2], [0, 3, 2]]} + + Notes + ----- + There may be many shortest paths between the source and target. If G + contains zero-weight cycles, this function will not produce all shortest + paths because doing so would produce infinitely many paths of unbounded + length -- instead, we only produce the shortest simple paths. + + See Also + -------- + shortest_path + all_shortest_paths + single_source_shortest_path + all_pairs_shortest_path + all_pairs_all_shortest_paths + """ + method = "unweighted" if weight is None else method + if method == "unweighted": + pred = nx.predecessor(G, source) + elif method == "dijkstra": + pred, dist = nx.dijkstra_predecessor_and_distance(G, source, weight=weight) + elif method == "bellman-ford": + pred, dist = nx.bellman_ford_predecessor_and_distance(G, source, weight=weight) + else: + raise ValueError(f"method not supported: {method}") + for n in pred: + yield n, list(_build_paths_from_predecessors({source}, n, pred)) + + +@nx._dispatchable(edge_attrs="weight") +def all_pairs_all_shortest_paths(G, weight=None, method="dijkstra"): + """Compute all shortest paths between all nodes. + + Parameters + ---------- + G : NetworkX graph + + weight : None, string or function, optional (default = None) + If None, every edge has weight/distance/cost 1. + If a string, use this edge attribute as the edge weight. + Any edge attribute not present defaults to 1. + If this is a function, the weight of an edge is the value + returned by the function. The function must accept exactly + three positional arguments: the two endpoints of an edge and + the dictionary of edge attributes for that edge. + The function must return a number. + + method : string, optional (default = 'dijkstra') + The algorithm to use to compute the path lengths. + Supported options: 'dijkstra', 'bellman-ford'. + Other inputs produce a ValueError. + If `weight` is None, unweighted graph methods are used, and this + suggestion is ignored. + + Returns + ------- + paths : generator of dictionary + Dictionary of arrays, keyed by source and target, of all shortest paths. + + Raises + ------ + ValueError + If `method` is not among the supported options. + + Examples + -------- + >>> G = nx.cycle_graph(4) + >>> dict(nx.all_pairs_all_shortest_paths(G))[0][2] + [[0, 1, 2], [0, 3, 2]] + >>> dict(nx.all_pairs_all_shortest_paths(G))[0][3] + [[0, 3]] + + Notes + ----- + There may be multiple shortest paths with equal lengths. Unlike + all_pairs_shortest_path, this method returns all shortest paths. + + See Also + -------- + all_pairs_shortest_path + single_source_all_shortest_paths + """ + for n in G: + yield ( + n, + dict(single_source_all_shortest_paths(G, n, weight=weight, method=method)), + ) + + +def _build_paths_from_predecessors(sources, target, pred): + """Compute all simple paths to target, given the predecessors found in + pred, terminating when any source in sources is found. + + Parameters + ---------- + sources : set + Starting nodes for path. + + target : node + Ending node for path. + + pred : dict + A dictionary of predecessor lists, keyed by node + + Returns + ------- + paths : generator of lists + A generator of all paths between source and target. + + Raises + ------ + NetworkXNoPath + If `target` cannot be reached from `source`. + + Notes + ----- + There may be many paths between the sources and target. If there are + cycles among the predecessors, this function will not produce all + possible paths because doing so would produce infinitely many paths + of unbounded length -- instead, we only produce simple paths. + + See Also + -------- + shortest_path + single_source_shortest_path + all_pairs_shortest_path + all_shortest_paths + bellman_ford_path + """ + if target not in pred: + raise nx.NetworkXNoPath(f"Target {target} cannot be reached from given sources") + + seen = {target} + stack = [[target, 0]] + top = 0 + while top >= 0: + node, i = stack[top] + if node in sources: + yield [p for p, n in reversed(stack[: top + 1])] + if len(pred[node]) > i: + stack[top][1] = i + 1 + next = pred[node][i] + if next in seen: + continue + else: + seen.add(next) + top += 1 + if top == len(stack): + stack.append([next, 0]) + else: + stack[top][:] = [next, 0] + else: + seen.discard(node) + top -= 1 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/shortest_paths/unweighted.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/shortest_paths/unweighted.py new file mode 100644 index 0000000000000000000000000000000000000000..4dffeb51825dca4c04ed09fffc48fc93a3d7e63d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/shortest_paths/unweighted.py @@ -0,0 +1,625 @@ +""" +Shortest path algorithms for unweighted graphs. +""" + +import operator + +import networkx as nx + +__all__ = [ + "bidirectional_shortest_path", + "single_source_shortest_path", + "single_source_shortest_path_length", + "single_target_shortest_path", + "single_target_shortest_path_length", + "all_pairs_shortest_path", + "all_pairs_shortest_path_length", + "predecessor", +] + + +@nx._dispatchable +def single_source_shortest_path_length(G, source, cutoff=None): + """Compute the shortest path lengths from `source` to all reachable nodes in `G`. + + Parameters + ---------- + G : NetworkX graph + + source : node + Starting node for path + + cutoff : integer, optional + Depth to stop the search. Only target nodes where the shortest path to + this node from the source node contains <= ``cutoff + 1`` nodes will be + included in the returned results. + + Returns + ------- + lengths : dict + Dict keyed by node to shortest path length from source node. + + Examples + -------- + >>> G = nx.path_graph(5) + >>> length = nx.single_source_shortest_path_length(G, 0) + >>> length[4] + 4 + >>> for node in sorted(length): + ... print(f"{node}: {length[node]}") + 0: 0 + 1: 1 + 2: 2 + 3: 3 + 4: 4 + + Only include paths with length less than or equal to the `cutoff` keyword + argument: + + >>> length = nx.single_source_shortest_path_length(G, 0, cutoff=2) + >>> for node in sorted(length): + ... print(f"{node}: {length[node]}") + 0: 0 + 1: 1 + 2: 2 + + See Also + -------- + :any:`shortest_path_length` : + Shortest path length with specifiable source, target, and weight. + :any:`single_source_dijkstra_path_length` : + Shortest weighted path length from source with Dijkstra algorithm. + :any:`single_source_bellman_ford_path_length` : + Shortest weighted path length from source with Bellman-Ford algorithm. + """ + if source not in G: + raise nx.NodeNotFound(f"Source {source} is not in G") + if cutoff is None: + cutoff = float("inf") + nextlevel = [source] + return dict(_single_shortest_path_length(G._adj, nextlevel, cutoff)) + + +def _single_shortest_path_length(adj, firstlevel, cutoff): + """Yields (node, level) in a breadth first search + + Shortest Path Length helper function + Parameters + ---------- + adj : dict + Adjacency dict or view + firstlevel : list + starting nodes, e.g. [source] or [target] + cutoff : int or float + level at which we stop the process + """ + seen = set(firstlevel) + nextlevel = firstlevel + level = 0 + n = len(adj) + for v in nextlevel: + yield (v, level) + while nextlevel and cutoff > level: + level += 1 + thislevel = nextlevel + nextlevel = [] + for v in thislevel: + for w in adj[v]: + if w not in seen: + seen.add(w) + nextlevel.append(w) + yield (w, level) + if len(seen) == n: + return + + +@nx._dispatchable +def single_target_shortest_path_length(G, target, cutoff=None): + """Compute the shortest path lengths to target from all reachable nodes. + + Parameters + ---------- + G : NetworkX graph + + target : node + Target node for path + + cutoff : integer, optional + Depth to stop the search. Only source nodes where the shortest path + from this node to the target node contains <= ``cutoff + 1`` nodes will + be included in the returned results. + + Returns + ------- + lengths : dictionary + Dictionary, keyed by source, of shortest path lengths. + + Examples + -------- + >>> G = nx.path_graph(5, create_using=nx.DiGraph()) + >>> length = nx.single_target_shortest_path_length(G, 4) + >>> length[0] + 4 + >>> for node in sorted(length): + ... print(f"{node}: {length[node]}") + 0: 4 + 1: 3 + 2: 2 + 3: 1 + 4: 0 + + Only include paths with length less than or equal to the `cutoff` keyword + argument: + + >>> length = nx.single_target_shortest_path_length(G, 4, cutoff=2) + >>> for node in sorted(length): + ... print(f"{node}: {length[node]}") + 2: 2 + 3: 1 + 4: 0 + + See Also + -------- + single_source_shortest_path_length, shortest_path_length + """ + if target not in G: + raise nx.NodeNotFound(f"Target {target} is not in G") + if cutoff is None: + cutoff = float("inf") + # handle either directed or undirected + adj = G._pred if G.is_directed() else G._adj + nextlevel = [target] + return dict(_single_shortest_path_length(adj, nextlevel, cutoff)) + + +@nx._dispatchable +def all_pairs_shortest_path_length(G, cutoff=None): + """Computes the shortest path lengths between all nodes in `G`. + + Parameters + ---------- + G : NetworkX graph + + cutoff : integer, optional + Depth at which to stop the search. Only paths of length at most + `cutoff` (i.e. paths containing <= ``cutoff + 1`` nodes) are returned. + + Returns + ------- + lengths : iterator + (source, dictionary) iterator with dictionary keyed by target and + shortest path length as the key value. + + Notes + ----- + The iterator returned only has reachable node pairs. + + Examples + -------- + >>> G = nx.path_graph(5) + >>> length = dict(nx.all_pairs_shortest_path_length(G)) + >>> for node in [0, 1, 2, 3, 4]: + ... print(f"1 - {node}: {length[1][node]}") + 1 - 0: 1 + 1 - 1: 0 + 1 - 2: 1 + 1 - 3: 2 + 1 - 4: 3 + >>> length[3][2] + 1 + >>> length[2][2] + 0 + + Only include paths with length less than or equal to the `cutoff` keyword + argument: + + >>> path_lengths = dict(nx.all_pairs_shortest_path_length(G, cutoff=2)) + >>> path_lengths[1] # node 4 is too far away to appear + {1: 0, 0: 1, 2: 1, 3: 2} + """ + length = single_source_shortest_path_length + # TODO This can be trivially parallelized. + for n in G: + yield (n, length(G, n, cutoff=cutoff)) + + +@nx._dispatchable +def bidirectional_shortest_path(G, source, target): + """Returns a list of nodes in a shortest path between source and target. + + Parameters + ---------- + G : NetworkX graph + + source : node label + starting node for path + + target : node label + ending node for path + + Returns + ------- + path: list + List of nodes in a path from source to target. + + Raises + ------ + NetworkXNoPath + If no path exists between source and target. + + Examples + -------- + >>> G = nx.Graph() + >>> nx.add_path(G, [0, 1, 2, 3, 0, 4, 5, 6, 7, 4]) + >>> nx.bidirectional_shortest_path(G, 2, 6) + [2, 1, 0, 4, 5, 6] + + See Also + -------- + shortest_path + + Notes + ----- + This algorithm is used by shortest_path(G, source, target). + """ + + if source not in G: + raise nx.NodeNotFound(f"Source {source} is not in G") + + if target not in G: + raise nx.NodeNotFound(f"Target {target} is not in G") + + # call helper to do the real work + results = _bidirectional_pred_succ(G, source, target) + pred, succ, w = results + + # build path from pred+w+succ + path = [] + # from source to w + while w is not None: + path.append(w) + w = pred[w] + path.reverse() + # from w to target + w = succ[path[-1]] + while w is not None: + path.append(w) + w = succ[w] + + return path + + +def _bidirectional_pred_succ(G, source, target): + """Bidirectional shortest path helper. + + Returns (pred, succ, w) where + pred is a dictionary of predecessors from w to the source, and + succ is a dictionary of successors from w to the target. + """ + # does BFS from both source and target and meets in the middle + if target == source: + return ({target: None}, {source: None}, source) + + # handle either directed or undirected + if G.is_directed(): + Gpred = G.pred + Gsucc = G.succ + else: + Gpred = G.adj + Gsucc = G.adj + + # predecessor and successors in search + pred = {source: None} + succ = {target: None} + + # initialize fringes, start with forward + forward_fringe = [source] + reverse_fringe = [target] + + while forward_fringe and reverse_fringe: + if len(forward_fringe) <= len(reverse_fringe): + this_level = forward_fringe + forward_fringe = [] + for v in this_level: + for w in Gsucc[v]: + if w not in pred: + forward_fringe.append(w) + pred[w] = v + if w in succ: # path found + return pred, succ, w + else: + this_level = reverse_fringe + reverse_fringe = [] + for v in this_level: + for w in Gpred[v]: + if w not in succ: + succ[w] = v + reverse_fringe.append(w) + if w in pred: # found path + return pred, succ, w + + raise nx.NetworkXNoPath(f"No path between {source} and {target}.") + + +@nx._dispatchable +def single_source_shortest_path(G, source, cutoff=None): + """Compute shortest path between source + and all other nodes reachable from source. + + Parameters + ---------- + G : NetworkX graph + + source : node label + Starting node for path + + cutoff : integer, optional + Depth to stop the search. Only target nodes where the shortest path to + this node from the source node contains <= ``cutoff + 1`` nodes will be + included in the returned results. + + Returns + ------- + paths : dictionary + Dictionary, keyed by target, of shortest paths. + + Examples + -------- + >>> G = nx.path_graph(5) + >>> nx.single_source_shortest_path(G, 0) + {0: [0], 1: [0, 1], 2: [0, 1, 2], 3: [0, 1, 2, 3], 4: [0, 1, 2, 3, 4]} + + Only include paths with length less than or equal to the `cutoff` keyword + argument: + + >>> nx.single_source_shortest_path(G, 0, cutoff=2) + {0: [0], 1: [0, 1], 2: [0, 1, 2]} + + Notes + ----- + The shortest path is not necessarily unique. So there can be multiple + paths between the source and each target node, all of which have the + same 'shortest' length. For each target node, this function returns + only one of those paths. + + See Also + -------- + shortest_path + """ + if source not in G: + raise nx.NodeNotFound(f"Source {source} not in G") + if cutoff is None: + cutoff = float("inf") + nextlevel = [source] # list of nodes to check at next level + paths = {source: [source]} # paths dictionary (paths to key from source) + return dict(_single_shortest_path(G._adj, nextlevel, paths, cutoff, operator.add)) + + +def _single_shortest_path(adj, firstlevel, paths, cutoff, join): + """Returns shortest paths + + Shortest Path helper function + Parameters + ---------- + adj : dict + Adjacency dict or view + firstlevel : list + starting nodes, e.g. [source] or [target] + paths : dict + paths for starting nodes, e.g. {source: [source]} + cutoff : int or float + level at which we stop the process + join : function + function to construct a path from two partial paths. Requires two + list inputs `p1` and `p2`, and returns a list. Usually returns + `p1 + p2` (forward from source) or `p2 + p1` (backward from target) + """ + level = 0 + nextlevel = firstlevel + n = len(adj) + while nextlevel and cutoff > level: + thislevel = nextlevel + nextlevel = [] + for v in thislevel: + for w in adj[v]: + if w not in paths: + paths[w] = join(paths[v], [w]) + nextlevel.append(w) + if len(paths) == n: + return paths + level += 1 + return paths + + +@nx._dispatchable +def single_target_shortest_path(G, target, cutoff=None): + """Compute shortest path to target from all nodes that reach target. + + Parameters + ---------- + G : NetworkX graph + + target : node label + Target node for path + + cutoff : integer, optional + Depth to stop the search. Only source nodes where the shortest path + from this node to the target node contains <= ``cutoff + 1`` nodes will + be included in the returned results. + + Returns + ------- + paths : dictionary + Dictionary, keyed by source, of shortest paths. + + Examples + -------- + >>> G = nx.path_graph(5, create_using=nx.DiGraph()) + >>> nx.single_target_shortest_path(G, 4) + {4: [4], 3: [3, 4], 2: [2, 3, 4], 1: [1, 2, 3, 4], 0: [0, 1, 2, 3, 4]} + + Only include paths with length less than or equal to the `cutoff` keyword + argument: + + >>> nx.single_target_shortest_path(G, 4, cutoff=2) + {4: [4], 3: [3, 4], 2: [2, 3, 4]} + + Notes + ----- + The shortest path is not necessarily unique. So there can be multiple + paths between the source and each target node, all of which have the + same 'shortest' length. For each target node, this function returns + only one of those paths. + + See Also + -------- + shortest_path, single_source_shortest_path + """ + if target not in G: + raise nx.NodeNotFound(f"Target {target} not in G") + + def join(p1, p2): + return p2 + p1 + + # handle undirected graphs + adj = G._pred if G.is_directed() else G._adj + if cutoff is None: + cutoff = float("inf") + nextlevel = [target] # list of nodes to check at next level + paths = {target: [target]} # paths dictionary (paths to key from source) + return dict(_single_shortest_path(adj, nextlevel, paths, cutoff, join)) + + +@nx._dispatchable +def all_pairs_shortest_path(G, cutoff=None): + """Compute shortest paths between all nodes. + + Parameters + ---------- + G : NetworkX graph + + cutoff : integer, optional + Depth at which to stop the search. Only paths containing at most + ``cutoff + 1`` nodes are returned. + + Returns + ------- + paths : iterator + Dictionary, keyed by source and target, of shortest paths. + + Examples + -------- + >>> G = nx.path_graph(5) + >>> path = dict(nx.all_pairs_shortest_path(G)) + >>> print(path[0]) + {0: [0], 1: [0, 1], 2: [0, 1, 2], 3: [0, 1, 2, 3], 4: [0, 1, 2, 3, 4]} + + Only include paths with length less than or equal to the `cutoff` keyword + argument: + + >>> path = dict(nx.all_pairs_shortest_path(G, cutoff=2)) + >>> print(path[0]) + {0: [0], 1: [0, 1], 2: [0, 1, 2]} + + Notes + ----- + There may be multiple shortest paths with the same length between + two nodes. For each pair, this function returns only one of those paths. + + See Also + -------- + floyd_warshall + all_pairs_all_shortest_paths + + """ + # TODO This can be trivially parallelized. + for n in G: + yield (n, single_source_shortest_path(G, n, cutoff=cutoff)) + + +@nx._dispatchable +def predecessor(G, source, target=None, cutoff=None, return_seen=None): + """Returns dict of predecessors for the path from source to all nodes in G. + + Parameters + ---------- + G : NetworkX graph + + source : node label + Starting node for path + + target : node label, optional + Ending node for path. If provided only predecessors between + source and target are returned + + cutoff : integer, optional + Depth to stop the search. Only paths of length <= `cutoff` are + returned. + + return_seen : bool, optional (default=None) + Whether to return a dictionary, keyed by node, of the level (number of + hops) to reach the node (as seen during breadth-first-search). + + Returns + ------- + pred : dictionary + Dictionary, keyed by node, of predecessors in the shortest path. + + + (pred, seen): tuple of dictionaries + If `return_seen` argument is set to `True`, then a tuple of dictionaries + is returned. The first element is the dictionary, keyed by node, of + predecessors in the shortest path. The second element is the dictionary, + keyed by node, of the level (number of hops) to reach the node (as seen + during breadth-first-search). + + Examples + -------- + >>> G = nx.path_graph(4) + >>> list(G) + [0, 1, 2, 3] + >>> nx.predecessor(G, 0) + {0: [], 1: [0], 2: [1], 3: [2]} + >>> nx.predecessor(G, 0, cutoff=2) + {0: [], 1: [0], 2: [1]} + >>> nx.predecessor(G, 0, return_seen=True) + ({0: [], 1: [0], 2: [1], 3: [2]}, {0: 0, 1: 1, 2: 2, 3: 3}) + + + """ + if source not in G: + raise nx.NodeNotFound(f"Source {source} not in G") + + level = 0 # the current level + nextlevel = [source] # list of nodes to check at next level + seen = {source: level} # level (number of hops) when seen in BFS + pred = {source: []} # predecessor dictionary + while nextlevel: + level = level + 1 + thislevel = nextlevel + nextlevel = [] + for v in thislevel: + for w in G[v]: + if w not in seen: + pred[w] = [v] + seen[w] = level + nextlevel.append(w) + elif seen[w] == level: # add v to predecessor list if it + pred[w].append(v) # is at the correct level + if cutoff and cutoff <= level: + break + + if target is not None: + if return_seen: + if target not in pred: + return ([], -1) # No predecessor + return (pred[target], seen[target]) + else: + if target not in pred: + return [] # No predecessor + return pred[target] + else: + if return_seen: + return (pred, seen) + else: + return pred diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/shortest_paths/weighted.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/shortest_paths/weighted.py new file mode 100644 index 0000000000000000000000000000000000000000..a75398be79a83bee9bb1ecc6f6ce8fd62bca5ee4 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/networkx/algorithms/shortest_paths/weighted.py @@ -0,0 +1,2542 @@ +""" +Shortest path algorithms for weighted graphs. +""" + +from collections import deque +from heapq import heappop, heappush +from itertools import count, islice + +import networkx as nx +from networkx.algorithms.shortest_paths.generic import _build_paths_from_predecessors + +__all__ = [ + "dijkstra_path", + "dijkstra_path_length", + "bidirectional_dijkstra", + "single_source_dijkstra", + "single_source_dijkstra_path", + "single_source_dijkstra_path_length", + "multi_source_dijkstra", + "multi_source_dijkstra_path", + "multi_source_dijkstra_path_length", + "all_pairs_dijkstra", + "all_pairs_dijkstra_path", + "all_pairs_dijkstra_path_length", + "dijkstra_predecessor_and_distance", + "bellman_ford_path", + "bellman_ford_path_length", + "single_source_bellman_ford", + "single_source_bellman_ford_path", + "single_source_bellman_ford_path_length", + "all_pairs_bellman_ford_path", + "all_pairs_bellman_ford_path_length", + "bellman_ford_predecessor_and_distance", + "negative_edge_cycle", + "find_negative_cycle", + "goldberg_radzik", + "johnson", +] + + +def _weight_function(G, weight): + """Returns a function that returns the weight of an edge. + + The returned function is specifically suitable for input to + functions :func:`_dijkstra` and :func:`_bellman_ford_relaxation`. + + Parameters + ---------- + G : NetworkX graph. + + weight : string or function + If it is callable, `weight` itself is returned. If it is a string, + it is assumed to be the name of the edge attribute that represents + the weight of an edge. In that case, a function is returned that + gets the edge weight according to the specified edge attribute. + + Returns + ------- + function + This function returns a callable that accepts exactly three inputs: + a node, an node adjacent to the first one, and the edge attribute + dictionary for the eedge joining those nodes. That function returns + a number representing the weight of an edge. + + If `G` is a multigraph, and `weight` is not callable, the + minimum edge weight over all parallel edges is returned. If any edge + does not have an attribute with key `weight`, it is assumed to + have weight one. + + """ + if callable(weight): + return weight + # If the weight keyword argument is not callable, we assume it is a + # string representing the edge attribute containing the weight of + # the edge. + if G.is_multigraph(): + return lambda u, v, d: min(attr.get(weight, 1) for attr in d.values()) + return lambda u, v, data: data.get(weight, 1) + + +@nx._dispatchable(edge_attrs="weight") +def dijkstra_path(G, source, target, weight="weight"): + """Returns the shortest weighted path from source to target in G. + + Uses Dijkstra's Method to compute the shortest weighted path + between two nodes in a graph. + + Parameters + ---------- + G : NetworkX graph + + source : node + Starting node + + target : node + Ending node + + weight : string or function + If this is a string, then edge weights will be accessed via the + edge attribute with this key (that is, the weight of the edge + joining `u` to `v` will be ``G.edges[u, v][weight]``). If no + such edge attribute exists, the weight of the edge is assumed to + be one. + + If this is a function, the weight of an edge is the value + returned by the function. The function must accept exactly three + positional arguments: the two endpoints of an edge and the + dictionary of edge attributes for that edge. The function must + return a number or None to indicate a hidden edge. + + Returns + ------- + path : list + List of nodes in a shortest path. + + Raises + ------ + NodeNotFound + If `source` is not in `G`. + + NetworkXNoPath + If no path exists between source and target. + + Examples + -------- + >>> G = nx.path_graph(5) + >>> print(nx.dijkstra_path(G, 0, 4)) + [0, 1, 2, 3, 4] + + Find edges of shortest path in Multigraph + + >>> G = nx.MultiDiGraph() + >>> G.add_weighted_edges_from([(1, 2, 0.75), (1, 2, 0.5), (2, 3, 0.5), (1, 3, 1.5)]) + >>> nodes = nx.dijkstra_path(G, 1, 3) + >>> edges = nx.utils.pairwise(nodes) + >>> list( + ... (u, v, min(G[u][v], key=lambda k: G[u][v][k].get("weight", 1))) + ... for u, v in edges + ... ) + [(1, 2, 1), (2, 3, 0)] + + Notes + ----- + Edge weight attributes must be numerical. + Distances are calculated as sums of weighted edges traversed. + + The weight function can be used to hide edges by returning None. + So ``weight = lambda u, v, d: 1 if d['color']=="red" else None`` + will find the shortest red path. + + The weight function can be used to include node weights. + + >>> def func(u, v, d): + ... node_u_wt = G.nodes[u].get("node_weight", 1) + ... node_v_wt = G.nodes[v].get("node_weight", 1) + ... edge_wt = d.get("weight", 1) + ... return node_u_wt / 2 + node_v_wt / 2 + edge_wt + + In this example we take the average of start and end node + weights of an edge and add it to the weight of the edge. + + The function :func:`single_source_dijkstra` computes both + path and length-of-path if you need both, use that. + + See Also + -------- + bidirectional_dijkstra + bellman_ford_path + single_source_dijkstra + """ + (length, path) = single_source_dijkstra(G, source, target=target, weight=weight) + return path + + +@nx._dispatchable(edge_attrs="weight") +def dijkstra_path_length(G, source, target, weight="weight"): + """Returns the shortest weighted path length in G from source to target. + + Uses Dijkstra's Method to compute the shortest weighted path length + between two nodes in a graph. + + Parameters + ---------- + G : NetworkX graph + + source : node label + starting node for path + + target : node label + ending node for path + + weight : string or function + If this is a string, then edge weights will be accessed via the + edge attribute with this key (that is, the weight of the edge + joining `u` to `v` will be ``G.edges[u, v][weight]``). If no + such edge attribute exists, the weight of the edge is assumed to + be one. + + If this is a function, the weight of an edge is the value + returned by the function. The function must accept exactly three + positional arguments: the two endpoints of an edge and the + dictionary of edge attributes for that edge. The function must + return a number or None to indicate a hidden edge. + + Returns + ------- + length : number + Shortest path length. + + Raises + ------ + NodeNotFound + If `source` is not in `G`. + + NetworkXNoPath + If no path exists between source and target. + + Examples + -------- + >>> G = nx.path_graph(5) + >>> nx.dijkstra_path_length(G, 0, 4) + 4 + + Notes + ----- + Edge weight attributes must be numerical. + Distances are calculated as sums of weighted edges traversed. + + The weight function can be used to hide edges by returning None. + So ``weight = lambda u, v, d: 1 if d['color']=="red" else None`` + will find the shortest red path. + + The function :func:`single_source_dijkstra` computes both + path and length-of-path if you need both, use that. + + See Also + -------- + bidirectional_dijkstra + bellman_ford_path_length + single_source_dijkstra + + """ + if source not in G: + raise nx.NodeNotFound(f"Node {source} not found in graph") + if source == target: + return 0 + weight = _weight_function(G, weight) + length = _dijkstra(G, source, weight, target=target) + try: + return length[target] + except KeyError as err: + raise nx.NetworkXNoPath(f"Node {target} not reachable from {source}") from err + + +@nx._dispatchable(edge_attrs="weight") +def single_source_dijkstra_path(G, source, cutoff=None, weight="weight"): + """Find shortest weighted paths in G from a source node. + + Compute shortest path between source and all other reachable + nodes for a weighted graph. + + Parameters + ---------- + G : NetworkX graph + + source : node + Starting node for path. + + cutoff : integer or float, optional + Length (sum of edge weights) at which the search is stopped. + If cutoff is provided, only return paths with summed weight <= cutoff. + + weight : string or function + If this is a string, then edge weights will be accessed via the + edge attribute with this key (that is, the weight of the edge + joining `u` to `v` will be ``G.edges[u, v][weight]``). If no + such edge attribute exists, the weight of the edge is assumed to + be one. + + If this is a function, the weight of an edge is the value + returned by the function. The function must accept exactly three + positional arguments: the two endpoints of an edge and the + dictionary of edge attributes for that edge. The function must + return a number or None to indicate a hidden edge. + + Returns + ------- + paths : dictionary + Dictionary of shortest path lengths keyed by target. + + Raises + ------ + NodeNotFound + If `source` is not in `G`. + + Examples + -------- + >>> G = nx.path_graph(5) + >>> path = nx.single_source_dijkstra_path(G, 0) + >>> path[4] + [0, 1, 2, 3, 4] + + Notes + ----- + Edge weight attributes must be numerical. + Distances are calculated as sums of weighted edges traversed. + + The weight function can be used to hide edges by returning None. + So ``weight = lambda u, v, d: 1 if d['color']=="red" else None`` + will find the shortest red path. + + See Also + -------- + single_source_dijkstra, single_source_bellman_ford + + """ + return multi_source_dijkstra_path(G, {source}, cutoff=cutoff, weight=weight) + + +@nx._dispatchable(edge_attrs="weight") +def single_source_dijkstra_path_length(G, source, cutoff=None, weight="weight"): + """Find shortest weighted path lengths in G from a source node. + + Compute the shortest path length between source and all other + reachable nodes for a weighted graph. + + Parameters + ---------- + G : NetworkX graph + + source : node label + Starting node for path + + cutoff : integer or float, optional + Length (sum of edge weights) at which the search is stopped. + If cutoff is provided, only return paths with summed weight <= cutoff. + + weight : string or function + If this is a string, then edge weights will be accessed via the + edge attribute with this key (that is, the weight of the edge + joining `u` to `v` will be ``G.edges[u, v][weight]``). If no + such edge attribute exists, the weight of the edge is assumed to + be one. + + If this is a function, the weight of an edge is the value + returned by the function. The function must accept exactly three + positional arguments: the two endpoints of an edge and the + dictionary of edge attributes for that edge. The function must + return a number or None to indicate a hidden edge. + + Returns + ------- + length : dict + Dict keyed by node to shortest path length from source. + + Raises + ------ + NodeNotFound + If `source` is not in `G`. + + Examples + -------- + >>> G = nx.path_graph(5) + >>> length = nx.single_source_dijkstra_path_length(G, 0) + >>> length[4] + 4 + >>> for node in [0, 1, 2, 3, 4]: + ... print(f"{node}: {length[node]}") + 0: 0 + 1: 1 + 2: 2 + 3: 3 + 4: 4 + + Notes + ----- + Edge weight attributes must be numerical. + Distances are calculated as sums of weighted edges traversed. + + The weight function can be used to hide edges by returning None. + So ``weight = lambda u, v, d: 1 if d['color']=="red" else None`` + will find the shortest red path. + + See Also + -------- + single_source_dijkstra, single_source_bellman_ford_path_length + + """ + return multi_source_dijkstra_path_length(G, {source}, cutoff=cutoff, weight=weight) + + +@nx._dispatchable(edge_attrs="weight") +def single_source_dijkstra(G, source, target=None, cutoff=None, weight="weight"): + """Find shortest weighted paths and lengths from a source node. + + Compute the shortest path length between source and all other + reachable nodes for a weighted graph. + + Uses Dijkstra's algorithm to compute shortest paths and lengths + between a source and all other reachable nodes in a weighted graph. + + Parameters + ---------- + G : NetworkX graph + + source : node label + Starting node for path + + target : node label, optional + Ending node for path + + cutoff : integer or float, optional + Length (sum of edge weights) at which the search is stopped. + If cutoff is provided, only return paths with summed weight <= cutoff. + + + weight : string or function + If this is a string, then edge weights will be accessed via the + edge attribute with this key (that is, the weight of the edge + joining `u` to `v` will be ``G.edges[u, v][weight]``). If no + such edge attribute exists, the weight of the edge is assumed to + be one. + + If this is a function, the weight of an edge is the value + returned by the function. The function must accept exactly three + positional arguments: the two endpoints of an edge and the + dictionary of edge attributes for that edge. The function must + return a number or None to indicate a hidden edge. + + Returns + ------- + distance, path : pair of dictionaries, or numeric and list. + If target is None, paths and lengths to all nodes are computed. + The return value is a tuple of two dictionaries keyed by target nodes. + The first dictionary stores distance to each target node. + The second stores the path to each target node. + If target is not None, returns a tuple (distance, path), where + distance is the distance from source to target and path is a list + representing the path from source to target. + + Raises + ------ + NodeNotFound + If `source` is not in `G`. + + Examples + -------- + >>> G = nx.path_graph(5) + >>> length, path = nx.single_source_dijkstra(G, 0) + >>> length[4] + 4 + >>> for node in [0, 1, 2, 3, 4]: + ... print(f"{node}: {length[node]}") + 0: 0 + 1: 1 + 2: 2 + 3: 3 + 4: 4 + >>> path[4] + [0, 1, 2, 3, 4] + >>> length, path = nx.single_source_dijkstra(G, 0, 1) + >>> length + 1 + >>> path + [0, 1] + + Notes + ----- + Edge weight attributes must be numerical. + Distances are calculated as sums of weighted edges traversed. + + The weight function can be used to hide edges by returning None. + So ``weight = lambda u, v, d: 1 if d['color']=="red" else None`` + will find the shortest red path. + + Based on the Python cookbook recipe (119466) at + https://code.activestate.com/recipes/119466/ + + This algorithm is not guaranteed to work if edge weights + are negative or are floating point numbers + (overflows and roundoff errors can cause problems). + + See Also + -------- + single_source_dijkstra_path + single_source_dijkstra_path_length + single_source_bellman_ford + """ + return multi_source_dijkstra( + G, {source}, cutoff=cutoff, target=target, weight=weight + ) + + +@nx._dispatchable(edge_attrs="weight") +def multi_source_dijkstra_path(G, sources, cutoff=None, weight="weight"): + """Find shortest weighted paths in G from a given set of source + nodes. + + Compute shortest path between any of the source nodes and all other + reachable nodes for a weighted graph. + + Parameters + ---------- + G : NetworkX graph + + sources : non-empty set of nodes + Starting nodes for paths. If this is just a set containing a + single node, then all paths computed by this function will start + from that node. If there are two or more nodes in the set, the + computed paths may begin from any one of the start nodes. + + cutoff : integer or float, optional + Length (sum of edge weights) at which the search is stopped. + If cutoff is provided, only return paths with summed weight <= cutoff. + + weight : string or function + If this is a string, then edge weights will be accessed via the + edge attribute with this key (that is, the weight of the edge + joining `u` to `v` will be ``G.edges[u, v][weight]``). If no + such edge attribute exists, the weight of the edge is assumed to + be one. + + If this is a function, the weight of an edge is the value + returned by the function. The function must accept exactly three + positional arguments: the two endpoints of an edge and the + dictionary of edge attributes for that edge. The function must + return a number or None to indicate a hidden edge. + + Returns + ------- + paths : dictionary + Dictionary of shortest paths keyed by target. + + Examples + -------- + >>> G = nx.path_graph(5) + >>> path = nx.multi_source_dijkstra_path(G, {0, 4}) + >>> path[1] + [0, 1] + >>> path[3] + [4, 3] + + Notes + ----- + Edge weight attributes must be numerical. + Distances are calculated as sums of weighted edges traversed. + + The weight function can be used to hide edges by returning None. + So ``weight = lambda u, v, d: 1 if d['color']=="red" else None`` + will find the shortest red path. + + Raises + ------ + ValueError + If `sources` is empty. + NodeNotFound + If any of `sources` is not in `G`. + + See Also + -------- + multi_source_dijkstra, multi_source_bellman_ford + + """ + length, path = multi_source_dijkstra(G, sources, cutoff=cutoff, weight=weight) + return path + + +@nx._dispatchable(edge_attrs="weight") +def multi_source_dijkstra_path_length(G, sources, cutoff=None, weight="weight"): + """Find shortest weighted path lengths in G from a given set of + source nodes. + + Compute the shortest path length between any of the source nodes and + all other reachable nodes for a weighted graph. + + Parameters + ---------- + G : NetworkX graph + + sources : non-empty set of nodes + Starting nodes for paths. If this is just a set containing a + single node, then all paths computed by this function will start + from that node. If there are two or more nodes in the set, the + computed paths may begin from any one of the start nodes. + + cutoff : integer or float, optional + Length (sum of edge weights) at which the search is stopped. + If cutoff is provided, only return paths with summed weight <= cutoff. + + weight : string or function + If this is a string, then edge weights will be accessed via the + edge attribute with this key (that is, the weight of the edge + joining `u` to `v` will be ``G.edges[u, v][weight]``). If no + such edge attribute exists, the weight of the edge is assumed to + be one. + + If this is a function, the weight of an edge is the value + returned by the function. The function must accept exactly three + positional arguments: the two endpoints of an edge and the + dictionary of edge attributes for that edge. The function must + return a number or None to indicate a hidden edge. + + Returns + ------- + length : dict + Dict keyed by node to shortest path length to nearest source. + + Examples + -------- + >>> G = nx.path_graph(5) + >>> length = nx.multi_source_dijkstra_path_length(G, {0, 4}) + >>> for node in [0, 1, 2, 3, 4]: + ... print(f"{node}: {length[node]}") + 0: 0 + 1: 1 + 2: 2 + 3: 1 + 4: 0 + + Notes + ----- + Edge weight attributes must be numerical. + Distances are calculated as sums of weighted edges traversed. + + The weight function can be used to hide edges by returning None. + So ``weight = lambda u, v, d: 1 if d['color']=="red" else None`` + will find the shortest red path. + + Raises + ------ + ValueError + If `sources` is empty. + NodeNotFound + If any of `sources` is not in `G`. + + See Also + -------- + multi_source_dijkstra + + """ + if not sources: + raise ValueError("sources must not be empty") + for s in sources: + if s not in G: + raise nx.NodeNotFound(f"Node {s} not found in graph") + weight = _weight_function(G, weight) + return _dijkstra_multisource(G, sources, weight, cutoff=cutoff) + + +@nx._dispatchable(edge_attrs="weight") +def multi_source_dijkstra(G, sources, target=None, cutoff=None, weight="weight"): + """Find shortest weighted paths and lengths from a given set of + source nodes. + + Uses Dijkstra's algorithm to compute the shortest paths and lengths + between one of the source nodes and the given `target`, or all other + reachable nodes if not specified, for a weighted graph. + + Parameters + ---------- + G : NetworkX graph + + sources : non-empty set of nodes + Starting nodes for paths. If this is just a set containing a + single node, then all paths computed by this function will start + from that node. If there are two or more nodes in the set, the + computed paths may begin from any one of the start nodes. + + target : node label, optional + Ending node for path + + cutoff : integer or float, optional + Length (sum of edge weights) at which the search is stopped. + If cutoff is provided, only return paths with summed weight <= cutoff. + + weight : string or function + If this is a string, then edge weights will be accessed via the + edge attribute with this key (that is, the weight of the edge + joining `u` to `v` will be ``G.edges[u, v][weight]``). If no + such edge attribute exists, the weight of the edge is assumed to + be one. + + If this is a function, the weight of an edge is the value + returned by the function. The function must accept exactly three + positional arguments: the two endpoints of an edge and the + dictionary of edge attributes for that edge. The function must + return a number or None to indicate a hidden edge. + + Returns + ------- + distance, path : pair of dictionaries, or numeric and list + If target is None, returns a tuple of two dictionaries keyed by node. + The first dictionary stores distance from one of the source nodes. + The second stores the path from one of the sources to that node. + If target is not None, returns a tuple of (distance, path) where + distance is the distance from source to target and path is a list + representing the path from source to target. + + Examples + -------- + >>> G = nx.path_graph(5) + >>> length, path = nx.multi_source_dijkstra(G, {0, 4}) + >>> for node in [0, 1, 2, 3, 4]: + ... print(f"{node}: {length[node]}") + 0: 0 + 1: 1 + 2: 2 + 3: 1 + 4: 0 + >>> path[1] + [0, 1] + >>> path[3] + [4, 3] + + >>> length, path = nx.multi_source_dijkstra(G, {0, 4}, 1) + >>> length + 1 + >>> path + [0, 1] + + Notes + ----- + Edge weight attributes must be numerical. + Distances are calculated as sums of weighted edges traversed. + + The weight function can be used to hide edges by returning None. + So ``weight = lambda u, v, d: 1 if d['color']=="red" else None`` + will find the shortest red path. + + Based on the Python cookbook recipe (119466) at + https://code.activestate.com/recipes/119466/ + + This algorithm is not guaranteed to work if edge weights + are negative or are floating point numbers + (overflows and roundoff errors can cause problems). + + Raises + ------ + ValueError + If `sources` is empty. + NodeNotFound + If any of `sources` is not in `G`. + + See Also + -------- + multi_source_dijkstra_path + multi_source_dijkstra_path_length + + """ + if not sources: + raise ValueError("sources must not be empty") + for s in sources: + if s not in G: + raise nx.NodeNotFound(f"Node {s} not found in graph") + if target in sources: + return (0, [target]) + weight = _weight_function(G, weight) + paths = {source: [source] for source in sources} # dictionary of paths + dist = _dijkstra_multisource( + G, sources, weight, paths=paths, cutoff=cutoff, target=target + ) + if target is None: + return (dist, paths) + try: + return (dist[target], paths[target]) + except KeyError as err: + raise nx.NetworkXNoPath(f"No path to {target}.") from err + + +def _dijkstra(G, source, weight, pred=None, paths=None, cutoff=None, target=None): + """Uses Dijkstra's algorithm to find shortest weighted paths from a + single source. + + This is a convenience function for :func:`_dijkstra_multisource` + with all the arguments the same, except the keyword argument + `sources` set to ``[source]``. + + """ + return _dijkstra_multisource( + G, [source], weight, pred=pred, paths=paths, cutoff=cutoff, target=target + ) + + +def _dijkstra_multisource( + G, sources, weight, pred=None, paths=None, cutoff=None, target=None +): + """Uses Dijkstra's algorithm to find shortest weighted paths + + Parameters + ---------- + G : NetworkX graph + + sources : non-empty iterable of nodes + Starting nodes for paths. If this is just an iterable containing + a single node, then all paths computed by this function will + start from that node. If there are two or more nodes in this + iterable, the computed paths may begin from any one of the start + nodes. + + weight: function + Function with (u, v, data) input that returns that edge's weight + or None to indicate a hidden edge + + pred: dict of lists, optional(default=None) + dict to store a list of predecessors keyed by that node + If None, predecessors are not stored. + + paths: dict, optional (default=None) + dict to store the path list from source to each node, keyed by node. + If None, paths are not stored. + + target : node label, optional + Ending node for path. Search is halted when target is found. + + cutoff : integer or float, optional + Length (sum of edge weights) at which the search is stopped. + If cutoff is provided, only return paths with summed weight <= cutoff. + + Returns + ------- + distance : dictionary + A mapping from node to shortest distance to that node from one + of the source nodes. + + Raises + ------ + NodeNotFound + If any of `sources` is not in `G`. + + Notes + ----- + The optional predecessor and path dictionaries can be accessed by + the caller through the original pred and paths objects passed + as arguments. No need to explicitly return pred or paths. + + """ + # If `paths` is specified, we use a temporary internal dictionary (`pred_dict`) to + # store predecessors, used to reconstruct paths. However, if the caller + # passed in a `pred` dictionary, we must compute *all* predecessors, since the caller + # expects the full predecessor structure. + pred_dict = pred if paths is None or pred is not None else {} + + G_succ = G._adj # For speed-up (and works for both directed and undirected graphs) + + dist = {} # dictionary of final distances + seen = {} + # fringe is heapq with 3-tuples (distance,c,node) + # use the count c to avoid comparing nodes (may not be able to) + c = count() + fringe = [] + for source in sources: + seen[source] = 0 + heappush(fringe, (0, next(c), source)) + number_of_sources = len(seen) + while fringe: + (dist_v, _, v) = heappop(fringe) + if v in dist: + continue # already searched this node. + dist[v] = dist_v + if v == target: + break + for u, e in G_succ[v].items(): + cost = weight(v, u, e) + if cost is None: + continue + vu_dist = dist_v + cost + if cutoff is not None and vu_dist > cutoff: + continue + if u in dist: + u_dist = dist[u] + if vu_dist < u_dist: + raise ValueError("Contradictory paths found:", "negative weights?") + elif pred is not None and vu_dist == u_dist: + # Found another shortest path to u with equal distance (including zero-weight edges). + # We must store *all* predecessors because `pred` was provided by the caller. + pred_dict[u].append(v) + elif u not in seen or vu_dist < seen[u]: + seen[u] = vu_dist + heappush(fringe, (vu_dist, next(c), u)) + if pred_dict is not None: + pred_dict[u] = [v] + elif pred is not None and vu_dist == seen[u]: + # Found another shortest path to u + # We must store *all* predecessors because `pred` was provided by the caller. + pred_dict[u].append(v) + + if paths is not None: + # Reconstruct the path from source to target using the predecessor dictionary. + if target is None: + # Since `dist` is in increasing distance order, each predecessor's path is + # already computed by the time we process `v`. We skip the first + # `number_of_sources` entries because sources already have their paths defined. + for v in islice(dist, number_of_sources, None): + # `v` must be in `pred_dict`: any node with a distance (and not a source) + # has a predecessor. + paths[v] = paths[pred_dict[v][0]] + [v] + else: + # Caller requested the path to a specific target node. + path = paths[target] = [target] + while (current_preds := pred_dict.get(path[-1])) is not None: + path.append(current_preds[0]) + # The path was built in reverse order, so reverse it at the end. + path.reverse() + + # The optional predecessor and path dictionaries can be accessed + # by the caller via the pred and paths objects passed as arguments. + return dist + + +@nx._dispatchable(edge_attrs="weight") +def dijkstra_predecessor_and_distance(G, source, cutoff=None, weight="weight"): + """Compute weighted shortest path length and predecessors. + + Uses Dijkstra's Method to obtain the shortest weighted paths + and return dictionaries of predecessors for each node and + distance for each node from the `source`. + + Parameters + ---------- + G : NetworkX graph + + source : node label + Starting node for path + + cutoff : integer or float, optional + Length (sum of edge weights) at which the search is stopped. + If cutoff is provided, only return paths with summed weight <= cutoff. + + weight : string or function + If this is a string, then edge weights will be accessed via the + edge attribute with this key (that is, the weight of the edge + joining `u` to `v` will be ``G.edges[u, v][weight]``). If no + such edge attribute exists, the weight of the edge is assumed to + be one. + + If this is a function, the weight of an edge is the value + returned by the function. The function must accept exactly three + positional arguments: the two endpoints of an edge and the + dictionary of edge attributes for that edge. The function must + return a number or None to indicate a hidden edge. + + Returns + ------- + pred, distance : dictionaries + Returns two dictionaries representing a list of predecessors + of a node and the distance to each node. + + Raises + ------ + NodeNotFound + If `source` is not in `G`. + + Notes + ----- + Edge weight attributes must be numerical. + Distances are calculated as sums of weighted edges traversed. + + The list of predecessors contains more than one element only when + there are more than one shortest paths to the key node. + + Examples + -------- + >>> G = nx.path_graph(5, create_using=nx.DiGraph()) + >>> pred, dist = nx.dijkstra_predecessor_and_distance(G, 0) + >>> sorted(pred.items()) + [(0, []), (1, [0]), (2, [1]), (3, [2]), (4, [3])] + >>> sorted(dist.items()) + [(0, 0), (1, 1), (2, 2), (3, 3), (4, 4)] + + >>> pred, dist = nx.dijkstra_predecessor_and_distance(G, 0, 1) + >>> sorted(pred.items()) + [(0, []), (1, [0])] + >>> sorted(dist.items()) + [(0, 0), (1, 1)] + """ + if source not in G: + raise nx.NodeNotFound(f"Node {source} is not found in the graph") + weight = _weight_function(G, weight) + pred = {source: []} # dictionary of predecessors + return (pred, _dijkstra(G, source, weight, pred=pred, cutoff=cutoff)) + + +@nx._dispatchable(edge_attrs="weight") +def all_pairs_dijkstra(G, cutoff=None, weight="weight"): + """Find shortest weighted paths and lengths between all nodes. + + Parameters + ---------- + G : NetworkX graph + + cutoff : integer or float, optional + Length (sum of edge weights) at which the search is stopped. + If cutoff is provided, only return paths with summed weight <= cutoff. + + weight : string or function + If this is a string, then edge weights will be accessed via the + edge attribute with this key (that is, the weight of the edge + joining `u` to `v` will be ``G.edge[u][v][weight]``). If no + such edge attribute exists, the weight of the edge is assumed to + be one. + + If this is a function, the weight of an edge is the value + returned by the function. The function must accept exactly three + positional arguments: the two endpoints of an edge and the + dictionary of edge attributes for that edge. The function must + return a number or None to indicate a hidden edge. + + Yields + ------ + (node, (distance, path)) : (node obj, (dict, dict)) + Each source node has two associated dicts. The first holds distance + keyed by target and the second holds paths keyed by target. + (See single_source_dijkstra for the source/target node terminology.) + If desired you can apply `dict()` to this function to create a dict + keyed by source node to the two dicts. + + Examples + -------- + >>> G = nx.path_graph(5) + >>> len_path = dict(nx.all_pairs_dijkstra(G)) + >>> len_path[3][0][1] + 2 + >>> for node in [0, 1, 2, 3, 4]: + ... print(f"3 - {node}: {len_path[3][0][node]}") + 3 - 0: 3 + 3 - 1: 2 + 3 - 2: 1 + 3 - 3: 0 + 3 - 4: 1 + >>> len_path[3][1][1] + [3, 2, 1] + >>> for n, (dist, path) in nx.all_pairs_dijkstra(G): + ... print(path[1]) + [0, 1] + [1] + [2, 1] + [3, 2, 1] + [4, 3, 2, 1] + + Notes + ----- + Edge weight attributes must be numerical. + Distances are calculated as sums of weighted edges traversed. + + The yielded dicts only have keys for reachable nodes. + """ + for n in G: + dist, path = single_source_dijkstra(G, n, cutoff=cutoff, weight=weight) + yield (n, (dist, path)) + + +@nx._dispatchable(edge_attrs="weight") +def all_pairs_dijkstra_path_length(G, cutoff=None, weight="weight"): + """Compute shortest path lengths between all nodes in a weighted graph. + + Parameters + ---------- + G : NetworkX graph + + cutoff : integer or float, optional + Length (sum of edge weights) at which the search is stopped. + If cutoff is provided, only return paths with summed weight <= cutoff. + + weight : string or function + If this is a string, then edge weights will be accessed via the + edge attribute with this key (that is, the weight of the edge + joining `u` to `v` will be ``G.edges[u, v][weight]``). If no + such edge attribute exists, the weight of the edge is assumed to + be one. + + If this is a function, the weight of an edge is the value + returned by the function. The function must accept exactly three + positional arguments: the two endpoints of an edge and the + dictionary of edge attributes for that edge. The function must + return a number or None to indicate a hidden edge. + + Returns + ------- + distance : iterator + (source, dictionary) iterator with dictionary keyed by target and + shortest path length as the key value. + + Examples + -------- + >>> G = nx.path_graph(5) + >>> length = dict(nx.all_pairs_dijkstra_path_length(G)) + >>> for node in [0, 1, 2, 3, 4]: + ... print(f"1 - {node}: {length[1][node]}") + 1 - 0: 1 + 1 - 1: 0 + 1 - 2: 1 + 1 - 3: 2 + 1 - 4: 3 + >>> length[3][2] + 1 + >>> length[2][2] + 0 + + Notes + ----- + Edge weight attributes must be numerical. + Distances are calculated as sums of weighted edges traversed. + + The dictionary returned only has keys for reachable node pairs. + """ + length = single_source_dijkstra_path_length + for n in G: + yield (n, length(G, n, cutoff=cutoff, weight=weight)) + + +@nx._dispatchable(edge_attrs="weight") +def all_pairs_dijkstra_path(G, cutoff=None, weight="weight"): + """Compute shortest paths between all nodes in a weighted graph. + + Parameters + ---------- + G : NetworkX graph + + cutoff : integer or float, optional + Length (sum of edge weights) at which the search is stopped. + If cutoff is provided, only return paths with summed weight <= cutoff. + + weight : string or function + If this is a string, then edge weights will be accessed via the + edge attribute with this key (that is, the weight of the edge + joining `u` to `v` will be ``G.edges[u, v][weight]``). If no + such edge attribute exists, the weight of the edge is assumed to + be one. + + If this is a function, the weight of an edge is the value + returned by the function. The function must accept exactly three + positional arguments: the two endpoints of an edge and the + dictionary of edge attributes for that edge. The function must + return a number or None to indicate a hidden edge. + + Returns + ------- + paths : iterator + (source, dictionary) iterator with dictionary keyed by target and + shortest path as the key value. + + Examples + -------- + >>> G = nx.path_graph(5) + >>> path = dict(nx.all_pairs_dijkstra_path(G)) + >>> path[0][4] + [0, 1, 2, 3, 4] + + Notes + ----- + Edge weight attributes must be numerical. + Distances are calculated as sums of weighted edges traversed. + + See Also + -------- + floyd_warshall, all_pairs_bellman_ford_path + + """ + path = single_source_dijkstra_path + # TODO This can be trivially parallelized. + for n in G: + yield (n, path(G, n, cutoff=cutoff, weight=weight)) + + +@nx._dispatchable(edge_attrs="weight") +def bellman_ford_predecessor_and_distance( + G, source, target=None, weight="weight", heuristic=False +): + """Compute shortest path lengths and predecessors on shortest paths + in weighted graphs. + + The algorithm has a running time of $O(mn)$ where $n$ is the number of + nodes and $m$ is the number of edges. It is slower than Dijkstra but + can handle negative edge weights. + + If a negative cycle is detected, you can use :func:`find_negative_cycle` + to return the cycle and examine it. Shortest paths are not defined when + a negative cycle exists because once reached, the path can cycle forever + to build up arbitrarily low weights. + + Parameters + ---------- + G : NetworkX graph + The algorithm works for all types of graphs, including directed + graphs and multigraphs. + + source: node label + Starting node for path + + target : node label, optional + Ending node for path + + weight : string or function + If this is a string, then edge weights will be accessed via the + edge attribute with this key (that is, the weight of the edge + joining `u` to `v` will be ``G.edges[u, v][weight]``). If no + such edge attribute exists, the weight of the edge is assumed to + be one. + + If this is a function, the weight of an edge is the value + returned by the function. The function must accept exactly three + positional arguments: the two endpoints of an edge and the + dictionary of edge attributes for that edge. The function must + return a number. + + heuristic : bool + Determines whether to use a heuristic to early detect negative + cycles at a hopefully negligible cost. + + Returns + ------- + pred, dist : dictionaries + Returns two dictionaries keyed by node to predecessor in the + path and to the distance from the source respectively. + + Raises + ------ + NodeNotFound + If `source` is not in `G`. + + NetworkXUnbounded + If the (di)graph contains a negative (di)cycle, the + algorithm raises an exception to indicate the presence of the + negative (di)cycle. Note: any negative weight edge in an + undirected graph is a negative cycle. + + Examples + -------- + >>> G = nx.path_graph(5, create_using=nx.DiGraph()) + >>> pred, dist = nx.bellman_ford_predecessor_and_distance(G, 0) + >>> sorted(pred.items()) + [(0, []), (1, [0]), (2, [1]), (3, [2]), (4, [3])] + >>> sorted(dist.items()) + [(0, 0), (1, 1), (2, 2), (3, 3), (4, 4)] + + >>> pred, dist = nx.bellman_ford_predecessor_and_distance(G, 0, 1) + >>> sorted(pred.items()) + [(0, []), (1, [0]), (2, [1]), (3, [2]), (4, [3])] + >>> sorted(dist.items()) + [(0, 0), (1, 1), (2, 2), (3, 3), (4, 4)] + + >>> G = nx.cycle_graph(5, create_using=nx.DiGraph()) + >>> G[1][2]["weight"] = -7 + >>> nx.bellman_ford_predecessor_and_distance(G, 0) + Traceback (most recent call last): + ... + networkx.exception.NetworkXUnbounded: Negative cycle detected. + + See Also + -------- + find_negative_cycle + + Notes + ----- + Edge weight attributes must be numerical. + Distances are calculated as sums of weighted edges traversed. + + The dictionaries returned only have keys for nodes reachable from + the source. + + In the case where the (di)graph is not connected, if a component + not containing the source contains a negative (di)cycle, it + will not be detected. + + In NetworkX v2.1 and prior, the source node had predecessor `[None]`. + In NetworkX v2.2 this changed to the source node having predecessor `[]` + """ + if source not in G: + raise nx.NodeNotFound(f"Node {source} is not found in the graph") + weight = _weight_function(G, weight) + if G.is_multigraph(): + if any( + weight(u, v, {k: d}) < 0 + for u, v, k, d in nx.selfloop_edges(G, keys=True, data=True) + ): + raise nx.NetworkXUnbounded("Negative cycle detected.") + else: + if any(weight(u, v, d) < 0 for u, v, d in nx.selfloop_edges(G, data=True)): + raise nx.NetworkXUnbounded("Negative cycle detected.") + + dist = {source: 0} + pred = {source: []} + + if len(G) == 1: + return pred, dist + + weight = _weight_function(G, weight) + + dist = _bellman_ford( + G, [source], weight, pred=pred, dist=dist, target=target, heuristic=heuristic + ) + return (pred, dist) + + +def _bellman_ford( + G, + source, + weight, + pred=None, + paths=None, + dist=None, + target=None, + heuristic=True, +): + """Calls relaxation loop for Bellman–Ford algorithm and builds paths + + This is an implementation of the SPFA variant. + See https://en.wikipedia.org/wiki/Shortest_Path_Faster_Algorithm + + Parameters + ---------- + G : NetworkX graph + + source: list + List of source nodes. The shortest path from any of the source + nodes will be found if multiple sources are provided. + + weight : function + The weight of an edge is the value returned by the function. The + function must accept exactly three positional arguments: the two + endpoints of an edge and the dictionary of edge attributes for + that edge. The function must return a number. + + pred: dict of lists, optional (default=None) + dict to store a list of predecessors keyed by that node + If None, predecessors are not stored + + paths: dict, optional (default=None) + dict to store the path list from source to each node, keyed by node + If None, paths are not stored + + dist: dict, optional (default=None) + dict to store distance from source to the keyed node + If None, returned dist dict contents default to 0 for every node in the + source list + + target: node label, optional + Ending node for path. Path lengths to other destinations may (and + probably will) be incorrect. + + heuristic : bool + Determines whether to use a heuristic to early detect negative + cycles at a hopefully negligible cost. + + Returns + ------- + dist : dict + Returns a dict keyed by node to the distance from the source. + Dicts for paths and pred are in the mutated input dicts by those names. + + Raises + ------ + NodeNotFound + If any of `source` is not in `G`. + + NetworkXUnbounded + If the (di)graph contains a negative (di)cycle, the + algorithm raises an exception to indicate the presence of the + negative (di)cycle. Note: any negative weight edge in an + undirected graph is a negative cycle + """ + if pred is None: + pred = {v: [] for v in source} + + if dist is None: + dist = {v: 0 for v in source} + + negative_cycle_found = _inner_bellman_ford( + G, + source, + weight, + pred, + dist, + heuristic, + ) + if negative_cycle_found is not None: + raise nx.NetworkXUnbounded("Negative cycle detected.") + + if paths is not None: + sources = set(source) + dsts = [target] if target is not None else pred + for dst in dsts: + gen = _build_paths_from_predecessors(sources, dst, pred) + paths[dst] = next(gen) + + return dist + + +def _inner_bellman_ford( + G, + sources, + weight, + pred, + dist=None, + heuristic=True, +): + """Inner Relaxation loop for Bellman–Ford algorithm. + + This is an implementation of the SPFA variant. + See https://en.wikipedia.org/wiki/Shortest_Path_Faster_Algorithm + + Parameters + ---------- + G : NetworkX graph + + source: list + List of source nodes. The shortest path from any of the source + nodes will be found if multiple sources are provided. + + weight : function + The weight of an edge is the value returned by the function. The + function must accept exactly three positional arguments: the two + endpoints of an edge and the dictionary of edge attributes for + that edge. The function must return a number. + + pred: dict of lists + dict to store a list of predecessors keyed by that node + + dist: dict, optional (default=None) + dict to store distance from source to the keyed node + If None, returned dist dict contents default to 0 for every node in the + source list + + heuristic : bool + Determines whether to use a heuristic to early detect negative + cycles at a hopefully negligible cost. + + Returns + ------- + node or None + Return a node `v` where processing discovered a negative cycle. + If no negative cycle found, return None. + + Raises + ------ + NodeNotFound + If any of `source` is not in `G`. + """ + for s in sources: + if s not in G: + raise nx.NodeNotFound(f"Source {s} not in G") + + if pred is None: + pred = {v: [] for v in sources} + + if dist is None: + dist = {v: 0 for v in sources} + + # Heuristic Storage setup. Note: use None because nodes cannot be None + nonexistent_edge = (None, None) + pred_edge = {v: None for v in sources} + recent_update = {v: nonexistent_edge for v in sources} + + G_succ = G._adj # For speed-up (and works for both directed and undirected graphs) + inf = float("inf") + n = len(G) + + count = {} + q = deque(sources) + in_q = set(sources) + while q: + u = q.popleft() + in_q.remove(u) + + # Skip relaxations if any of the predecessors of u is in the queue. + if all(pred_u not in in_q for pred_u in pred[u]): + dist_u = dist[u] + for v, e in G_succ[u].items(): + dist_v = dist_u + weight(u, v, e) + + if dist_v < dist.get(v, inf): + # In this conditional branch we are updating the path with v. + # If it happens that some earlier update also added node v + # that implies the existence of a negative cycle since + # after the update node v would lie on the update path twice. + # The update path is stored up to one of the source nodes, + # therefore u is always in the dict recent_update + if heuristic: + if v in recent_update[u]: + # Negative cycle found! + pred[v].append(u) + return v + + # Transfer the recent update info from u to v if the + # same source node is the head of the update path. + # If the source node is responsible for the cost update, + # then clear the history and use it instead. + if v in pred_edge and pred_edge[v] == u: + recent_update[v] = recent_update[u] + else: + recent_update[v] = (u, v) + + if v not in in_q: + q.append(v) + in_q.add(v) + count_v = count.get(v, 0) + 1 + if count_v == n: + # Negative cycle found! + return v + + count[v] = count_v + dist[v] = dist_v + pred[v] = [u] + pred_edge[v] = u + + elif dist.get(v) is not None and dist_v == dist.get(v): + pred[v].append(u) + + # successfully found shortest_path. No negative cycles found. + return None + + +@nx._dispatchable(edge_attrs="weight") +def bellman_ford_path(G, source, target, weight="weight"): + """Returns the shortest path from source to target in a weighted graph G. + + Parameters + ---------- + G : NetworkX graph + + source : node + Starting node + + target : node + Ending node + + weight : string or function (default="weight") + If this is a string, then edge weights will be accessed via the + edge attribute with this key (that is, the weight of the edge + joining `u` to `v` will be ``G.edges[u, v][weight]``). If no + such edge attribute exists, the weight of the edge is assumed to + be one. + + If this is a function, the weight of an edge is the value + returned by the function. The function must accept exactly three + positional arguments: the two endpoints of an edge and the + dictionary of edge attributes for that edge. The function must + return a number. + + Returns + ------- + path : list + List of nodes in a shortest path. + + Raises + ------ + NodeNotFound + If `source` is not in `G`. + + NetworkXNoPath + If no path exists between source and target. + + Examples + -------- + >>> G = nx.path_graph(5) + >>> nx.bellman_ford_path(G, 0, 4) + [0, 1, 2, 3, 4] + + Notes + ----- + Edge weight attributes must be numerical. + Distances are calculated as sums of weighted edges traversed. + + See Also + -------- + dijkstra_path, bellman_ford_path_length + """ + length, path = single_source_bellman_ford(G, source, target=target, weight=weight) + return path + + +@nx._dispatchable(edge_attrs="weight") +def bellman_ford_path_length(G, source, target, weight="weight"): + """Returns the shortest path length from source to target + in a weighted graph. + + Parameters + ---------- + G : NetworkX graph + + source : node label + starting node for path + + target : node label + ending node for path + + weight : string or function (default="weight") + If this is a string, then edge weights will be accessed via the + edge attribute with this key (that is, the weight of the edge + joining `u` to `v` will be ``G.edges[u, v][weight]``). If no + such edge attribute exists, the weight of the edge is assumed to + be one. + + If this is a function, the weight of an edge is the value + returned by the function. The function must accept exactly three + positional arguments: the two endpoints of an edge and the + dictionary of edge attributes for that edge. The function must + return a number. + + Returns + ------- + length : number + Shortest path length. + + Raises + ------ + NodeNotFound + If `source` is not in `G`. + + NetworkXNoPath + If no path exists between source and target. + + Examples + -------- + >>> G = nx.path_graph(5) + >>> nx.bellman_ford_path_length(G, 0, 4) + 4 + + Notes + ----- + Edge weight attributes must be numerical. + Distances are calculated as sums of weighted edges traversed. + + See Also + -------- + dijkstra_path_length, bellman_ford_path + """ + if source == target: + if source not in G: + raise nx.NodeNotFound(f"Node {source} not found in graph") + return 0 + + weight = _weight_function(G, weight) + + length = _bellman_ford(G, [source], weight, target=target) + + try: + return length[target] + except KeyError as err: + raise nx.NetworkXNoPath(f"node {target} not reachable from {source}") from err + + +@nx._dispatchable(edge_attrs="weight") +def single_source_bellman_ford_path(G, source, weight="weight"): + """Compute shortest path between source and all other reachable + nodes for a weighted graph. + + Parameters + ---------- + G : NetworkX graph + + source : node + Starting node for path. + + weight : string or function (default="weight") + If this is a string, then edge weights will be accessed via the + edge attribute with this key (that is, the weight of the edge + joining `u` to `v` will be ``G.edges[u, v][weight]``). If no + such edge attribute exists, the weight of the edge is assumed to + be one. + + If this is a function, the weight of an edge is the value + returned by the function. The function must accept exactly three + positional arguments: the two endpoints of an edge and the + dictionary of edge attributes for that edge. The function must + return a number. + + Returns + ------- + paths : dictionary + Dictionary of shortest path lengths keyed by target. + + Raises + ------ + NodeNotFound + If `source` is not in `G`. + + Examples + -------- + >>> G = nx.path_graph(5) + >>> path = nx.single_source_bellman_ford_path(G, 0) + >>> path[4] + [0, 1, 2, 3, 4] + + Notes + ----- + Edge weight attributes must be numerical. + Distances are calculated as sums of weighted edges traversed. + + See Also + -------- + single_source_dijkstra, single_source_bellman_ford + + """ + (length, path) = single_source_bellman_ford(G, source, weight=weight) + return path + + +@nx._dispatchable(edge_attrs="weight") +def single_source_bellman_ford_path_length(G, source, weight="weight"): + """Compute the shortest path length between source and all other + reachable nodes for a weighted graph. + + Parameters + ---------- + G : NetworkX graph + + source : node label + Starting node for path + + weight : string or function (default="weight") + If this is a string, then edge weights will be accessed via the + edge attribute with this key (that is, the weight of the edge + joining `u` to `v` will be ``G.edges[u, v][weight]``). If no + such edge attribute exists, the weight of the edge is assumed to + be one. + + If this is a function, the weight of an edge is the value + returned by the function. The function must accept exactly three + positional arguments: the two endpoints of an edge and the + dictionary of edge attributes for that edge. The function must + return a number. + + Returns + ------- + length : dictionary + Dictionary of shortest path length keyed by target + + Raises + ------ + NodeNotFound + If `source` is not in `G`. + + Examples + -------- + >>> G = nx.path_graph(5) + >>> length = nx.single_source_bellman_ford_path_length(G, 0) + >>> length[4] + 4 + >>> for node in [0, 1, 2, 3, 4]: + ... print(f"{node}: {length[node]}") + 0: 0 + 1: 1 + 2: 2 + 3: 3 + 4: 4 + + Notes + ----- + Edge weight attributes must be numerical. + Distances are calculated as sums of weighted edges traversed. + + See Also + -------- + single_source_dijkstra, single_source_bellman_ford + + """ + weight = _weight_function(G, weight) + return _bellman_ford(G, [source], weight) + + +@nx._dispatchable(edge_attrs="weight") +def single_source_bellman_ford(G, source, target=None, weight="weight"): + """Compute shortest paths and lengths in a weighted graph G. + + Uses Bellman-Ford algorithm for shortest paths. + + Parameters + ---------- + G : NetworkX graph + + source : node label + Starting node for path + + target : node label, optional + Ending node for path + + weight : string or function + If this is a string, then edge weights will be accessed via the + edge attribute with this key (that is, the weight of the edge + joining `u` to `v` will be ``G.edges[u, v][weight]``). If no + such edge attribute exists, the weight of the edge is assumed to + be one. + + If this is a function, the weight of an edge is the value + returned by the function. The function must accept exactly three + positional arguments: the two endpoints of an edge and the + dictionary of edge attributes for that edge. The function must + return a number. + + Returns + ------- + distance, path : pair of dictionaries, or numeric and list + If target is None, returns a tuple of two dictionaries keyed by node. + The first dictionary stores distance from one of the source nodes. + The second stores the path from one of the sources to that node. + If target is not None, returns a tuple of (distance, path) where + distance is the distance from source to target and path is a list + representing the path from source to target. + + Raises + ------ + NodeNotFound + If `source` is not in `G`. + + Examples + -------- + >>> G = nx.path_graph(5) + >>> length, path = nx.single_source_bellman_ford(G, 0) + >>> length[4] + 4 + >>> for node in [0, 1, 2, 3, 4]: + ... print(f"{node}: {length[node]}") + 0: 0 + 1: 1 + 2: 2 + 3: 3 + 4: 4 + >>> path[4] + [0, 1, 2, 3, 4] + >>> length, path = nx.single_source_bellman_ford(G, 0, 1) + >>> length + 1 + >>> path + [0, 1] + + Notes + ----- + Edge weight attributes must be numerical. + Distances are calculated as sums of weighted edges traversed. + + See Also + -------- + single_source_dijkstra + single_source_bellman_ford_path + single_source_bellman_ford_path_length + """ + if source == target: + if source not in G: + raise nx.NodeNotFound(f"Node {source} is not found in the graph") + return (0, [source]) + + weight = _weight_function(G, weight) + + paths = {source: [source]} # dictionary of paths + dist = _bellman_ford(G, [source], weight, paths=paths, target=target) + if target is None: + return (dist, paths) + try: + return (dist[target], paths[target]) + except KeyError as err: + msg = f"Node {target} not reachable from {source}" + raise nx.NetworkXNoPath(msg) from err + + +@nx._dispatchable(edge_attrs="weight") +def all_pairs_bellman_ford_path_length(G, weight="weight"): + """Compute shortest path lengths between all nodes in a weighted graph. + + Parameters + ---------- + G : NetworkX graph + + weight : string or function (default="weight") + If this is a string, then edge weights will be accessed via the + edge attribute with this key (that is, the weight of the edge + joining `u` to `v` will be ``G.edges[u, v][weight]``). If no + such edge attribute exists, the weight of the edge is assumed to + be one. + + If this is a function, the weight of an edge is the value + returned by the function. The function must accept exactly three + positional arguments: the two endpoints of an edge and the + dictionary of edge attributes for that edge. The function must + return a number. + + Returns + ------- + distance : iterator + (source, dictionary) iterator with dictionary keyed by target and + shortest path length as the key value. + + Examples + -------- + >>> G = nx.path_graph(5) + >>> length = dict(nx.all_pairs_bellman_ford_path_length(G)) + >>> for node in [0, 1, 2, 3, 4]: + ... print(f"1 - {node}: {length[1][node]}") + 1 - 0: 1 + 1 - 1: 0 + 1 - 2: 1 + 1 - 3: 2 + 1 - 4: 3 + >>> length[3][2] + 1 + >>> length[2][2] + 0 + + Notes + ----- + Edge weight attributes must be numerical. + Distances are calculated as sums of weighted edges traversed. + + The dictionary returned only has keys for reachable node pairs. + """ + length = single_source_bellman_ford_path_length + for n in G: + yield (n, dict(length(G, n, weight=weight))) + + +@nx._dispatchable(edge_attrs="weight") +def all_pairs_bellman_ford_path(G, weight="weight"): + """Compute shortest paths between all nodes in a weighted graph. + + Parameters + ---------- + G : NetworkX graph + + weight : string or function (default="weight") + If this is a string, then edge weights will be accessed via the + edge attribute with this key (that is, the weight of the edge + joining `u` to `v` will be ``G.edges[u, v][weight]``). If no + such edge attribute exists, the weight of the edge is assumed to + be one. + + If this is a function, the weight of an edge is the value + returned by the function. The function must accept exactly three + positional arguments: the two endpoints of an edge and the + dictionary of edge attributes for that edge. The function must + return a number. + + Returns + ------- + paths : iterator + (source, dictionary) iterator with dictionary keyed by target and + shortest path as the key value. + + Examples + -------- + >>> G = nx.path_graph(5) + >>> path = dict(nx.all_pairs_bellman_ford_path(G)) + >>> path[0][4] + [0, 1, 2, 3, 4] + + Notes + ----- + Edge weight attributes must be numerical. + Distances are calculated as sums of weighted edges traversed. + + See Also + -------- + floyd_warshall, all_pairs_dijkstra_path + + """ + path = single_source_bellman_ford_path + for n in G: + yield (n, path(G, n, weight=weight)) + + +@nx._dispatchable(edge_attrs="weight") +def goldberg_radzik(G, source, weight="weight"): + """Compute shortest path lengths and predecessors on shortest paths + in weighted graphs. + + The algorithm has a running time of $O(mn)$ where $n$ is the number of + nodes and $m$ is the number of edges. It is slower than Dijkstra but + can handle negative edge weights. + + Parameters + ---------- + G : NetworkX graph + The algorithm works for all types of graphs, including directed + graphs and multigraphs. + + source: node label + Starting node for path + + weight : string or function + If this is a string, then edge weights will be accessed via the + edge attribute with this key (that is, the weight of the edge + joining `u` to `v` will be ``G.edges[u, v][weight]``). If no + such edge attribute exists, the weight of the edge is assumed to + be one. + + If this is a function, the weight of an edge is the value + returned by the function. The function must accept exactly three + positional arguments: the two endpoints of an edge and the + dictionary of edge attributes for that edge. The function must + return a number. + + Returns + ------- + pred, dist : dictionaries + Returns two dictionaries keyed by node to predecessor in the + path and to the distance from the source respectively. + + Raises + ------ + NodeNotFound + If `source` is not in `G`. + + NetworkXUnbounded + If the (di)graph contains a negative (di)cycle, the + algorithm raises an exception to indicate the presence of the + negative (di)cycle. Note: any negative weight edge in an + undirected graph is a negative cycle. + + As of NetworkX v3.2, a zero weight cycle is no longer + incorrectly reported as a negative weight cycle. + + + Examples + -------- + >>> G = nx.path_graph(5, create_using=nx.DiGraph()) + >>> pred, dist = nx.goldberg_radzik(G, 0) + >>> sorted(pred.items()) + [(0, None), (1, 0), (2, 1), (3, 2), (4, 3)] + >>> sorted(dist.items()) + [(0, 0), (1, 1), (2, 2), (3, 3), (4, 4)] + + >>> G = nx.cycle_graph(5, create_using=nx.DiGraph()) + >>> G[1][2]["weight"] = -7 + >>> nx.goldberg_radzik(G, 0) + Traceback (most recent call last): + ... + networkx.exception.NetworkXUnbounded: Negative cycle detected. + + Notes + ----- + Edge weight attributes must be numerical. + Distances are calculated as sums of weighted edges traversed. + + The dictionaries returned only have keys for nodes reachable from + the source. + + In the case where the (di)graph is not connected, if a component + not containing the source contains a negative (di)cycle, it + will not be detected. + + """ + if source not in G: + raise nx.NodeNotFound(f"Node {source} is not found in the graph") + weight = _weight_function(G, weight) + if G.is_multigraph(): + if any( + weight(u, v, {k: d}) < 0 + for u, v, k, d in nx.selfloop_edges(G, keys=True, data=True) + ): + raise nx.NetworkXUnbounded("Negative cycle detected.") + else: + if any(weight(u, v, d) < 0 for u, v, d in nx.selfloop_edges(G, data=True)): + raise nx.NetworkXUnbounded("Negative cycle detected.") + + if len(G) == 1: + return {source: None}, {source: 0} + + G_succ = G._adj # For speed-up (and works for both directed and undirected graphs) + + inf = float("inf") + d = {u: inf for u in G} + d[source] = 0 + pred = {source: None} + + def topo_sort(relabeled): + """Topologically sort nodes relabeled in the previous round and detect + negative cycles. + """ + # List of nodes to scan in this round. Denoted by A in Goldberg and + # Radzik's paper. + to_scan = [] + # In the DFS in the loop below, neg_count records for each node the + # number of edges of negative reduced costs on the path from a DFS root + # to the node in the DFS forest. The reduced cost of an edge (u, v) is + # defined as d[u] + weight[u][v] - d[v]. + # + # neg_count also doubles as the DFS visit marker array. + neg_count = {} + for u in relabeled - neg_count.keys(): + d_u = d[u] + # Skip nodes without out-edges of negative reduced costs. + if all(d_u + weight(u, v, e) >= d[v] for v, e in G_succ[u].items()): + continue + # Nonrecursive DFS that inserts nodes reachable from u via edges of + # nonpositive reduced costs into to_scan in (reverse) topological + # order. + stack = [(u, iter(G_succ[u].items()))] + in_stack = {u} + neg_count[u] = 0 + while stack: + u, it = stack[-1] + try: + v, e = next(it) + except StopIteration: + to_scan.append(u) + stack.pop() + in_stack.remove(u) + continue + t = d[u] + weight(u, v, e) + d_v = d[v] + if t < d_v: + d[v] = t + pred[v] = u + if v not in neg_count: + neg_count[v] = neg_count[u] + 1 + stack.append((v, iter(G_succ[v].items()))) + in_stack.add(v) + elif v in in_stack and neg_count[u] + 1 > neg_count[v]: + # (u, v) is a back edge, and the cycle formed by the + # path v to u and (u, v) contains at least one edge of + # negative reduced cost. The cycle must be of negative + # cost. + raise nx.NetworkXUnbounded("Negative cycle detected.") + to_scan.reverse() + return to_scan + + def relax(to_scan): + """Relax out-edges of relabeled nodes.""" + relabeled = set() + # Scan nodes in to_scan in topological order and relax incident + # out-edges. Add the relabeled nodes to labeled. + for u in to_scan: + d_u = d[u] + for v, e in G_succ[u].items(): + w_e = weight(u, v, e) + if d_u + w_e < d[v]: + d[v] = d_u + w_e + pred[v] = u + relabeled.add(v) + return relabeled + + # Set of nodes relabeled in the last round of scan operations. Denoted by B + # in Goldberg and Radzik's paper. + relabeled = {source} + + while relabeled: + to_scan = topo_sort(relabeled) + relabeled = relax(to_scan) + + d = {u: d[u] for u in pred} + return pred, d + + +@nx._dispatchable(edge_attrs="weight") +def negative_edge_cycle(G, weight="weight", heuristic=True): + """Returns True if there exists a negative edge cycle anywhere in G. + + Parameters + ---------- + G : NetworkX graph + + weight : string or function + If this is a string, then edge weights will be accessed via the + edge attribute with this key (that is, the weight of the edge + joining `u` to `v` will be ``G.edges[u, v][weight]``). If no + such edge attribute exists, the weight of the edge is assumed to + be one. + + If this is a function, the weight of an edge is the value + returned by the function. The function must accept exactly three + positional arguments: the two endpoints of an edge and the + dictionary of edge attributes for that edge. The function must + return a number. + + heuristic : bool + Determines whether to use a heuristic to early detect negative + cycles at a negligible cost. In case of graphs with a negative cycle, + the performance of detection increases by at least an order of magnitude. + + Returns + ------- + negative_cycle : bool + True if a negative edge cycle exists, otherwise False. + + Examples + -------- + >>> G = nx.cycle_graph(5, create_using=nx.DiGraph()) + >>> print(nx.negative_edge_cycle(G)) + False + >>> G[1][2]["weight"] = -7 + >>> print(nx.negative_edge_cycle(G)) + True + + Notes + ----- + Edge weight attributes must be numerical. + Distances are calculated as sums of weighted edges traversed. + + This algorithm uses bellman_ford_predecessor_and_distance() but finds + negative cycles on any component by first adding a new node connected to + every node, and starting bellman_ford_predecessor_and_distance on that + node. It then removes that extra node. + """ + if G.size() == 0: + return False + + # find unused node to use temporarily + newnode = -1 + while newnode in G: + newnode -= 1 + # connect it to all nodes + G.add_edges_from([(newnode, n) for n in G]) + + try: + bellman_ford_predecessor_and_distance( + G, newnode, weight=weight, heuristic=heuristic + ) + except nx.NetworkXUnbounded: + return True + finally: + G.remove_node(newnode) + return False + + +@nx._dispatchable(edge_attrs="weight") +def find_negative_cycle(G, source, weight="weight"): + """Returns a cycle with negative total weight if it exists. + + Bellman-Ford is used to find shortest_paths. That algorithm + stops if there exists a negative cycle. This algorithm + picks up from there and returns the found negative cycle. + + The cycle consists of a list of nodes in the cycle order. The last + node equals the first to make it a cycle. + You can look up the edge weights in the original graph. In the case + of multigraphs the relevant edge is the minimal weight edge between + the nodes in the 2-tuple. + + If the graph has no negative cycle, a NetworkXError is raised. + + Parameters + ---------- + G : NetworkX graph + + source: node label + The search for the negative cycle will start from this node. + + weight : string or function + If this is a string, then edge weights will be accessed via the + edge attribute with this key (that is, the weight of the edge + joining `u` to `v` will be ``G.edges[u, v][weight]``). If no + such edge attribute exists, the weight of the edge is assumed to + be one. + + If this is a function, the weight of an edge is the value + returned by the function. The function must accept exactly three + positional arguments: the two endpoints of an edge and the + dictionary of edge attributes for that edge. The function must + return a number. + + Examples + -------- + >>> G = nx.DiGraph() + >>> G.add_weighted_edges_from( + ... [(0, 1, 2), (1, 2, 2), (2, 0, 1), (1, 4, 2), (4, 0, -5)] + ... ) + >>> nx.find_negative_cycle(G, 0) + [4, 0, 1, 4] + + Returns + ------- + cycle : list + A list of nodes in the order of the cycle found. The last node + equals the first to indicate a cycle. + + Raises + ------ + NetworkXError + If no negative cycle is found. + """ + weight = _weight_function(G, weight) + pred = {source: []} + + v = _inner_bellman_ford(G, [source], weight, pred=pred) + if v is None: + raise nx.NetworkXError("No negative cycles detected.") + + # negative cycle detected... find it + neg_cycle = [] + stack = [(v, list(pred[v]))] + seen = {v} + while stack: + node, preds = stack[-1] + if v in preds: + # found the cycle + neg_cycle.extend([node, v]) + neg_cycle = list(reversed(neg_cycle)) + return neg_cycle + + if preds: + nbr = preds.pop() + if nbr not in seen: + stack.append((nbr, list(pred[nbr]))) + neg_cycle.append(node) + seen.add(nbr) + else: + stack.pop() + if neg_cycle: + neg_cycle.pop() + else: + if v in G[v] and weight(G, v, v) < 0: + return [v, v] + # should not reach here + raise nx.NetworkXError("Negative cycle is detected but not found") + # should not get here... + msg = "negative cycle detected but not identified" + raise nx.NetworkXUnbounded(msg) + + +@nx._dispatchable(edge_attrs="weight") +def bidirectional_dijkstra(G, source, target, weight="weight"): + r"""Dijkstra's algorithm for shortest paths using bidirectional search. + + Parameters + ---------- + G : NetworkX graph + + source : node + Starting node. + + target : node + Ending node. + + weight : string or function + If this is a string, then edge weights will be accessed via the + edge attribute with this key (that is, the weight of the edge + joining `u` to `v` will be ``G.edges[u, v][weight]``). If no + such edge attribute exists, the weight of the edge is assumed to + be one. + + If this is a function, the weight of an edge is the value + returned by the function. The function must accept exactly three + positional arguments: the two endpoints of an edge and the + dictionary of edge attributes for that edge. The function must + return a number or None to indicate a hidden edge. + + Returns + ------- + length, path : number and list + length is the distance from source to target. + path is a list of nodes on a path from source to target. + + Raises + ------ + NodeNotFound + If `source` or `target` is not in `G`. + + NetworkXNoPath + If no path exists between source and target. + + Examples + -------- + >>> G = nx.path_graph(5) + >>> length, path = nx.bidirectional_dijkstra(G, 0, 4) + >>> print(length) + 4 + >>> print(path) + [0, 1, 2, 3, 4] + + Notes + ----- + Edge weight attributes must be numerical. + Distances are calculated as sums of weighted edges traversed. + + The weight function can be used to hide edges by returning None. + So ``weight = lambda u, v, d: 1 if d['color']=="red" else None`` + will find the shortest red path. + + In practice bidirectional Dijkstra is much more than twice as fast as + ordinary Dijkstra. + + Ordinary Dijkstra expands nodes in a sphere-like manner from the + source. The radius of this sphere will eventually be the length + of the shortest path. Bidirectional Dijkstra will expand nodes + from both the source and the target, making two spheres of half + this radius. Volume of the first sphere is `\pi*r*r` while the + others are `2*\pi*r/2*r/2`, making up half the volume. + + This algorithm is not guaranteed to work if edge weights + are negative or are floating point numbers + (overflows and roundoff errors can cause problems). + + See Also + -------- + shortest_path + shortest_path_length + """ + if source not in G: + raise nx.NodeNotFound(f"Source {source} is not in G") + + if target not in G: + raise nx.NodeNotFound(f"Target {target} is not in G") + + if source == target: + return (0, [source]) + + weight = _weight_function(G, weight) + # Init: [Forward, Backward] + dists = [{}, {}] # dictionary of final distances + preds = [{source: None}, {target: None}] # dictionary of preds + + def path(curr, direction): + ret = [] + while curr is not None: + ret.append(curr) + curr = preds[direction][curr] + return list(reversed(ret)) if direction == 0 else ret + + fringe = [[], []] # heap of (distance, node) for choosing node to expand + seen = [{source: 0}, {target: 0}] # dict of distances to seen nodes + c = count() + # initialize fringe heap + heappush(fringe[0], (0, next(c), source)) + heappush(fringe[1], (0, next(c), target)) + # neighbors for extracting correct neighbor information + if G.is_directed(): + neighbors = [G._succ, G._pred] + else: + neighbors = [G._adj, G._adj] + # variables to hold shortest discovered path + finaldist = None + meetnode = None + direction = 1 + while fringe[0] and fringe[1]: + # choose direction + # direction == 0 is forward direction and direction == 1 is back + direction = 1 - direction + # extract closest to expand + (dist, _, v) = heappop(fringe[direction]) + if v in dists[direction]: + # Shortest path to v has already been found + continue + # update distance + dists[direction][v] = dist # equal to seen[direction][v] + if v in dists[1 - direction]: + # if we have scanned v in both directions we are done + # we have now discovered the shortest path + return (finaldist, path(meetnode, 0) + path(preds[1][meetnode], 1)) + + for w, d in neighbors[direction][v].items(): + # weight(v, w, d) for forward and weight(w, v, d) for back direction + cost = weight(v, w, d) if direction == 0 else weight(w, v, d) + if cost is None: + continue + vwLength = dist + cost + if w in dists[direction]: + if vwLength < dists[direction][w]: + raise ValueError("Contradictory paths found: negative weights?") + elif w not in seen[direction] or vwLength < seen[direction][w]: + # relaxing + seen[direction][w] = vwLength + heappush(fringe[direction], (vwLength, next(c), w)) + preds[direction][w] = v + if w in seen[1 - direction]: + # see if this path is better than the already + # discovered shortest path + finaldist_w = vwLength + seen[1 - direction][w] + if finaldist is None or finaldist > finaldist_w: + finaldist, meetnode = finaldist_w, w + raise nx.NetworkXNoPath(f"No path between {source} and {target}.") + + +@nx._dispatchable(edge_attrs="weight") +def johnson(G, weight="weight"): + r"""Uses Johnson's Algorithm to compute shortest paths. + + Johnson's Algorithm finds a shortest path between each pair of + nodes in a weighted graph even if negative weights are present. + + Parameters + ---------- + G : NetworkX graph + + weight : string or function + If this is a string, then edge weights will be accessed via the + edge attribute with this key (that is, the weight of the edge + joining `u` to `v` will be ``G.edges[u, v][weight]``). If no + such edge attribute exists, the weight of the edge is assumed to + be one. + + If this is a function, the weight of an edge is the value + returned by the function. The function must accept exactly three + positional arguments: the two endpoints of an edge and the + dictionary of edge attributes for that edge. The function must + return a number. + + Returns + ------- + distance : dictionary + Dictionary, keyed by source and target, of shortest paths. + + Examples + -------- + >>> graph = nx.DiGraph() + >>> graph.add_weighted_edges_from( + ... [("0", "3", 3), ("0", "1", -5), ("0", "2", 2), ("1", "2", 4), ("2", "3", 1)] + ... ) + >>> paths = nx.johnson(graph, weight="weight") + >>> paths["0"]["2"] + ['0', '1', '2'] + + Notes + ----- + Johnson's algorithm is suitable even for graphs with negative weights. It + works by using the Bellman–Ford algorithm to compute a transformation of + the input graph that removes all negative weights, allowing Dijkstra's + algorithm to be used on the transformed graph. + + The time complexity of this algorithm is $O(n^2 \log n + n m)$, + where $n$ is the number of nodes and $m$ the number of edges in the + graph. For dense graphs, this may be faster than the Floyd–Warshall + algorithm. + + See Also + -------- + floyd_warshall_predecessor_and_distance + floyd_warshall_numpy + all_pairs_shortest_path + all_pairs_shortest_path_length + all_pairs_dijkstra_path + bellman_ford_predecessor_and_distance + all_pairs_bellman_ford_path + all_pairs_bellman_ford_path_length + + """ + dist = {v: 0 for v in G} + pred = {v: [] for v in G} + weight = _weight_function(G, weight) + + # Calculate distance of shortest paths + dist_bellman = _bellman_ford(G, list(G), weight, pred=pred, dist=dist) + + # Update the weight function to take into account the Bellman--Ford + # relaxation distances. + def new_weight(u, v, d): + return weight(u, v, d) + dist_bellman[u] - dist_bellman[v] + + def dist_path(v): + paths = {v: [v]} + _dijkstra(G, v, new_weight, paths=paths) + return paths + + return {v: dist_path(v) for v in G} diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/nvidia_cudnn_cu12-9.10.2.21.dist-info/licenses/License.txt b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/nvidia_cudnn_cu12-9.10.2.21.dist-info/licenses/License.txt new file mode 100644 index 0000000000000000000000000000000000000000..f0d485c1c82d2c86b62ac0deeb8568fcdb58e0bb --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/nvidia_cudnn_cu12-9.10.2.21.dist-info/licenses/License.txt @@ -0,0 +1,154 @@ +LICENSE AGREEMENT FOR NVIDIA SOFTWARE DEVELOPMENT KITS + +This license agreement, including exhibits attached ("Agreement”) is a legal agreement between you and NVIDIA Corporation ("NVIDIA") and governs your use of a NVIDIA software development kit (“SDK”). + +Each SDK has its own set of software and materials, but here is a description of the types of items that may be included in a SDK: source code, header files, APIs, data sets and assets (examples include images, textures, models, scenes, videos, native API input/output files), binary software, sample code, libraries, utility programs, programming code and documentation. + +This Agreement can be accepted only by an adult of legal age of majority in the country in which the SDK is used. + +If you are entering into this Agreement on behalf of a company or other legal entity, you represent that you have the legal authority to bind the entity to this Agreement, in which case “you” will mean the entity you represent. + +If you don’t have the required age or authority to accept this Agreement, or if you don’t accept all the terms and conditions of this Agreement, do not download, install or use the SDK. + +You agree to use the SDK only for purposes that are permitted by (a) this Agreement, and (b) any applicable law, regulation or generally accepted practices or guidelines in the relevant jurisdictions. + +1. License. + +1.1 Grant + +Subject to the terms of this Agreement, NVIDIA hereby grants you a non-exclusive, non-transferable license, without the right to sublicense (except as expressly provided in this Agreement) to: + +(i) Install and use the SDK, + +(ii) Modify and create derivative works of sample source code delivered in the SDK, and + +(iii) Distribute those portions of the SDK that are identified in this Agreement as distributable, as incorporated in object code format into a software application that meets the distribution requirements indicated in this Agreement. + +1.2 Distribution Requirements + +These are the distribution requirements for you to exercise the distribution grant: + +(i) Your application must have material additional functionality, beyond the included portions of the SDK. + +(ii) The distributable portions of the SDK shall only be accessed by your application. + +(iii) The following notice shall be included in modifications and derivative works of sample source code distributed: “This software contains source code provided by NVIDIA Corporation.” + +(iv) Unless a developer tool is identified in this Agreement as distributable, it is delivered for your internal use only. + +(v) The terms under which you distribute your application must be consistent with the terms of this Agreement, including (without limitation) terms relating to the license grant and license restrictions and protection of NVIDIA’s intellectual property rights. Additionally, you agree that you will protect the privacy, security and legal rights of your application users. + +(vi) You agree to notify NVIDIA in writing of any known or suspected distribution or use of the SDK not in compliance with the requirements of this Agreement, and to enforce the terms of your agreements with respect to distributed SDK. + +1.3 Authorized Users + +You may allow employees and contractors of your entity or of your subsidiary(ies) to access and use the SDK from your secure network to perform work on your behalf. + +If you are an academic institution you may allow users enrolled or employed by the academic institution to access and use the SDK from your secure network. + +You are responsible for the compliance with the terms of this Agreement by your authorized users. If you become aware that your authorized users didn’t follow the terms of this Agreement, you agree to take reasonable steps to resolve the non-compliance and prevent new occurrences. + +1.4 Pre-Release SDK +The SDK versions identified as alpha, beta, preview or otherwise as pre-release, may not be fully functional, may contain errors or design flaws, and may have reduced or different security, privacy, accessibility, availability, and reliability standards relative to commercial versions of NVIDIA software and materials. Use of a pre-release SDK may result in unexpected results, loss of data, project delays or other unpredictable damage or loss. +You may use a pre-release SDK at your own risk, understanding that pre-release SDKs are not intended for use in production or business-critical systems. +NVIDIA may choose not to make available a commercial version of any pre-release SDK. NVIDIA may also choose to abandon development and terminate the availability of a pre-release SDK at any time without liability. +1.5 Updates + +NVIDIA may, at its option, make available patches, workarounds or other updates to this SDK. Unless the updates are provided with their separate governing terms, they are deemed part of the SDK licensed to you as provided in this Agreement. + +You agree that the form and content of the SDK that NVIDIA provides may change without prior notice to you. While NVIDIA generally maintains compatibility between versions, NVIDIA may in some cases make changes that introduce incompatibilities in future versions of the SDK. + +1.6 Third Party Licenses + +The SDK may come bundled with, or otherwise include or be distributed with, third party software licensed by a NVIDIA supplier and/or open source software provided under an open source license. Use of third party software is subject to the third-party license terms, or in the absence of third party terms, the terms of this Agreement. Copyright to third party software is held by the copyright holders indicated in the third-party software or license. + +1.7 Reservation of Rights + +NVIDIA reserves all rights, title and interest in and to the SDK not expressly granted to you under this Agreement. + +2. Limitations. + +The following license limitations apply to your use of the SDK: + +2.1 You may not reverse engineer, decompile or disassemble, or remove copyright or other proprietary notices from any portion of the SDK or copies of the SDK. + +2.2 Except as expressly provided in this Agreement, you may not copy, sell, rent, sublicense, transfer, distribute, modify, or create derivative works of any portion of the SDK. + +2.3 Unless you have an agreement with NVIDIA for this purpose, you may not indicate that an application created with the SDK is sponsored or endorsed by NVIDIA. + +2.4 You may not bypass, disable, or circumvent any encryption, security, digital rights management or authentication mechanism in the SDK. + +2.5 You may not use the SDK in any manner that would cause it to become subject to an open source software license. As examples, licenses that require as a condition of use, modification, and/or distribution that the SDK be (i) disclosed or distributed in source code form; (ii) licensed for the purpose of making derivative works; or (iii) redistributable at no charge. + +2.6 Unless you have an agreement with NVIDIA for this purpose, you may not use the SDK with any system or application where the use or failure of the system or application can reasonably be expected to threaten or result in personal injury, death, or catastrophic loss. Examples include use in avionics, navigation, military, medical, life support or other life critical applications. NVIDIA does not design, test or manufacture the SDK for these critical uses and NVIDIA shall not be liable to you or any third party, in whole or in part, for any claims or damages arising from such uses. + +2.7 You agree to defend, indemnify and hold harmless NVIDIA and its affiliates, and their respective employees, contractors, agents, officers and directors, from and against any and all claims, damages, obligations, losses, liabilities, costs or debt, fines, restitutions and expenses (including but not limited to attorney’s fees and costs incident to establishing the right of indemnification) arising out of or related to your use of the SDK outside of the scope of this Agreement, or not in compliance with its terms. + +3. Ownership. + +3.1 NVIDIA or its licensors hold all rights, title and interest in and to the SDK and its modifications and derivative works, including their respective intellectual property rights, subject to your rights under Section 3.2. This SDK may include software and materials from NVIDIA’s licensors, and these licensors are intended third party beneficiaries that may enforce this Agreement with respect to their intellectual property rights. + +3.2 You hold all rights, title and interest in and to your applications and your derivative works of the sample source code delivered in the SDK, including their respective intellectual property rights, subject to NVIDIA’s rights under section 3.1. + +3.3 You may, but don’t have to, provide to NVIDIA suggestions, feature requests or other feedback regarding the SDK, including possible enhancements or modifications to the SDK. For any feedback that you voluntarily provide, you hereby grant NVIDIA and its affiliates a perpetual, non-exclusive, worldwide, irrevocable license to use, reproduce, modify, license, sublicense (through multiple tiers of sublicensees), and distribute (through multiple tiers of distributors) it without the payment of any royalties or fees to you. NVIDIA will use feedback at its choice. NVIDIA is constantly looking for ways to improve its products, so you may send feedback to NVIDIA through the developer portal at https://developer.nvidia.com. + +4. No Warranties. + +THE SDK IS PROVIDED BY NVIDIA “AS IS” AND “WITH ALL FAULTS.” TO THE MAXIMUM EXTENT PERMITTED BY LAW, NVIDIA AND ITS AFFILIATES EXPRESSLY DISCLAIM ALL WARRANTIES OF ANY KIND OR NATURE, WHETHER EXPRESS, IMPLIED OR STATUTORY, INCLUDING, BUT NOT LIMITED TO, ANY WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE, NON-INFRINGEMENT, OR THE ABSENCE OF ANY DEFECTS THEREIN, WHETHER LATENT OR PATENT. NO WARRANTY IS MADE ON THE BASIS OF TRADE USAGE, COURSE OF DEALING OR COURSE OF TRADE. + +5. Limitations of Liability. + +TO THE MAXIMUM EXTENT PERMITTED BY LAW, NVIDIA AND ITS AFFILIATES SHALL NOT BE LIABLE FOR ANY SPECIAL, INCIDENTAL, PUNITIVE OR CONSEQUENTIAL DAMAGES, OR ANY LOST PROFITS, LOSS OF USE, LOSS OF DATA OR LOSS OF GOODWILL, OR THE COSTS OF PROCURING SUBSTITUTE PRODUCTS, ARISING OUT OF OR IN CONNECTION WITH THIS AGREEMENT OR THE USE OR PERFORMANCE OF THE SDK, WHETHER SUCH LIABILITY ARISES FROM ANY CLAIM BASED UPON BREACH OF CONTRACT, BREACH OF WARRANTY, TORT (INCLUDING NEGLIGENCE), PRODUCT LIABILITY OR ANY OTHER CAUSE OF ACTION OR THEORY OF LIABILITY. IN NO EVENT WILL NVIDIA’S AND ITS AFFILIATES TOTAL CUMULATIVE LIABILITY UNDER OR ARISING OUT OF THIS AGREEMENT EXCEED US$10.00. THE NATURE OF THE LIABILITY OR THE NUMBER OF CLAIMS OR SUITS SHALL NOT ENLARGE OR EXTEND THIS LIMIT. + +These exclusions and limitations of liability shall apply regardless if NVIDIA or its affiliates have been advised of the possibility of such damages, and regardless of whether a remedy fails its essential purpose. These exclusions and limitations of liability form an essential basis of the bargain between the parties, and, absent any of these exclusions or limitations of liability, the provisions of this Agreement, including, without limitation, the economic terms, would be substantially different. + +6. Termination. + +6.1 This Agreement will continue to apply until terminated by either you or NVIDIA as described below. + +6.2 If you want to terminate this Agreement, you may do so by stopping to use the SDK. + +6.3 NVIDIA may, at any time, terminate this Agreement if: (i) you fail to comply with any term of this Agreement and the non-compliance is not fixed within thirty (30) days following notice from NVIDIA (or immediately if you violate NVIDIA’s intellectual property rights); (ii) you commence or participate in any legal proceeding against NVIDIA with respect to the SDK; or (iii) NVIDIA decides to no longer provide the SDK in a country or, in NVIDIA’s sole discretion, the continued use of it is no longer commercially viable. + +6.4 Upon any termination of this Agreement, you agree to promptly discontinue use of the SDK and destroy all copies in your possession or control. Your prior distributions in accordance with this Agreement are not affected by the termination of this Agreement. Upon written request, you will certify in writing that you have complied with your commitments under this section. Upon any termination of this Agreement all provisions survive except for the licenses granted to you. + +7. General. + +If you wish to assign this Agreement or your rights and obligations, including by merger, consolidation, dissolution or operation of law, contact NVIDIA to ask for permission. Any attempted assignment not approved by NVIDIA in writing shall be void and of no effect. NVIDIA may assign, delegate or transfer this Agreement and its rights and obligations, and if to a non-affiliate you will be notified. + +You agree to cooperate with NVIDIA and provide reasonably requested information to verify your compliance with this Agreement. + +This Agreement will be governed in all respects by the laws of the United States and of the State of Delaware as those laws are applied to contracts entered into and performed entirely within Delaware by Delaware residents, without regard to the conflicts of laws principles. The United Nations Convention on Contracts for the International Sale of Goods is specifically disclaimed. You agree to all terms of this Agreement in the English language. + +The state or federal courts residing in Santa Clara County, California shall have exclusive jurisdiction over any dispute or claim arising out of this Agreement. Notwithstanding this, you agree that NVIDIA shall still be allowed to apply for injunctive remedies or an equivalent type of urgent legal relief in any jurisdiction. + +If any court of competent jurisdiction determines that any provision of this Agreement is illegal, invalid or unenforceable, such provision will be construed as limited to the extent necessary to be consistent with and fully enforceable under the law and the remaining provisions will remain in full force and effect. Unless otherwise specified, remedies are cumulative. + +Each party acknowledges and agrees that the other is an independent contractor in the performance of this Agreement. + +The SDK has been developed entirely at private expense and is “commercial items” consisting of “commercial computer software” and “commercial computer software documentation” provided with RESTRICTED RIGHTS. Use, duplication or disclosure by the U.S. Government or a U.S. Government subcontractor is subject to the restrictions in this Agreement pursuant to DFARS 227.7202-3(a) or as set forth in subparagraphs (b)(1) and (2) of the Commercial Computer Software - Restricted Rights clause at FAR 52.227-19, as applicable. Contractor/manufacturer is NVIDIA, 2788 San Tomas Expressway, Santa Clara, CA 95051. + +The SDK is subject to United States export laws and regulations. You agree that you will not ship, transfer or export the SDK into any country, or use the SDK in any manner, prohibited by the United States Bureau of Industry and Security or economic sanctions regulations administered by the U.S. Department of Treasury’s Office of Foreign Assets Control (OFAC), or any applicable export laws, restrictions or regulations. These laws include restrictions on destinations, end users and end use. By accepting this Agreement, you confirm that you are not a resident or citizen of any country currently embargoed by the U.S. and that you are not otherwise prohibited from receiving the SDK. + +Any notice delivered by NVIDIA to you under this Agreement will be delivered via mail, email or fax. You agree that any notices that NVIDIA sends you electronically will satisfy any legal communication requirements. Please direct your legal notices or other correspondence to NVIDIA Corporation, 2788 San Tomas Expressway, Santa Clara, California 95051, United States of America, Attention: Legal Department. + +This Agreement and any exhibits incorporated into this Agreement constitute the entire agreement of the parties with respect to the subject matter of this Agreement and supersede all prior negotiations or documentation exchanged between the parties relating to this SDK license. Any additional and/or conflicting terms on documents issued by you are null, void, and invalid. Any amendment or waiver under this Agreement shall be in writing and signed by representatives of both parties. + +(v. January 28, 2020) + + +cuDNN SUPPLEMENT TO SOFTWARE LICENSE AGREEMENT FOR NVIDIA SOFTWARE DEVELOPMENT KITS + +The terms in this supplement govern your use of the NVIDIA cuDNN SDK under the terms of your license agreement (“Agreement”) as modified by this supplement. Capitalized terms used but not defined below have the meaning assigned to them in the Agreement. + +This supplement is an exhibit to the Agreement and is incorporated as an integral part of the Agreement. In the event of conflict between the terms in this supplement and the terms in the Agreement, the terms in this supplement govern. + +4.1 License Scope. The SDK is licensed for you to develop applications only for use in systems with NVIDIA GPUs. + +2. Distribution. The following portions of the SDK are distributable under the Agreement: the runtime files .so and .h, cudnn64_7.dll, and cudnn.lib. + +In addition to the rights above, for parties that are developing software intended solely for use on Jetson development kits or Jetson modules and running Linux for Tegra software the following shall apply: the SDK may be distributed in its entirety, as provided by NVIDIA and without separation of its components, for you and/or your licensees to create software development kits for use only on the Jetson platform and running Linux for Tegra software. + +3. Licensing. If the distribution terms in this Agreement are not suitable for your organization, or for any questions regarding this Agreement, please contact NVIDIA at nvidia-compute-license-questions@nvidia.com. + (v. January 28, 2020) + diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/peft-0.18.1.dist-info/licenses/LICENSE b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/peft-0.18.1.dist-info/licenses/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/peft-0.18.1.dist-info/licenses/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/pyarrow_hotfix/__pycache__/__about__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/pyarrow_hotfix/__pycache__/__about__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..42758715ef8daddf8a12361d03f5819d306913af Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/pyarrow_hotfix/__pycache__/__about__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/pyarrow_hotfix/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/pyarrow_hotfix/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2391394117e50d19a40f3bee6734084bd36b50a4 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/pyarrow_hotfix/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sacrebleu/tokenizers/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sacrebleu/tokenizers/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..321a5752ce66cc9a9e9e40653c2c6ebcc3be3671 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sacrebleu/tokenizers/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sacrebleu/tokenizers/__pycache__/tokenizer_13a.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sacrebleu/tokenizers/__pycache__/tokenizer_13a.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..403037ddc24db72440e4c234ed381009ebc4ccd3 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sacrebleu/tokenizers/__pycache__/tokenizer_13a.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sacrebleu/tokenizers/__pycache__/tokenizer_base.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sacrebleu/tokenizers/__pycache__/tokenizer_base.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a82d1531587eb72a9ba85716b0568588fa6bfce0 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sacrebleu/tokenizers/__pycache__/tokenizer_base.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sacrebleu/tokenizers/__pycache__/tokenizer_char.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sacrebleu/tokenizers/__pycache__/tokenizer_char.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2a5a4740fa9337c1975dd8daf81b2c98ee7b45d7 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sacrebleu/tokenizers/__pycache__/tokenizer_char.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sacrebleu/tokenizers/__pycache__/tokenizer_intl.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sacrebleu/tokenizers/__pycache__/tokenizer_intl.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..867b306b49b0f05516bf1bbff71d8b26f57d8997 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sacrebleu/tokenizers/__pycache__/tokenizer_intl.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sacrebleu/tokenizers/__pycache__/tokenizer_ja_mecab.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sacrebleu/tokenizers/__pycache__/tokenizer_ja_mecab.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8de6eba3be51fdf80ba33491e59df42457702d7e Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sacrebleu/tokenizers/__pycache__/tokenizer_ja_mecab.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sacrebleu/tokenizers/__pycache__/tokenizer_re.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sacrebleu/tokenizers/__pycache__/tokenizer_re.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bb52875cf4e5b75b1bf8d9eb3cb5cea8ccfea27b Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sacrebleu/tokenizers/__pycache__/tokenizer_re.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sacrebleu/tokenizers/__pycache__/tokenizer_spm.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sacrebleu/tokenizers/__pycache__/tokenizer_spm.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5a98bfff5623350eff765b9b08baaabbae9494c8 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sacrebleu/tokenizers/__pycache__/tokenizer_spm.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f19f23e8fc574e79b4e1da13ee370e48ec074b22 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/__pycache__/__version__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/__pycache__/__version__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..671f341fa284da9c7d6d931880ac96fc8d308b11 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/__pycache__/__version__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/__pycache__/_common.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/__pycache__/_common.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6fe2c85073266a49701f8f08e8d2120f43b8a3b0 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/__pycache__/_common.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/__pycache__/_const.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/__pycache__/_const.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e9725b1e64e8c52368dd7dfaffbaa8bfa443caf1 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/__pycache__/_const.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/__pycache__/_function.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/__pycache__/_function.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c350334b2e36a0cd85681ed52979b7caa475c69c Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/__pycache__/_function.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/__pycache__/_typecode.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/__pycache__/_typecode.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2cff9b9ba7c7ad3279bf2f25e97c547b40bb79cb Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/__pycache__/_typecode.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/__pycache__/error.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/__pycache__/error.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f019bf306a5a4441f1df5602e6f6dc32244fb89d Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/__pycache__/error.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..be6d1d58e452c5904f4fa73d79be2f8118d93ff5 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__init__.py @@ -0,0 +1,35 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +from ._bool import BoolTypeChecker +from ._bytes import BytesTypeChecker +from ._datetime import DateTimeTypeChecker +from ._dictionary import DictionaryTypeChecker +from ._infinity import InfinityTypeChecker +from ._integer import IntegerTypeChecker +from ._interface import TypeCheckerInterface +from ._ipaddress import IpAddressTypeChecker +from ._list import ListTypeChecker +from ._nan import NanTypeChecker +from ._none import NoneTypeChecker +from ._realnumber import RealNumberTypeChecker +from ._string import NullStringTypeChecker, StringTypeChecker + + +__all__ = ( + "BoolTypeChecker", + "BytesTypeChecker", + "DateTimeTypeChecker", + "DictionaryTypeChecker", + "InfinityTypeChecker", + "IntegerTypeChecker", + "IpAddressTypeChecker", + "ListTypeChecker", + "NanTypeChecker", + "NoneTypeChecker", + "NullStringTypeChecker", + "RealNumberTypeChecker", + "StringTypeChecker", + "TypeCheckerInterface", +) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9f32ac3fb517085a88f0827c4bbfcca237bc78cc Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_bool.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_bool.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..90b9f593e832c64e1cd7aa7ee05691ccc84cdda3 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_bool.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_bytes.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_bytes.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c3da9b161041e9eb8f53712eadc898e51deb40d3 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_bytes.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_checker.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_checker.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e7e3cf0307ec67aff8891a7aab55fcdf90e69bc2 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_checker.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_common.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_common.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..68056e3a48402afdd8c530b14c2904a2f4fd80ee Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_common.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_datetime.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_datetime.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..71e5564e3d43f1545bde462f212b3935731ce4b2 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_datetime.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_dictionary.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_dictionary.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..936e814f10f056a19f9b8333d95bb4ab18d2b8db Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_dictionary.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_infinity.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_infinity.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..257b5859755ebe87c0199c43e9b9b801c011783e Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_infinity.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_integer.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_integer.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..df7ea42964a854bb678d1730c1e7be3f6ba073a1 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_integer.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_interface.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_interface.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cbce7f709e96438719e92958753a247e630161ae Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_interface.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_ipaddress.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_ipaddress.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b87f5dd89e2cf95cdb4ff6d17482d8425a44b0ef Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_ipaddress.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_list.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_list.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a5107cebc1ea9c85d9f4f4c89d7a5186c956202f Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_list.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_nan.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_nan.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3f28f7e7125e763e36415edf595725572716087e Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_nan.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_none.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_none.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..919f6cfdc04f2c4494f7bba8f95717103af7b938 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_none.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_realnumber.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_realnumber.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f8685eb23854d7d77f21f4d27ab3914aa4b30199 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_realnumber.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_string.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_string.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fbf116ecd740354343af55426d7f177f645f8ea6 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/__pycache__/_string.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_bool.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_bool.py new file mode 100644 index 0000000000000000000000000000000000000000..7c6638ee46f9ae1d92597626f132f80ab6242e1d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_bool.py @@ -0,0 +1,41 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +from .._const import StrictLevel +from ._checker import CheckerFactory, TypeCheckerBase, TypeCheckerDelegator +from ._common import isstring + + +class BoolTypeCheckerStrictLevel0(TypeCheckerBase): + def is_instance(self): + return isinstance(self._value, bool) + + def is_valid_after_convert(self, converted_value): + return isinstance(converted_value, bool) + + +class BoolTypeCheckerStrictLevel1(BoolTypeCheckerStrictLevel0): + def is_exclude_instance(self): + from ..type._integer import Integer + + return Integer(self._value, strict_level=StrictLevel.MAX).is_type() + + +class BoolTypeCheckerStrictLevel2(BoolTypeCheckerStrictLevel1): + def is_exclude_instance(self): + return super().is_exclude_instance() or isstring(self._value) + + +_factory = CheckerFactory( + checker_mapping={ + 0: BoolTypeCheckerStrictLevel0, + 1: BoolTypeCheckerStrictLevel1, + 2: BoolTypeCheckerStrictLevel2, + } +) + + +class BoolTypeChecker(TypeCheckerDelegator): + def __init__(self, value, strict_level): + super().__init__(value=value, checker_factory=_factory, strict_level=strict_level) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_bytes.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_bytes.py new file mode 100644 index 0000000000000000000000000000000000000000..06618b624d7849fb16ed07d4c3fd6fba2030369a --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_bytes.py @@ -0,0 +1,21 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +from ._checker import CheckerFactory, TypeCheckerBase, TypeCheckerDelegator + + +class BytesTypeCheckerStrictLevel0(TypeCheckerBase): + def is_instance(self): + return isinstance(self._value, bytes) + + def is_valid_after_convert(self, converted_value): + return isinstance(converted_value, bytes) + + +_factory = CheckerFactory(checker_mapping={0: BytesTypeCheckerStrictLevel0}) + + +class BytesTypeChecker(TypeCheckerDelegator): + def __init__(self, value, strict_level): + super().__init__(value=value, checker_factory=_factory, strict_level=strict_level) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_checker.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_checker.py new file mode 100644 index 0000000000000000000000000000000000000000..85e3a09aed128d2cdee350cd01ae313a50e32aa2 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_checker.py @@ -0,0 +1,90 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +import abc + +from ._interface import TypeCheckerInterface + + +class CheckerFactory: + __slots__ = ("__min_strict_level", "__max_strict_level", "__checker_mapping") + + @property + def min_strict_level(self): + return self.__min_strict_level + + @property + def max_strict_level(self): + return self.__max_strict_level + + def __init__(self, checker_mapping): + self.__checker_mapping = checker_mapping + + self.__min_strict_level = min(checker_mapping) + self.__max_strict_level = max(checker_mapping) + self.__checker_mapping[None] = self.__max_strict_level + + def get_checker_class(self, strict_level=None): + checker_class = self.__checker_mapping.get(strict_level) + if checker_class: + return checker_class + if strict_level < self.min_strict_level: + return self.__checker_mapping[self.min_strict_level] + if strict_level > self.max_strict_level: + return self.__checker_mapping[self.max_strict_level] + + raise ValueError(f"unexpected strict level: {strict_level}") + + +class TypeCheckerBase(TypeCheckerInterface): + __slots__ = ("_value",) + + def __init__(self, value): + self._value = value + + @abc.abstractmethod + def is_instance(self): + pass + + def is_type(self) -> bool: + return self.is_instance() and not self.is_exclude_instance() + + def validate(self) -> None: + """ + :raises TypeError: + If the value is not matched the type to be expected. + """ + + if self.is_type(): + return + + raise TypeError(f"invalid value type: actual={type(self._value)}") + + def is_exclude_instance(self): + return False + + def is_valid_after_convert(self, converted_value): + return True + + +class TypeCheckerDelegator(TypeCheckerInterface): + __slots__ = ("__checker",) + + def __init__(self, value, checker_factory, strict_level): + self.__checker = checker_factory.get_checker_class(strict_level)(value) + + def is_type(self) -> bool: + return self.__checker.is_type() + + def is_valid_after_convert(self, value): + return self.__checker.is_valid_after_convert(value) + + def is_instance(self): + return self.__checker.is_instance() + + def is_exclude_instance(self): + return self.__checker.is_exclude_instance() + + def validate(self) -> None: + self.__checker.validate() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_common.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_common.py new file mode 100644 index 0000000000000000000000000000000000000000..ed4ef4e44384bab1c22c483dbe75cde24967f5a5 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_common.py @@ -0,0 +1,28 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +import decimal +import math + + +def isstring(value): + return isinstance(value, (str,) + (str, bytes)) + + +def isinf(value): + try: + return decimal.Decimal(value).is_infinite() + except OverflowError: + return True + except TypeError: + return False + except (ValueError, decimal.InvalidOperation): + return False + + +def isnan(value): + try: + return math.isnan(value) + except (TypeError, OverflowError): + return False diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_datetime.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_datetime.py new file mode 100644 index 0000000000000000000000000000000000000000..1ce258cf6ffb614bb1fae46c380046f5a26983db --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_datetime.py @@ -0,0 +1,40 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +from datetime import date, datetime + +from ._checker import CheckerFactory, TypeCheckerBase, TypeCheckerDelegator +from ._common import isstring + + +class DateTimeTypeCheckerStrictLevel0(TypeCheckerBase): + def is_instance(self): + return isinstance(self._value, (date, datetime)) + + +class DateTimeTypeCheckerStrictLevel1(DateTimeTypeCheckerStrictLevel0): + def is_exclude_instance(self): + from ..type._integer import Integer + + # exclude timestamp + return Integer(self._value, strict_level=1).is_type() + + +class DateTimeTypeCheckerStrictLevel2(DateTimeTypeCheckerStrictLevel1): + def is_exclude_instance(self): + return isstring(self._value) or super().is_exclude_instance() + + +_factory = CheckerFactory( + checker_mapping={ + 0: DateTimeTypeCheckerStrictLevel0, + 1: DateTimeTypeCheckerStrictLevel1, + 2: DateTimeTypeCheckerStrictLevel2, + } +) + + +class DateTimeTypeChecker(TypeCheckerDelegator): + def __init__(self, value, strict_level): + super().__init__(value=value, checker_factory=_factory, strict_level=strict_level) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_dictionary.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_dictionary.py new file mode 100644 index 0000000000000000000000000000000000000000..1f1106a78326b638b17c0748de1128ec51b6f90f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_dictionary.py @@ -0,0 +1,28 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +from ._checker import CheckerFactory, TypeCheckerBase, TypeCheckerDelegator + + +class DictionaryTypeCheckerStrictLevel0(TypeCheckerBase): + def is_instance(self): + return isinstance(self._value, dict) + + def is_valid_after_convert(self, converted_value): + return isinstance(converted_value, dict) and converted_value + + +class DictionaryTypeCheckerStrictLevel1(DictionaryTypeCheckerStrictLevel0): + def is_exclude_instance(self): + return not isinstance(self._value, dict) + + +_factory = CheckerFactory( + checker_mapping={0: DictionaryTypeCheckerStrictLevel0, 1: DictionaryTypeCheckerStrictLevel1} +) + + +class DictionaryTypeChecker(TypeCheckerDelegator): + def __init__(self, value, strict_level): + super().__init__(value=value, checker_factory=_factory, strict_level=strict_level) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_infinity.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_infinity.py new file mode 100644 index 0000000000000000000000000000000000000000..cf263081769d98df2b87e34f72eee45575247572 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_infinity.py @@ -0,0 +1,32 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +from ._checker import CheckerFactory, TypeCheckerBase, TypeCheckerDelegator +from ._common import isinf, isstring + + +class InfinityCheckerStrictLevel0(TypeCheckerBase): + def is_instance(self): + return isinf(self._value) + + def is_valid_after_convert(self, converted_value): + try: + return converted_value.is_infinite() + except AttributeError: + return False + + +class InfinityCheckerStrictLevel1(InfinityCheckerStrictLevel0): + def is_exclude_instance(self): + return isstring(self._value) + + +_factory = CheckerFactory( + checker_mapping={0: InfinityCheckerStrictLevel0, 1: InfinityCheckerStrictLevel1} +) + + +class InfinityTypeChecker(TypeCheckerDelegator): + def __init__(self, value, strict_level): + super().__init__(value=value, checker_factory=_factory, strict_level=strict_level) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_integer.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_integer.py new file mode 100644 index 0000000000000000000000000000000000000000..bf6deea8e9f5c8267f74650221ede5927244f2d7 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_integer.py @@ -0,0 +1,72 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +import re +from decimal import Decimal + +from ._checker import CheckerFactory, TypeCheckerBase, TypeCheckerDelegator +from ._common import isinf, isnan + + +RE_SCIENTIFIC_NOTATION = re.compile(r"^-?\d+(?:\.\d*)?[eE][+\-]?\d{3,}$") + + +class IntegerTypeCheckerStrictLevel0(TypeCheckerBase): + def is_instance(self): + if isinstance(self._value, int): + return not isinstance(self._value, bool) + + if isinstance(self._value, (float, Decimal)): + return True + + return False + + def is_exclude_instance(self): + return isnan(self._value) or isinf(self._value) + + +class IntegerTypeCheckerStrictLevel1(IntegerTypeCheckerStrictLevel0): + def is_instance(self): + if not super().is_instance(): + return False + + if isinstance(self._value, (float, Decimal)): + if float(self._value).is_integer(): + return True + + try: + return self._value.is_integer() + except AttributeError: + pass + + return False + + def is_exclude_instance(self): + from ..type._realnumber import RealNumber + + return ( + super().is_exclude_instance() + or isinstance(self._value, bool) + or RealNumber(self._value, strict_level=1).is_type() + or (isinstance(self._value, str) and RE_SCIENTIFIC_NOTATION.search(self._value)) + ) + + +class IntegerTypeCheckerStrictLevel2(IntegerTypeCheckerStrictLevel1): + def is_exclude_instance(self): + return isinstance(self._value, (str,) + (bool, float, Decimal)) + + +_factory = CheckerFactory( + checker_mapping={ + 0: IntegerTypeCheckerStrictLevel0, + 1: IntegerTypeCheckerStrictLevel1, + 2: IntegerTypeCheckerStrictLevel2, + } +) + + +class IntegerTypeChecker(TypeCheckerDelegator): + def __init__(self, value, strict_level): + super().__init__(value=value, checker_factory=_factory, strict_level=strict_level) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_interface.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_interface.py new file mode 100644 index 0000000000000000000000000000000000000000..002c9bbe4a9b04c853af5eeb3d67687ffaa76d23 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_interface.py @@ -0,0 +1,15 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +import abc + + +class TypeCheckerInterface(metaclass=abc.ABCMeta): + @abc.abstractmethod + def is_type(self) -> bool: # pragma: no cover + pass + + @abc.abstractmethod + def validate(self) -> None: # pragma: no cover + pass diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_ipaddress.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_ipaddress.py new file mode 100644 index 0000000000000000000000000000000000000000..697f929a041e09ad596e7b4dd41d7f98fb2deef0 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_ipaddress.py @@ -0,0 +1,35 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +from ._checker import CheckerFactory, TypeCheckerBase, TypeCheckerDelegator +from ._common import isstring + + +class IpAddressTypeCheckerStrictLevel0(TypeCheckerBase): + def is_instance(self): + return self._is_ipaddress(self._value) + + def is_valid_after_convert(self, converted_value): + return self._is_ipaddress(converted_value) + + @staticmethod + def _is_ipaddress(value): + import ipaddress + + return isinstance(value, (ipaddress.IPv4Address, ipaddress.IPv6Address)) + + +class IpAddressTypeCheckerStrictLevel1(IpAddressTypeCheckerStrictLevel0): + def is_exclude_instance(self): + return isstring(self._value) or super().is_exclude_instance() + + +_factory = CheckerFactory( + checker_mapping={0: IpAddressTypeCheckerStrictLevel0, 1: IpAddressTypeCheckerStrictLevel1} +) + + +class IpAddressTypeChecker(TypeCheckerDelegator): + def __init__(self, value, strict_level): + super().__init__(value=value, checker_factory=_factory, strict_level=strict_level) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_list.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_list.py new file mode 100644 index 0000000000000000000000000000000000000000..d66d2c0eae9f0ca71ffdf92f398f4ba88fc29010 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_list.py @@ -0,0 +1,32 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +from ._checker import CheckerFactory, TypeCheckerBase, TypeCheckerDelegator +from ._common import isstring + + +class ListTypeCheckerStrictLevel0(TypeCheckerBase): + def is_instance(self): + return isinstance(self._value, list) + + def is_valid_after_convert(self, converted_value): + return isinstance(converted_value, list) and converted_value + + def is_exclude_instance(self): + return isstring(self._value) + + +class ListTypeCheckerStrictLevel1(ListTypeCheckerStrictLevel0): + def is_exclude_instance(self): + return super().is_exclude_instance() or not isinstance(self._value, list) + + +_factory = CheckerFactory( + checker_mapping={0: ListTypeCheckerStrictLevel0, 1: ListTypeCheckerStrictLevel1} +) + + +class ListTypeChecker(TypeCheckerDelegator): + def __init__(self, value, strict_level): + super().__init__(value=value, checker_factory=_factory, strict_level=strict_level) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_nan.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_nan.py new file mode 100644 index 0000000000000000000000000000000000000000..ceab20c1b7d008028c8ca4cdfe4c17b9fed9e267 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_nan.py @@ -0,0 +1,29 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +from ._checker import CheckerFactory, TypeCheckerBase, TypeCheckerDelegator +from ._common import isnan, isstring + + +class NanTypeCheckerStrictLevel0(TypeCheckerBase): + def is_instance(self): + return isnan(self._value) + + def is_valid_after_convert(self, converted_value): + return isnan(converted_value) + + +class NanTypeCheckerStrictLevel1(NanTypeCheckerStrictLevel0): + def is_exclude_instance(self): + return isstring(self._value) + + +_factory = CheckerFactory( + checker_mapping={0: NanTypeCheckerStrictLevel0, 1: NanTypeCheckerStrictLevel1} +) + + +class NanTypeChecker(TypeCheckerDelegator): + def __init__(self, value, strict_level): + super().__init__(value=value, checker_factory=_factory, strict_level=strict_level) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_none.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_none.py new file mode 100644 index 0000000000000000000000000000000000000000..f9c0d1812866eab7d6e7d5d8cb1f5e49ba431dbf --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_none.py @@ -0,0 +1,21 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +from ._checker import CheckerFactory, TypeCheckerBase, TypeCheckerDelegator + + +class NoneTypeCheckerStrictLevel0(TypeCheckerBase): + def is_instance(self): + return self._value is None + + def is_valid_after_convert(self, converted_value): + return self._value is None + + +_factory = CheckerFactory(checker_mapping={0: NoneTypeCheckerStrictLevel0}) + + +class NoneTypeChecker(TypeCheckerDelegator): + def __init__(self, value, strict_level): + super().__init__(value=value, checker_factory=_factory, strict_level=strict_level) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_realnumber.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_realnumber.py new file mode 100644 index 0000000000000000000000000000000000000000..25a509f0363d1188338f663b33447d8b96925896 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_realnumber.py @@ -0,0 +1,61 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +import re +from decimal import Decimal + +from ._checker import CheckerFactory, TypeCheckerBase, TypeCheckerDelegator +from ._common import isinf, isnan, isstring + + +RE_E = re.compile("[eE]") +RE_SCIENTIFIC_NOTATION = re.compile(r"^-?\d+(?:\.\d*)?[eE][+\-]?\d{,2}$") + + +class RealNumberTypeCheckerStrictLevel0(TypeCheckerBase): + def is_instance(self): + return isinstance(self._value, (float, Decimal)) + + def is_exclude_instance(self): + return isinstance(self._value, bool) or isnan(self._value) or isinf(self._value) + + def is_valid_after_convert(self, converted_value): + return not isinf(converted_value) and not isnan(converted_value) + + +class RealNumberTypeCheckerStrictLevel1(RealNumberTypeCheckerStrictLevel0): + def is_instance(self): + return super().is_instance() and not float(self._value).is_integer() + + def is_exclude_instance(self): + if ( + isinstance(self._value, str) + and RE_E.search(self._value) + and RE_SCIENTIFIC_NOTATION.search(self._value) is None + ): + return True + + return isinstance(self._value, int) or super().is_exclude_instance() + + def is_valid_after_convert(self, converted_value): + return not float(converted_value).is_integer() + + +class RealNumberTypeCheckerStrictLevel2(RealNumberTypeCheckerStrictLevel1): + def is_exclude_instance(self): + return super().is_exclude_instance() or isstring(self._value) + + +_factory = CheckerFactory( + checker_mapping={ + 0: RealNumberTypeCheckerStrictLevel0, + 1: RealNumberTypeCheckerStrictLevel1, + 2: RealNumberTypeCheckerStrictLevel2, + } +) + + +class RealNumberTypeChecker(TypeCheckerDelegator): + def __init__(self, value, strict_level): + super().__init__(value=value, checker_factory=_factory, strict_level=strict_level) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_string.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_string.py new file mode 100644 index 0000000000000000000000000000000000000000..5ab57c926b5d94fffc4412681aa99be140f05511 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/checker/_string.py @@ -0,0 +1,74 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +from ._checker import CheckerFactory, TypeCheckerBase, TypeCheckerDelegator +from ._common import isstring + + +class StringTypeCheckerStrictLevel0(TypeCheckerBase): + def is_instance(self): + return isstring(self._value) + + def is_valid_after_convert(self, converted_value): + return isstring(converted_value) + + def _is_null_string(self, value): + try: + value = value.strip() + except AttributeError: + return False + + return len(value) == 0 + + +class StringTypeCheckerStrictLevel1(StringTypeCheckerStrictLevel0): + def is_exclude_instance(self): + return not isstring(self._value) + + +class StringTypeCheckerStrictLevel2(StringTypeCheckerStrictLevel1): + def is_exclude_instance(self): + if super().is_exclude_instance(): + return True + + return self._is_null_string(self._value) + + +_string_factory = CheckerFactory( + checker_mapping={ + 0: StringTypeCheckerStrictLevel0, + 1: StringTypeCheckerStrictLevel1, + 2: StringTypeCheckerStrictLevel2, + } +) + + +class StringTypeChecker(TypeCheckerDelegator): + def __init__(self, value, strict_level): + super().__init__(value=value, checker_factory=_string_factory, strict_level=strict_level) + + +class NullStringTypeCheckerStrictLevel0(StringTypeCheckerStrictLevel0): + def is_instance(self): + return self._value is None + + def is_valid_after_convert(self, converted_value): + return self._is_null_string(converted_value) + + +class NullStringTypeCheckerStrictLevel1(NullStringTypeCheckerStrictLevel0): + def is_instance(self): + return False + + +_null_string_factory = CheckerFactory( + checker_mapping={0: NullStringTypeCheckerStrictLevel0, 1: NullStringTypeCheckerStrictLevel1} +) + + +class NullStringTypeChecker(TypeCheckerDelegator): + def __init__(self, value, strict_level): + super().__init__( + value=value, checker_factory=_null_string_factory, strict_level=strict_level + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1fab0dfda09685e1ac2ca6ef9212c24e3ae8e0b7 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__init__.py @@ -0,0 +1,31 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +from ._bool import BoolConverter +from ._bytes import BytesConverter +from ._datetime import DateTimeConverter +from ._dictionary import DictionaryConverter +from ._integer import IntegerConverter +from ._interface import ValueConverterInterface +from ._ipaddress import IpAddressConverter +from ._list import ListConverter +from ._nop import NopConverter +from ._realnumber import FloatConverter +from ._string import NullStringConverter, StringConverter + + +__all__ = ( + "BoolConverter", + "BytesConverter", + "DateTimeConverter", + "DictionaryConverter", + "FloatConverter", + "IntegerConverter", + "IpAddressConverter", + "ListConverter", + "NopConverter", + "NullStringConverter", + "StringConverter", + "ValueConverterInterface", +) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..30f5f19e68c76c20ecc05fc8b836fb59281b33da Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__pycache__/_bool.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__pycache__/_bool.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ef1c63e0e4819bd9d5c56528ad57d945755e92d3 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__pycache__/_bool.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__pycache__/_bytes.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__pycache__/_bytes.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cefec6415824251bcc21f0ca192da494fc5d4786 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__pycache__/_bytes.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__pycache__/_datetime.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__pycache__/_datetime.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..85f926eb9ed99e51bcae3155a7106dd075f996a6 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__pycache__/_datetime.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__pycache__/_dictionary.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__pycache__/_dictionary.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7166e4f43c1cb34e28b03efa03991242d6e44eab Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__pycache__/_dictionary.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__pycache__/_integer.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__pycache__/_integer.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6a7af00d648dea83d50bcbd66a2fb391bd26b75a Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__pycache__/_integer.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__pycache__/_interface.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__pycache__/_interface.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..699192e35fb4225b0238e46acf340da955ebe4fb Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__pycache__/_interface.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__pycache__/_ipaddress.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__pycache__/_ipaddress.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..718de3432e9e5fd9233361c0bdb782055ffdf688 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__pycache__/_ipaddress.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__pycache__/_list.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__pycache__/_list.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2a7e7ac69c22e6b2994a654825540e51bc27c837 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__pycache__/_list.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__pycache__/_nop.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__pycache__/_nop.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a4be1aac4bbb21c8820b5ddc430fbbf9a4b2a356 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__pycache__/_nop.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__pycache__/_realnumber.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__pycache__/_realnumber.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fa76a8c16f1d7a2184f96b929851fe7906c19d39 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__pycache__/_realnumber.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__pycache__/_string.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__pycache__/_string.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c5e444b587591c4a7bcf427ab5ebf26a88f421c7 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/__pycache__/_string.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/_bool.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/_bool.py new file mode 100644 index 0000000000000000000000000000000000000000..8f89a4e415ec2fec66c4298343f5efe125176e6d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/_bool.py @@ -0,0 +1,44 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +from .._common import strip_ansi_escape +from .._const import DefaultValue, ParamKey +from ..error import TypeConversionError +from ._interface import AbstractValueConverter + + +class BoolConverter(AbstractValueConverter): + def force_convert(self): + if isinstance(self._value, int): + return bool(self._value) + + try: + return self.__strict_strtobool(self._value) + except ValueError: + pass + + if self._params.get(ParamKey.STRIP_ANSI_ESCAPE, DefaultValue.STRIP_ANSI_ESCAPE): + try: + return self.__strict_strtobool(strip_ansi_escape(self._value)) + except (TypeError, ValueError): + pass + + raise TypeConversionError(f"failed to force_convert to bool: type={type(self._value)}") + + @staticmethod + def __strict_strtobool(value): + if isinstance(value, bool): + return value + + try: + lower_text = value.casefold() + except AttributeError: + raise ValueError(f"invalid value '{str(value)}'") + + if lower_text in ["true"]: + return True + elif lower_text in ["false"]: + return False + + raise ValueError(f"invalid value '{str(value)}'") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/_bytes.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/_bytes.py new file mode 100644 index 0000000000000000000000000000000000000000..0fda2aaeefee4f6369ce5353638eb573e8e0dd88 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/_bytes.py @@ -0,0 +1,11 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +from ..error import TypeConversionError +from ._interface import AbstractValueConverter + + +class BytesConverter(AbstractValueConverter): + def force_convert(self): + raise TypeConversionError("not inmplemented") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/_datetime.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/_datetime.py new file mode 100644 index 0000000000000000000000000000000000000000..86ba07a52a3ad63752da6a0a967ca92f551c35c7 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/_datetime.py @@ -0,0 +1,160 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +from datetime import date, datetime + +from .._common import strip_ansi_escape +from .._const import DefaultValue, ParamKey +from ..error import TypeConversionError +from ._interface import AbstractValueConverter + + +class DateTimeConverter(AbstractValueConverter): + __DAYS_TO_SECONDS_COEF = 60**2 * 24 + __MICROSECONDS_TO_SECONDS_COEF = 1000**2 + __COMMON_DST_TIMEZONE_TABLE = { + -36000: "America/Adak", # -1000 + -32400: "US/Alaska", # -0900 + -28800: "US/Pacific", # -0800 + -25200: "US/Mountain", # -0700 + -21600: "US/Central", # -0600 + -18000: "US/Eastern", # -0500 + -14400: "Canada/Atlantic", # -0400 + -12600: "America/St_Johns", # -0330 + -10800: "America/Miquelon", # -0300 + 7200: "Africa/Tripoli", # 0200 + } + + def __init__(self, value, params): + super().__init__(value, params) + + self.__datetime = None + self.__timezone = self._params.get(ParamKey.TIMEZONE) + + def force_convert(self): + self.__datetime = self.__from_datetime() + if self.__datetime: + return self.__datetime + + self.__datetime = self.__from_timestamp() + if self.__datetime: + return self.__datetime + + return self.__from_datetime_string() + + def __from_datetime(self): + if not isinstance(self._value, (date, datetime)): + return None + + if isinstance(self._value, datetime): + self.__datetime = self._value + elif isinstance(self._value, date): + self.__datetime = datetime( + year=self._value.year, month=self._value.month, day=self._value.day + ) + + if self.__timezone: + if self.__datetime.tzinfo is None: + self.__datetime = self.__timezone.localize(self.__datetime) + else: + self.__datetime = datetime.fromtimestamp( + self.__datetime.timestamp(), tz=self.__timezone + ) + + return self.__datetime + + def __from_timestamp(self): + from ..type._integer import Integer + from ..type._realnumber import RealNumber + + conv_error = TypeConversionError( + "timestamp is out of the range of values supported by the platform" + ) + + timestamp = Integer(self._value, strict_level=1).try_convert() + if timestamp: + try: + self.__datetime = datetime.fromtimestamp(timestamp, self.__timezone) + except (ValueError, OSError, OverflowError): + raise conv_error + + return self.__datetime + + timestamp = RealNumber(self._value, strict_level=1).try_convert() + if timestamp: + try: + self.__datetime = datetime.fromtimestamp(int(timestamp), self.__timezone).replace( + microsecond=int((timestamp - int(timestamp)) * 1000000) + ) + except (ValueError, OSError, OverflowError): + raise conv_error + + return self.__datetime + + return None + + def __from_datetime_string(self): + import dateutil.parser + import pytz + + self.__validate_datetime_string() + + try: + self.__datetime = dateutil.parser.parse(self._value) + except (AttributeError, ValueError, OverflowError): + if self._params.get(ParamKey.STRIP_ANSI_ESCAPE, DefaultValue.STRIP_ANSI_ESCAPE): + try: + self.__datetime = dateutil.parser.parse(strip_ansi_escape(self._value)) + except (AttributeError, ValueError, OverflowError): + pass + + if self.__datetime is None: + raise TypeConversionError(f"failed to parse as a datetime: type={type(self._value)}") + + if self.__timezone: + pytz_timezone = self.__timezone + else: + try: + dst_timezone_name = self.__get_dst_timezone_name(self.__get_timedelta_sec()) + except (AttributeError, KeyError): + return self.__datetime + + pytz_timezone = pytz.timezone(dst_timezone_name) + + self.__datetime = self.__datetime.replace(tzinfo=None) + self.__datetime = pytz_timezone.localize(self.__datetime) + + return self.__datetime + + def __get_timedelta_sec(self): + dt = self.__datetime.utcoffset() + + return int( + dt.days * self.__DAYS_TO_SECONDS_COEF + + float(dt.seconds) + + dt.microseconds / self.__MICROSECONDS_TO_SECONDS_COEF + ) + + def __get_dst_timezone_name(self, offset): + return self.__COMMON_DST_TIMEZONE_TABLE[offset] + + def __validate_datetime_string(self): + """ + This will require validating version string (such as "3.3.5"). + A version string could be converted to a datetime value if this + validation is not executed. + """ + + from packaging.version import InvalidVersion, Version + + try: + try: + Version(self._value) + raise TypeConversionError( + f"invalid datetime string: version string found {self._value}" + ) + except InvalidVersion: + pass + except TypeError: + raise TypeConversionError(f"invalid datetime string: type={type(self._value)}") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/_dictionary.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/_dictionary.py new file mode 100644 index 0000000000000000000000000000000000000000..1be816f9f846baef433476cbb259b733bed931e8 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/_dictionary.py @@ -0,0 +1,26 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +import json + +from ..error import TypeConversionError +from ._interface import AbstractValueConverter + + +class DictionaryConverter(AbstractValueConverter): + def force_convert(self): + try: + return dict(self._value) + except (TypeError, ValueError): + pass + + if isinstance(self._value, str): + try: + return json.loads(self._value) + except json.JSONDecodeError: + pass + + raise TypeConversionError( + f"failed to force_convert to dictionary: type={type(self._value)}" + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/_integer.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/_integer.py new file mode 100644 index 0000000000000000000000000000000000000000..b62db15c4c8522456d854b0221adfcc7aedba293 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/_integer.py @@ -0,0 +1,31 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +from decimal import Decimal, InvalidOperation + +from .._common import remove_thousand_sep, strip_ansi_escape +from .._const import DefaultValue, ParamKey +from ..error import TypeConversionError +from ._interface import AbstractValueConverter + + +class IntegerConverter(AbstractValueConverter): + def force_convert(self): + try: + value = remove_thousand_sep(self._value) + except TypeError: + value = self._value + + try: + return int(Decimal(value)) + except (TypeError, OverflowError, ValueError, InvalidOperation): + pass + + if self._params.get(ParamKey.STRIP_ANSI_ESCAPE, DefaultValue.STRIP_ANSI_ESCAPE): + try: + return int(Decimal(strip_ansi_escape(value))) + except (TypeError, OverflowError, ValueError, InvalidOperation): + pass + + raise TypeConversionError(f"failed to force_convert to int: type={type(value)}") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/_interface.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/_interface.py new file mode 100644 index 0000000000000000000000000000000000000000..30bb2c2bdb73efa4a904ea9149735034ffc57c70 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/_interface.py @@ -0,0 +1,29 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +import abc + +from ..error import TypeConversionError + + +class ValueConverterInterface(metaclass=abc.ABCMeta): + @abc.abstractmethod + def force_convert(self): # pragma: no cover + pass + + +class AbstractValueConverter(ValueConverterInterface): + __slots__ = "_value" + + def __init__(self, value, params): + self._value = value + self._params = params + + def __repr__(self): + try: + string = str(self.force_convert()) + except TypeConversionError: + string = "[ValueConverter ERROR] failed to force_convert" + + return string diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/_ipaddress.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/_ipaddress.py new file mode 100644 index 0000000000000000000000000000000000000000..5b8306b48560efd7cbd19f2d2abac50c3adb3c97 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/_ipaddress.py @@ -0,0 +1,30 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +from .._common import strip_ansi_escape +from .._const import DefaultValue, ParamKey +from ..error import TypeConversionError +from ._interface import AbstractValueConverter + + +class IpAddressConverter(AbstractValueConverter): + def force_convert(self): + import ipaddress + + value = str(self._value) + + try: + return ipaddress.ip_address(value) + except ValueError: + pass + + if self._params.get(ParamKey.STRIP_ANSI_ESCAPE, DefaultValue.STRIP_ANSI_ESCAPE): + try: + return ipaddress.ip_address(strip_ansi_escape(value)) + except ValueError: + pass + + raise TypeConversionError( + f"failed to force_convert to dictionary: type={type(self._value)}" + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/_list.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/_list.py new file mode 100644 index 0000000000000000000000000000000000000000..9654dce70a7cfda24fae3265ff85d50d7a30e277 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/_list.py @@ -0,0 +1,14 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +from ..error import TypeConversionError +from ._interface import AbstractValueConverter + + +class ListConverter(AbstractValueConverter): + def force_convert(self): + try: + return list(self._value) + except (TypeError, ValueError): + raise TypeConversionError(f"failed to force_convert to list: type={type(self._value)}") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/_nop.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/_nop.py new file mode 100644 index 0000000000000000000000000000000000000000..50fc3bff4d33c3af093c2a65227d41ad2ef99ef7 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/_nop.py @@ -0,0 +1,10 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +from ._interface import AbstractValueConverter + + +class NopConverter(AbstractValueConverter): + def force_convert(self): + return self._value diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/_realnumber.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/_realnumber.py new file mode 100644 index 0000000000000000000000000000000000000000..b7879be0a9dad0eb844bbe2edfaa7384e20c3fde --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/_realnumber.py @@ -0,0 +1,39 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +import decimal + +from .._common import strip_ansi_escape +from .._const import DefaultValue, ParamKey +from ..error import TypeConversionError +from ._interface import AbstractValueConverter + + +class FloatConverter(AbstractValueConverter): + def __init__(self, value, params): + super().__init__(value, params) + + self.float_class = self._params.get("float_type") + if self.float_class is None: + self.float_class = DefaultValue.FLOAT_TYPE + + def force_convert(self): + if isinstance(self._value, self.float_class): + return self._value + + if isinstance(self._value, float): + return self.float_class(str(self._value)) + + try: + return self.float_class(self._value) + except (TypeError, ValueError, decimal.InvalidOperation): + pass + + if self._params.get(ParamKey.STRIP_ANSI_ESCAPE, DefaultValue.STRIP_ANSI_ESCAPE): + try: + return self.float_class(strip_ansi_escape(self._value)) + except (TypeError, ValueError, decimal.InvalidOperation): + pass + + raise TypeConversionError(f"failed to force_convert to float: type={type(self._value)}") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/_string.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/_string.py new file mode 100644 index 0000000000000000000000000000000000000000..054e1ea9842527a346bb240f3a8636421390c0d4 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/converter/_string.py @@ -0,0 +1,20 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +from mbstrdecoder import MultiByteStrDecoder + +from ._interface import AbstractValueConverter + + +class StringConverter(AbstractValueConverter): + def force_convert(self): + try: + return MultiByteStrDecoder(self._value).unicode_str + except ValueError: + return str(self._value) + + +class NullStringConverter(StringConverter): + def force_convert(self): + return super().force_convert() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2dd8046f87c933721fc90c4205c5458036b13bb7 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_base.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_base.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..78e92b4d993be4b1b72ef9232489ce19307c7bda Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_base.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_binary.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_binary.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6ab3cbb61dde0a091bc5089d42972ba0a03f1b1a Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_binary.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_bool.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_bool.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0dbc286e8892c1ab6d5d12950dfc1456bdf45f8f Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_bool.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_bytes.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_bytes.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8a707c99a967e4df88da8ef71bed3b5ab56c27e8 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_bytes.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_datetime.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_datetime.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ed5ce05b7cf61aae70416fb5b8db483f75b2acd0 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_datetime.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_dictionary.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_dictionary.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..98398157ca6bbc8348b518eb649ec98344a744b5 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_dictionary.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_infinity.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_infinity.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2374bacf74dafb4da99aaaebb9b0eef77b7d580e Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_infinity.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_integer.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_integer.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..40683b6fb62a4a81dc96a048bd6d9cce198364c7 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_integer.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_ipaddress.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_ipaddress.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cbaab2335ede335ee1b5f82264c8be7c7fbf4520 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_ipaddress.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_list.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_list.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3a4899f40335ab618ea3e8f2909a6ad1e6f19e9e Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_list.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_nan.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_nan.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1a14ef17b8bb1dabf854f649f2aee700b4fea55d Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_nan.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_none.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_none.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..19f3692c903df2f82625ac8a4669ddc6f6acb8fd Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_none.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_realnumber.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_realnumber.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..92980e14fe07fde273eb584e3f0e54bb7d241e8e Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_realnumber.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_string.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_string.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c2868f2e4b5af7ddfcc67b8f94bdaf4c235f6c5f Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/__pycache__/_string.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/_datetime.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/_datetime.py new file mode 100644 index 0000000000000000000000000000000000000000..c65d19d703584efe2df05cf8e1e5556b48cea56e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/_datetime.py @@ -0,0 +1,34 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +from typing import Any + +from .._typecode import Typecode +from ..checker import DateTimeTypeChecker +from ..converter import DateTimeConverter +from ._base import AbstractType + + +class DateTime(AbstractType): + """ + |result_matrix_desc| + + .. include:: matrix_datetime_type.txt + + :py:attr:`.strict_level` + |strict_level| + """ + + @property + def typecode(self) -> Typecode: + return Typecode.DATETIME + + def __init__(self, value: Any, strict_level: int = 2, **kwargs) -> None: + super().__init__(value, strict_level, **kwargs) + + def _create_type_checker(self): + return DateTimeTypeChecker(self._data, self._strict_level) + + def _create_type_converter(self): + return DateTimeConverter(self._data, self._params) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/_infinity.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/_infinity.py new file mode 100644 index 0000000000000000000000000000000000000000..687d6fa7ec6e64831f7a19404863f364e8e9bcc6 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/_infinity.py @@ -0,0 +1,34 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +from typing import Any + +from .._typecode import Typecode +from ..checker import InfinityTypeChecker +from ..converter import FloatConverter +from ._base import AbstractType + + +class Infinity(AbstractType): + """ + |result_matrix_desc| + + .. include:: matrix_infinity_type.txt + + :py:attr:`.strict_level` + |strict_level| + """ + + @property + def typecode(self) -> Typecode: + return Typecode.INFINITY + + def __init__(self, value: Any, strict_level: int = 1, **kwargs) -> None: + super().__init__(value, strict_level, **kwargs) + + def _create_type_checker(self): + return InfinityTypeChecker(self._data, self._strict_level) + + def _create_type_converter(self): + return FloatConverter(self._data, self._params) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/_integer.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/_integer.py new file mode 100644 index 0000000000000000000000000000000000000000..992b1f7a1cee4c03951df0fae593a4571be7ecba --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/_integer.py @@ -0,0 +1,36 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +from typing import Any + +from .._typecode import Typecode +from ._base import AbstractType + + +class Integer(AbstractType): + """ + |result_matrix_desc| + + .. include:: matrix_integer_type.txt + + :py:attr:`.strict_level` + |strict_level| + """ + + @property + def typecode(self) -> Typecode: + return Typecode.INTEGER + + def __init__(self, value: Any, strict_level: int = 1, **kwargs) -> None: + super().__init__(value, strict_level, **kwargs) + + def _create_type_checker(self): + from ..checker._integer import IntegerTypeChecker + + return IntegerTypeChecker(self._data, self._strict_level) + + def _create_type_converter(self): + from ..converter._integer import IntegerConverter + + return IntegerConverter(self._data, self._params) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/_nan.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/_nan.py new file mode 100644 index 0000000000000000000000000000000000000000..fcf630f1e7a5b805b277ce655e46ca7109068dca --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typepy/type/_nan.py @@ -0,0 +1,34 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +from typing import Any + +from .._typecode import Typecode +from ..checker import NanTypeChecker +from ..converter import FloatConverter +from ._base import AbstractType + + +class Nan(AbstractType): + """ + |result_matrix_desc| + + .. include:: matrix_nan_type.txt + + :py:attr:`.strict_level` + |strict_level| + """ + + @property + def typecode(self) -> Typecode: + return Typecode.NAN + + def __init__(self, value: Any, strict_level: int = 1, **kwargs) -> None: + super().__init__(value, strict_level, **kwargs) + + def _create_type_checker(self): + return NanTypeChecker(self._data, self._strict_level) + + def _create_type_converter(self): + return FloatConverter(self._data, self._params) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Algiers b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Algiers new file mode 100644 index 0000000000000000000000000000000000000000..56a4dd2a19fac5cc1bb1951dedf3ae93e0b9e321 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Algiers differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Banjul b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Banjul new file mode 100644 index 0000000000000000000000000000000000000000..8906e88c819d9ad3b794eb6356a240a74a95ffae Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Banjul differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Conakry b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Conakry new file mode 100644 index 0000000000000000000000000000000000000000..8906e88c819d9ad3b794eb6356a240a74a95ffae Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Conakry differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Johannesburg b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Johannesburg new file mode 100644 index 0000000000000000000000000000000000000000..bada0638f8a2224a603f13afff16c0d8e986a591 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Johannesburg differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Khartoum b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Khartoum new file mode 100644 index 0000000000000000000000000000000000000000..3f8e44b8a6e171a0fde96736ed9d4fcde1bcd4a8 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Khartoum differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Kigali b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Kigali new file mode 100644 index 0000000000000000000000000000000000000000..581bb0e08b616a433d422ccb8f958cbebdae1770 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Kigali differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Kinshasa b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Kinshasa new file mode 100644 index 0000000000000000000000000000000000000000..3d7a71ba0e96de946446fc96add2f38a806a44a5 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Kinshasa differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Libreville b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Libreville new file mode 100644 index 0000000000000000000000000000000000000000..3d7a71ba0e96de946446fc96add2f38a806a44a5 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Libreville differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Lome b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Lome new file mode 100644 index 0000000000000000000000000000000000000000..8906e88c819d9ad3b794eb6356a240a74a95ffae Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Lome differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Luanda b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Luanda new file mode 100644 index 0000000000000000000000000000000000000000..3d7a71ba0e96de946446fc96add2f38a806a44a5 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Luanda differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Malabo b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Malabo new file mode 100644 index 0000000000000000000000000000000000000000..3d7a71ba0e96de946446fc96add2f38a806a44a5 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Malabo differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Monrovia b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Monrovia new file mode 100644 index 0000000000000000000000000000000000000000..837780922f23fc58ff7f73930954840e9c63c908 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Monrovia differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Niamey b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Niamey new file mode 100644 index 0000000000000000000000000000000000000000..3d7a71ba0e96de946446fc96add2f38a806a44a5 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Niamey differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Nouakchott b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Nouakchott new file mode 100644 index 0000000000000000000000000000000000000000..8906e88c819d9ad3b794eb6356a240a74a95ffae Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Nouakchott differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Ouagadougou b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Ouagadougou new file mode 100644 index 0000000000000000000000000000000000000000..8906e88c819d9ad3b794eb6356a240a74a95ffae Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Ouagadougou differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Porto-Novo b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Porto-Novo new file mode 100644 index 0000000000000000000000000000000000000000..3d7a71ba0e96de946446fc96add2f38a806a44a5 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Africa/Porto-Novo differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/America/Costa_Rica b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/America/Costa_Rica new file mode 100644 index 0000000000000000000000000000000000000000..08f0128ee681d8f7e1df186d93514f3f4cff2830 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/America/Costa_Rica differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/America/St_Johns b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/America/St_Johns new file mode 100644 index 0000000000000000000000000000000000000000..94d790baaccb72298bb577041cf3c8400339a7da Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/America/St_Johns differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/America/St_Lucia b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/America/St_Lucia new file mode 100644 index 0000000000000000000000000000000000000000..47b4dc34160dd3ff97ebc35ccdc99fcf49c7ffbb Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/America/St_Lucia differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/America/Vancouver b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/America/Vancouver new file mode 100644 index 0000000000000000000000000000000000000000..c998491112ea5e4430b8266498cf7f23e1266bc5 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/America/Vancouver differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/CET b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/CET new file mode 100644 index 0000000000000000000000000000000000000000..31973271d2f87f7e9df4e8b0a1481f09a9e5407d Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/CET differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/CST6CDT b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/CST6CDT new file mode 100644 index 0000000000000000000000000000000000000000..b016880653929aa40dd5ac0e82e4094a9d787cdf Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/CST6CDT differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Cuba b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Cuba new file mode 100644 index 0000000000000000000000000000000000000000..e06629d36841463326ff3350bc2f94d0417c3cdd Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Cuba differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/EST b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/EST new file mode 100644 index 0000000000000000000000000000000000000000..9154643f4c9189998392afb8a93e2e2eb9eaecf5 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/EST differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/EST5EDT b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/EST5EDT new file mode 100644 index 0000000000000000000000000000000000000000..2b6c2eea14df07392729ae9f5712a44ec4f02bae Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/EST5EDT differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Egypt b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Egypt new file mode 100644 index 0000000000000000000000000000000000000000..1e6d48d1ca4e5416913c41e8814dc045c57d5b58 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Egypt differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Eire b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Eire new file mode 100644 index 0000000000000000000000000000000000000000..17d2b1582df89d5794f20fb028956dd9da154922 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Eire differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Factory b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Factory new file mode 100644 index 0000000000000000000000000000000000000000..b4dd7735ed5b945e3403f2dd7cd57712dc9184d3 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Factory differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/GB b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/GB new file mode 100644 index 0000000000000000000000000000000000000000..b9e95d92623c6cc4c35b16f7ceba3006f5ce4934 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/GB differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/GB-Eire b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/GB-Eire new file mode 100644 index 0000000000000000000000000000000000000000..b9e95d92623c6cc4c35b16f7ceba3006f5ce4934 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/GB-Eire differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/GMT b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/GMT new file mode 100644 index 0000000000000000000000000000000000000000..157573b1d340e0f57a0dd4d9698bd3798cbf7136 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/GMT differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/GMT+0 b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/GMT+0 new file mode 100644 index 0000000000000000000000000000000000000000..157573b1d340e0f57a0dd4d9698bd3798cbf7136 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/GMT+0 differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/GMT-0 b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/GMT-0 new file mode 100644 index 0000000000000000000000000000000000000000..157573b1d340e0f57a0dd4d9698bd3798cbf7136 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/GMT-0 differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/GMT0 b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/GMT0 new file mode 100644 index 0000000000000000000000000000000000000000..157573b1d340e0f57a0dd4d9698bd3798cbf7136 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/GMT0 differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Greenwich b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Greenwich new file mode 100644 index 0000000000000000000000000000000000000000..157573b1d340e0f57a0dd4d9698bd3798cbf7136 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Greenwich differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/HST b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/HST new file mode 100644 index 0000000000000000000000000000000000000000..40e3d492e6c22c30041c31f159d4fe0ee9451c03 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/HST differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Hongkong b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Hongkong new file mode 100644 index 0000000000000000000000000000000000000000..c80e364801be87687625f72e8e2c3dbd0f7ae4bc Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Hongkong differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Iceland b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Iceland new file mode 100644 index 0000000000000000000000000000000000000000..8906e88c819d9ad3b794eb6356a240a74a95ffae Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Iceland differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Iran b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Iran new file mode 100644 index 0000000000000000000000000000000000000000..824acb0426faaf659a7f209dda6e1ffd0f3ce2ec Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Iran differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Jamaica b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Jamaica new file mode 100644 index 0000000000000000000000000000000000000000..be6b1b6f1e77a8f13a7400bcfc10c63a7ee1d55d Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Jamaica differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Japan b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Japan new file mode 100644 index 0000000000000000000000000000000000000000..1aa066ce38fce7bd0a680f51d6f075718d153a77 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Japan differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Kwajalein b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Kwajalein new file mode 100644 index 0000000000000000000000000000000000000000..9416d522d0a3e19ab6b81317f5b94961c55e91fc Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Kwajalein differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/MET b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/MET new file mode 100644 index 0000000000000000000000000000000000000000..31973271d2f87f7e9df4e8b0a1481f09a9e5407d Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/MET differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/MST b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/MST new file mode 100644 index 0000000000000000000000000000000000000000..c2bd2f949b248b835c98216b4dc66f9f6eb0265e Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/MST differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/MST7MDT b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/MST7MDT new file mode 100644 index 0000000000000000000000000000000000000000..09e54e5c7c5bb2384e37626d4b985cfad29ed29b Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/MST7MDT differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/NZ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/NZ new file mode 100644 index 0000000000000000000000000000000000000000..afb3929318475d4f0ea9d6c9e94d0f5f81d8b82e Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/NZ differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/NZ-CHAT b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/NZ-CHAT new file mode 100644 index 0000000000000000000000000000000000000000..f06065ebd18315683f60cf87839d477a1d699f01 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/NZ-CHAT differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Navajo b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Navajo new file mode 100644 index 0000000000000000000000000000000000000000..09e54e5c7c5bb2384e37626d4b985cfad29ed29b Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Navajo differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/PRC b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/PRC new file mode 100644 index 0000000000000000000000000000000000000000..d6b66984a2f36ae36b35e174756707aa7286c292 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/PRC differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/PST8PDT b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/PST8PDT new file mode 100644 index 0000000000000000000000000000000000000000..aaf07787ad92b65eadae63b64bba290f9f961507 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/PST8PDT differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Portugal b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Portugal new file mode 100644 index 0000000000000000000000000000000000000000..7e9aae727b2b660e7f5e383121f445daf033a9c5 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Portugal differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/ROC b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/ROC new file mode 100644 index 0000000000000000000000000000000000000000..35d89d036d07c3f28dec64092ab1b533c21ae2bc Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/ROC differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/ROK b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/ROK new file mode 100644 index 0000000000000000000000000000000000000000..1755147fab44e07b7527ce1eaf3ae991473fb222 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/ROK differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Singapore b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Singapore new file mode 100644 index 0000000000000000000000000000000000000000..dbbdea3c8149004cfd525a0fc26e5da72b20e8a1 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Singapore differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Turkey b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Turkey new file mode 100644 index 0000000000000000000000000000000000000000..c89186687300068ac4e8505cc0012a1dbf6a9960 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Turkey differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/UCT b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/UCT new file mode 100644 index 0000000000000000000000000000000000000000..00841a62213e6cccf7f0b7353e5e8ae214185486 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/UCT differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/UTC b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/UTC new file mode 100644 index 0000000000000000000000000000000000000000..00841a62213e6cccf7f0b7353e5e8ae214185486 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/UTC differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Universal b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Universal new file mode 100644 index 0000000000000000000000000000000000000000..00841a62213e6cccf7f0b7353e5e8ae214185486 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Universal differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Zulu b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Zulu new file mode 100644 index 0000000000000000000000000000000000000000..00841a62213e6cccf7f0b7353e5e8ae214185486 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/Zulu differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/iso3166.tab b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/iso3166.tab new file mode 100644 index 0000000000000000000000000000000000000000..402c015ec6b1071499155f7d28739db68be7763f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/iso3166.tab @@ -0,0 +1,279 @@ +# ISO 3166 alpha-2 country codes +# +# This file is in the public domain, so clarified as of +# 2009-05-17 by Arthur David Olson. +# +# From Paul Eggert (2023-09-06): +# This file contains a table of two-letter country codes. Columns are +# separated by a single tab. Lines beginning with '#' are comments. +# All text uses UTF-8 encoding. The columns of the table are as follows: +# +# 1. ISO 3166-1 alpha-2 country code, current as of +# ISO/TC 46 N1108 (2023-04-05). See: ISO/TC 46 Documents +# https://www.iso.org/committee/48750.html?view=documents +# 2. The usual English name for the coded region. This sometimes +# departs from ISO-listed names, sometimes so that sorted subsets +# of names are useful (e.g., "Samoa (American)" and "Samoa +# (western)" rather than "American Samoa" and "Samoa"), +# sometimes to avoid confusion among non-experts (e.g., +# "Czech Republic" and "Turkey" rather than "Czechia" and "Türkiye"), +# and sometimes to omit needless detail or churn (e.g., "Netherlands" +# rather than "Netherlands (the)" or "Netherlands (Kingdom of the)"). +# +# The table is sorted by country code. +# +# This table is intended as an aid for users, to help them select time +# zone data appropriate for their practical needs. It is not intended +# to take or endorse any position on legal or territorial claims. +# +#country- +#code name of country, territory, area, or subdivision +AD Andorra +AE United Arab Emirates +AF Afghanistan +AG Antigua & Barbuda +AI Anguilla +AL Albania +AM Armenia +AO Angola +AQ Antarctica +AR Argentina +AS Samoa (American) +AT Austria +AU Australia +AW Aruba +AX Åland Islands +AZ Azerbaijan +BA Bosnia & Herzegovina +BB Barbados +BD Bangladesh +BE Belgium +BF Burkina Faso +BG Bulgaria +BH Bahrain +BI Burundi +BJ Benin +BL St Barthelemy +BM Bermuda +BN Brunei +BO Bolivia +BQ Caribbean NL +BR Brazil +BS Bahamas +BT Bhutan +BV Bouvet Island +BW Botswana +BY Belarus +BZ Belize +CA Canada +CC Cocos (Keeling) Islands +CD Congo (Dem. Rep.) +CF Central African Rep. +CG Congo (Rep.) +CH Switzerland +CI Côte d'Ivoire +CK Cook Islands +CL Chile +CM Cameroon +CN China +CO Colombia +CR Costa Rica +CU Cuba +CV Cape Verde +CW Curaçao +CX Christmas Island +CY Cyprus +CZ Czech Republic +DE Germany +DJ Djibouti +DK Denmark +DM Dominica +DO Dominican Republic +DZ Algeria +EC Ecuador +EE Estonia +EG Egypt +EH Western Sahara +ER Eritrea +ES Spain +ET Ethiopia +FI Finland +FJ Fiji +FK Falkland Islands +FM Micronesia +FO Faroe Islands +FR France +GA Gabon +GB Britain (UK) +GD Grenada +GE Georgia +GF French Guiana +GG Guernsey +GH Ghana +GI Gibraltar +GL Greenland +GM Gambia +GN Guinea +GP Guadeloupe +GQ Equatorial Guinea +GR Greece +GS South Georgia & the South Sandwich Islands +GT Guatemala +GU Guam +GW Guinea-Bissau +GY Guyana +HK Hong Kong +HM Heard Island & McDonald Islands +HN Honduras +HR Croatia +HT Haiti +HU Hungary +ID Indonesia +IE Ireland +IL Israel +IM Isle of Man +IN India +IO British Indian Ocean Territory +IQ Iraq +IR Iran +IS Iceland +IT Italy +JE Jersey +JM Jamaica +JO Jordan +JP Japan +KE Kenya +KG Kyrgyzstan +KH Cambodia +KI Kiribati +KM Comoros +KN St Kitts & Nevis +KP Korea (North) +KR Korea (South) +KW Kuwait +KY Cayman Islands +KZ Kazakhstan +LA Laos +LB Lebanon +LC St Lucia +LI Liechtenstein +LK Sri Lanka +LR Liberia +LS Lesotho +LT Lithuania +LU Luxembourg +LV Latvia +LY Libya +MA Morocco +MC Monaco +MD Moldova +ME Montenegro +MF St Martin (French) +MG Madagascar +MH Marshall Islands +MK North Macedonia +ML Mali +MM Myanmar (Burma) +MN Mongolia +MO Macau +MP Northern Mariana Islands +MQ Martinique +MR Mauritania +MS Montserrat +MT Malta +MU Mauritius +MV Maldives +MW Malawi +MX Mexico +MY Malaysia +MZ Mozambique +NA Namibia +NC New Caledonia +NE Niger +NF Norfolk Island +NG Nigeria +NI Nicaragua +NL Netherlands +NO Norway +NP Nepal +NR Nauru +NU Niue +NZ New Zealand +OM Oman +PA Panama +PE Peru +PF French Polynesia +PG Papua New Guinea +PH Philippines +PK Pakistan +PL Poland +PM St Pierre & Miquelon +PN Pitcairn +PR Puerto Rico +PS Palestine +PT Portugal +PW Palau +PY Paraguay +QA Qatar +RE Réunion +RO Romania +RS Serbia +RU Russia +RW Rwanda +SA Saudi Arabia +SB Solomon Islands +SC Seychelles +SD Sudan +SE Sweden +SG Singapore +SH St Helena +SI Slovenia +SJ Svalbard & Jan Mayen +SK Slovakia +SL Sierra Leone +SM San Marino +SN Senegal +SO Somalia +SR Suriname +SS South Sudan +ST Sao Tome & Principe +SV El Salvador +SX St Maarten (Dutch) +SY Syria +SZ Eswatini (Swaziland) +TC Turks & Caicos Is +TD Chad +TF French S. Terr. +TG Togo +TH Thailand +TJ Tajikistan +TK Tokelau +TL East Timor +TM Turkmenistan +TN Tunisia +TO Tonga +TR Turkey +TT Trinidad & Tobago +TV Tuvalu +TW Taiwan +TZ Tanzania +UA Ukraine +UG Uganda +UM US minor outlying islands +US United States +UY Uruguay +UZ Uzbekistan +VA Vatican City +VC St Vincent +VE Venezuela +VG Virgin Islands (UK) +VI Virgin Islands (US) +VN Vietnam +VU Vanuatu +WF Wallis & Futuna +WS Samoa (western) +YE Yemen +YT Mayotte +ZA South Africa +ZM Zambia +ZW Zimbabwe diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/leapseconds b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/leapseconds new file mode 100644 index 0000000000000000000000000000000000000000..76f771427f25b91e6944ffd2fee6031bfd73979a --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/leapseconds @@ -0,0 +1,79 @@ +# Allowance for leap seconds added to each time zone file. + +# This file is in the public domain. + +# This file is generated automatically from the data in the public-domain +# NIST/IERS format leap-seconds.list file, which can be copied from +# +# or, in a variant with different comments, from +# . +# For more about leap-seconds.list, please see +# The NTP Timescale and Leap Seconds +# . + +# The rules for leap seconds are specified in Annex 1 (Time scales) of: +# Standard-frequency and time-signal emissions. +# International Telecommunication Union - Radiocommunication Sector +# (ITU-R) Recommendation TF.460-6 (02/2002) +# . +# The International Earth Rotation and Reference Systems Service (IERS) +# periodically uses leap seconds to keep UTC to within 0.9 s of UT1 +# (a proxy for Earth's angle in space as measured by astronomers) +# and publishes leap second data in a copyrighted file +# . +# See: Levine J. Coordinated Universal Time and the leap second. +# URSI Radio Sci Bull. 2016;89(4):30-6. doi:10.23919/URSIRSB.2016.7909995 +# . + +# There were no leap seconds before 1972, as no official mechanism +# accounted for the discrepancy between atomic time (TAI) and the earth's +# rotation. The first ("1 Jan 1972") data line in leap-seconds.list +# does not denote a leap second; it denotes the start of the current definition +# of UTC. + +# All leap-seconds are Stationary (S) at the given UTC time. +# The correction (+ or -) is made at the given time, so in the unlikely +# event of a negative leap second, a line would look like this: +# Leap YEAR MON DAY 23:59:59 - S +# Typical lines look like this: +# Leap YEAR MON DAY 23:59:60 + S +Leap 1972 Jun 30 23:59:60 + S +Leap 1972 Dec 31 23:59:60 + S +Leap 1973 Dec 31 23:59:60 + S +Leap 1974 Dec 31 23:59:60 + S +Leap 1975 Dec 31 23:59:60 + S +Leap 1976 Dec 31 23:59:60 + S +Leap 1977 Dec 31 23:59:60 + S +Leap 1978 Dec 31 23:59:60 + S +Leap 1979 Dec 31 23:59:60 + S +Leap 1981 Jun 30 23:59:60 + S +Leap 1982 Jun 30 23:59:60 + S +Leap 1983 Jun 30 23:59:60 + S +Leap 1985 Jun 30 23:59:60 + S +Leap 1987 Dec 31 23:59:60 + S +Leap 1989 Dec 31 23:59:60 + S +Leap 1990 Dec 31 23:59:60 + S +Leap 1992 Jun 30 23:59:60 + S +Leap 1993 Jun 30 23:59:60 + S +Leap 1994 Jun 30 23:59:60 + S +Leap 1995 Dec 31 23:59:60 + S +Leap 1997 Jun 30 23:59:60 + S +Leap 1998 Dec 31 23:59:60 + S +Leap 2005 Dec 31 23:59:60 + S +Leap 2008 Dec 31 23:59:60 + S +Leap 2012 Jun 30 23:59:60 + S +Leap 2015 Jun 30 23:59:60 + S +Leap 2016 Dec 31 23:59:60 + S + +# UTC timestamp when this leap second list expires. +# Any additional leap seconds will come after this. +# This Expires line is commented out for now, +# so that pre-2020a zic implementations do not reject this file. +#Expires 2025 Dec 28 00:00:00 + +# POSIX timestamps for the data in this file: +#updated 1736208000 (2025-01-07 00:00:00 UTC) +#expires 1766880000 (2025-12-28 00:00:00 UTC) + +# Updated through IERS Bulletin C (https://hpiers.obspm.fr/iers/bul/bulc/bulletinc.dat) +# File expires on 28 December 2025 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/tzdata.zi b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/tzdata.zi new file mode 100644 index 0000000000000000000000000000000000000000..db6ba4af2b013d9886dd3b81dd4f7f0b34ebbb02 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/tzdata.zi @@ -0,0 +1,4286 @@ +# version 2025a +# This zic input file is in the public domain. +R d 1916 o - Jun 14 23s 1 S +R d 1916 1919 - O Su>=1 23s 0 - +R d 1917 o - Mar 24 23s 1 S +R d 1918 o - Mar 9 23s 1 S +R d 1919 o - Mar 1 23s 1 S +R d 1920 o - F 14 23s 1 S +R d 1920 o - O 23 23s 0 - +R d 1921 o - Mar 14 23s 1 S +R d 1921 o - Jun 21 23s 0 - +R d 1939 o - S 11 23s 1 S +R d 1939 o - N 19 1 0 - +R d 1944 1945 - Ap M>=1 2 1 S +R d 1944 o - O 8 2 0 - +R d 1945 o - S 16 1 0 - +R d 1971 o - Ap 25 23s 1 S +R d 1971 o - S 26 23s 0 - +R d 1977 o - May 6 0 1 S +R d 1977 o - O 21 0 0 - +R d 1978 o - Mar 24 1 1 S +R d 1978 o - S 22 3 0 - +R d 1980 o - Ap 25 0 1 S +R d 1980 o - O 31 2 0 - +R K 1940 o - Jul 15 0 1 S +R K 1940 o - O 1 0 0 - +R K 1941 o - Ap 15 0 1 S +R K 1941 o - S 16 0 0 - +R K 1942 1944 - Ap 1 0 1 S +R K 1942 o - O 27 0 0 - +R K 1943 1945 - N 1 0 0 - +R K 1945 o - Ap 16 0 1 S +R K 1957 o - May 10 0 1 S +R K 1957 1958 - O 1 0 0 - +R K 1958 o - May 1 0 1 S +R K 1959 1981 - May 1 1 1 S +R K 1959 1965 - S 30 3 0 - +R K 1966 1994 - O 1 3 0 - +R K 1982 o - Jul 25 1 1 S +R K 1983 o - Jul 12 1 1 S +R K 1984 1988 - May 1 1 1 S +R K 1989 o - May 6 1 1 S +R K 1990 1994 - May 1 1 1 S +R K 1995 2010 - Ap lastF 0s 1 S +R K 1995 2005 - S lastTh 24 0 - +R K 2006 o - S 21 24 0 - +R K 2007 o - S Th>=1 24 0 - +R K 2008 o - Au lastTh 24 0 - +R K 2009 o - Au 20 24 0 - +R K 2010 o - Au 10 24 0 - +R K 2010 o - S 9 24 1 S +R K 2010 o - S lastTh 24 0 - +R K 2014 o - May 15 24 1 S +R K 2014 o - Jun 26 24 0 - +R K 2014 o - Jul 31 24 1 S +R K 2014 o - S lastTh 24 0 - +R K 2023 ma - Ap lastF 0 1 S +R K 2023 ma - O lastTh 24 0 - +R L 1951 o - O 14 2 1 S +R L 1952 o - Ja 1 0 0 - +R L 1953 o - O 9 2 1 S +R L 1954 o - Ja 1 0 0 - +R L 1955 o - S 30 0 1 S +R L 1956 o - Ja 1 0 0 - +R L 1982 1984 - Ap 1 0 1 S +R L 1982 1985 - O 1 0 0 - +R L 1985 o - Ap 6 0 1 S +R L 1986 o - Ap 4 0 1 S +R L 1986 o - O 3 0 0 - +R L 1987 1989 - Ap 1 0 1 S +R L 1987 1989 - O 1 0 0 - +R L 1997 o - Ap 4 0 1 S +R L 1997 o - O 4 0 0 - +R L 2013 o - Mar lastF 1 1 S +R L 2013 o - O lastF 2 0 - +R MU 1982 o - O 10 0 1 - +R MU 1983 o - Mar 21 0 0 - +R MU 2008 o - O lastSu 2 1 - +R MU 2009 o - Mar lastSu 2 0 - +R M 1939 o - S 12 0 1 - +R M 1939 o - N 19 0 0 - +R M 1940 o - F 25 0 1 - +R M 1945 o - N 18 0 0 - +R M 1950 o - Jun 11 0 1 - +R M 1950 o - O 29 0 0 - +R M 1967 o - Jun 3 12 1 - +R M 1967 o - O 1 0 0 - +R M 1974 o - Jun 24 0 1 - +R M 1974 o - S 1 0 0 - +R M 1976 1977 - May 1 0 1 - +R M 1976 o - Au 1 0 0 - +R M 1977 o - S 28 0 0 - +R M 1978 o - Jun 1 0 1 - +R M 1978 o - Au 4 0 0 - +R M 2008 o - Jun 1 0 1 - +R M 2008 o - S 1 0 0 - +R M 2009 o - Jun 1 0 1 - +R M 2009 o - Au 21 0 0 - +R M 2010 o - May 2 0 1 - +R M 2010 o - Au 8 0 0 - +R M 2011 o - Ap 3 0 1 - +R M 2011 o - Jul 31 0 0 - +R M 2012 2013 - Ap lastSu 2 1 - +R M 2012 o - Jul 20 3 0 - +R M 2012 o - Au 20 2 1 - +R M 2012 o - S 30 3 0 - +R M 2013 o - Jul 7 3 0 - +R M 2013 o - Au 10 2 1 - +R M 2013 2018 - O lastSu 3 0 - +R M 2014 2018 - Mar lastSu 2 1 - +R M 2014 o - Jun 28 3 0 - +R M 2014 o - Au 2 2 1 - +R M 2015 o - Jun 14 3 0 - +R M 2015 o - Jul 19 2 1 - +R M 2016 o - Jun 5 3 0 - +R M 2016 o - Jul 10 2 1 - +R M 2017 o - May 21 3 0 - +R M 2017 o - Jul 2 2 1 - +R M 2018 o - May 13 3 0 - +R M 2018 o - Jun 17 2 1 - +R M 2019 o - May 5 3 -1 - +R M 2019 o - Jun 9 2 0 - +R M 2020 o - Ap 19 3 -1 - +R M 2020 o - May 31 2 0 - +R M 2021 o - Ap 11 3 -1 - +R M 2021 o - May 16 2 0 - +R M 2022 o - Mar 27 3 -1 - +R M 2022 o - May 8 2 0 - +R M 2023 o - Mar 19 3 -1 - +R M 2023 o - Ap 23 2 0 - +R M 2024 o - Mar 10 3 -1 - +R M 2024 o - Ap 14 2 0 - +R M 2025 o - F 23 3 -1 - +R M 2025 o - Ap 6 2 0 - +R M 2026 o - F 15 3 -1 - +R M 2026 o - Mar 22 2 0 - +R M 2027 o - F 7 3 -1 - +R M 2027 o - Mar 14 2 0 - +R M 2028 o - Ja 23 3 -1 - +R M 2028 o - Mar 5 2 0 - +R M 2029 o - Ja 14 3 -1 - +R M 2029 o - F 18 2 0 - +R M 2029 o - D 30 3 -1 - +R M 2030 o - F 10 2 0 - +R M 2030 o - D 22 3 -1 - +R M 2031 o - Ja 26 2 0 - +R M 2031 o - D 14 3 -1 - +R M 2032 o - Ja 18 2 0 - +R M 2032 o - N 28 3 -1 - +R M 2033 o - Ja 9 2 0 - +R M 2033 o - N 20 3 -1 - +R M 2033 o - D 25 2 0 - +R M 2034 o - N 5 3 -1 - +R M 2034 o - D 17 2 0 - +R M 2035 o - O 28 3 -1 - +R M 2035 o - D 9 2 0 - +R M 2036 o - O 19 3 -1 - +R M 2036 o - N 23 2 0 - +R M 2037 o - O 4 3 -1 - +R M 2037 o - N 15 2 0 - +R M 2038 o - S 26 3 -1 - +R M 2038 o - O 31 2 0 - +R M 2039 o - S 18 3 -1 - +R M 2039 o - O 23 2 0 - +R M 2040 o - S 2 3 -1 - +R M 2040 o - O 14 2 0 - +R M 2041 o - Au 25 3 -1 - +R M 2041 o - S 29 2 0 - +R M 2042 o - Au 10 3 -1 - +R M 2042 o - S 21 2 0 - +R M 2043 o - Au 2 3 -1 - +R M 2043 o - S 13 2 0 - +R M 2044 o - Jul 24 3 -1 - +R M 2044 o - Au 28 2 0 - +R M 2045 o - Jul 9 3 -1 - +R M 2045 o - Au 20 2 0 - +R M 2046 o - Jul 1 3 -1 - +R M 2046 o - Au 5 2 0 - +R M 2047 o - Jun 23 3 -1 - +R M 2047 o - Jul 28 2 0 - +R M 2048 o - Jun 7 3 -1 - +R M 2048 o - Jul 19 2 0 - +R M 2049 o - May 30 3 -1 - +R M 2049 o - Jul 4 2 0 - +R M 2050 o - May 15 3 -1 - +R M 2050 o - Jun 26 2 0 - +R M 2051 o - May 7 3 -1 - +R M 2051 o - Jun 18 2 0 - +R M 2052 o - Ap 28 3 -1 - +R M 2052 o - Jun 2 2 0 - +R M 2053 o - Ap 13 3 -1 - +R M 2053 o - May 25 2 0 - +R M 2054 o - Ap 5 3 -1 - +R M 2054 o - May 10 2 0 - +R M 2055 o - Mar 28 3 -1 - +R M 2055 o - May 2 2 0 - +R M 2056 o - Mar 12 3 -1 - +R M 2056 o - Ap 23 2 0 - +R M 2057 o - Mar 4 3 -1 - +R M 2057 o - Ap 8 2 0 - +R M 2058 o - F 17 3 -1 - +R M 2058 o - Mar 31 2 0 - +R M 2059 o - F 9 3 -1 - +R M 2059 o - Mar 23 2 0 - +R M 2060 o - F 1 3 -1 - +R M 2060 o - Mar 7 2 0 - +R M 2061 o - Ja 16 3 -1 - +R M 2061 o - F 27 2 0 - +R M 2062 o - Ja 8 3 -1 - +R M 2062 o - F 12 2 0 - +R M 2062 o - D 31 3 -1 - +R M 2063 o - F 4 2 0 - +R M 2063 o - D 16 3 -1 - +R M 2064 o - Ja 27 2 0 - +R M 2064 o - D 7 3 -1 - +R M 2065 o - Ja 11 2 0 - +R M 2065 o - N 22 3 -1 - +R M 2066 o - Ja 3 2 0 - +R M 2066 o - N 14 3 -1 - +R M 2066 o - D 26 2 0 - +R M 2067 o - N 6 3 -1 - +R M 2067 o - D 11 2 0 - +R M 2068 o - O 21 3 -1 - +R M 2068 o - D 2 2 0 - +R M 2069 o - O 13 3 -1 - +R M 2069 o - N 17 2 0 - +R M 2070 o - O 5 3 -1 - +R M 2070 o - N 9 2 0 - +R M 2071 o - S 20 3 -1 - +R M 2071 o - N 1 2 0 - +R M 2072 o - S 11 3 -1 - +R M 2072 o - O 16 2 0 - +R M 2073 o - Au 27 3 -1 - +R M 2073 o - O 8 2 0 - +R M 2074 o - Au 19 3 -1 - +R M 2074 o - S 30 2 0 - +R M 2075 o - Au 11 3 -1 - +R M 2075 o - S 15 2 0 - +R M 2076 o - Jul 26 3 -1 - +R M 2076 o - S 6 2 0 - +R M 2077 o - Jul 18 3 -1 - +R M 2077 o - Au 22 2 0 - +R M 2078 o - Jul 10 3 -1 - +R M 2078 o - Au 14 2 0 - +R M 2079 o - Jun 25 3 -1 - +R M 2079 o - Au 6 2 0 - +R M 2080 o - Jun 16 3 -1 - +R M 2080 o - Jul 21 2 0 - +R M 2081 o - Jun 1 3 -1 - +R M 2081 o - Jul 13 2 0 - +R M 2082 o - May 24 3 -1 - +R M 2082 o - Jun 28 2 0 - +R M 2083 o - May 16 3 -1 - +R M 2083 o - Jun 20 2 0 - +R M 2084 o - Ap 30 3 -1 - +R M 2084 o - Jun 11 2 0 - +R M 2085 o - Ap 22 3 -1 - +R M 2085 o - May 27 2 0 - +R M 2086 o - Ap 14 3 -1 - +R M 2086 o - May 19 2 0 - +R M 2087 o - Mar 30 3 -1 - +R M 2087 o - May 11 2 0 - +R NA 1994 o - Mar 21 0 -1 WAT +R NA 1994 2017 - S Su>=1 2 0 CAT +R NA 1995 2017 - Ap Su>=1 2 -1 WAT +R SA 1942 1943 - S Su>=15 2 1 - +R SA 1943 1944 - Mar Su>=15 2 0 - +R SD 1970 o - May 1 0 1 S +R SD 1970 1985 - O 15 0 0 - +R SD 1971 o - Ap 30 0 1 S +R SD 1972 1985 - Ap lastSu 0 1 S +R n 1939 o - Ap 15 23s 1 S +R n 1939 o - N 18 23s 0 - +R n 1940 o - F 25 23s 1 S +R n 1941 o - O 6 0 0 - +R n 1942 o - Mar 9 0 1 S +R n 1942 o - N 2 3 0 - +R n 1943 o - Mar 29 2 1 S +R n 1943 o - Ap 17 2 0 - +R n 1943 o - Ap 25 2 1 S +R n 1943 o - O 4 2 0 - +R n 1944 1945 - Ap M>=1 2 1 S +R n 1944 o - O 8 0 0 - +R n 1945 o - S 16 0 0 - +R n 1977 o - Ap 30 0s 1 S +R n 1977 o - S 24 0s 0 - +R n 1978 o - May 1 0s 1 S +R n 1978 o - O 1 0s 0 - +R n 1988 o - Jun 1 0s 1 S +R n 1988 1990 - S lastSu 0s 0 - +R n 1989 o - Mar 26 0s 1 S +R n 1990 o - May 1 0s 1 S +R n 2005 o - May 1 0s 1 S +R n 2005 o - S 30 1s 0 - +R n 2006 2008 - Mar lastSu 2s 1 S +R n 2006 2008 - O lastSu 2s 0 - +R Tr 2005 ma - Mar lastSu 1u 2 +02 +R Tr 2004 ma - O lastSu 1u 0 +00 +R AM 2011 o - Mar lastSu 2s 1 - +R AM 2011 o - O lastSu 2s 0 - +R AZ 1997 2015 - Mar lastSu 4 1 - +R AZ 1997 2015 - O lastSu 5 0 - +R BD 2009 o - Jun 19 23 1 - +R BD 2009 o - D 31 24 0 - +R Sh 1919 o - Ap 12 24 1 D +R Sh 1919 o - S 30 24 0 S +R Sh 1940 o - Jun 1 0 1 D +R Sh 1940 o - O 12 24 0 S +R Sh 1941 o - Mar 15 0 1 D +R Sh 1941 o - N 1 24 0 S +R Sh 1942 o - Ja 31 0 1 D +R Sh 1945 o - S 1 24 0 S +R Sh 1946 o - May 15 0 1 D +R Sh 1946 o - S 30 24 0 S +R Sh 1947 o - Ap 15 0 1 D +R Sh 1947 o - O 31 24 0 S +R Sh 1948 1949 - May 1 0 1 D +R Sh 1948 1949 - S 30 24 0 S +R CN 1986 o - May 4 2 1 D +R CN 1986 1991 - S Su>=11 2 0 S +R CN 1987 1991 - Ap Su>=11 2 1 D +R HK 1946 o - Ap 21 0 1 S +R HK 1946 o - D 1 3:30s 0 - +R HK 1947 o - Ap 13 3:30s 1 S +R HK 1947 o - N 30 3:30s 0 - +R HK 1948 o - May 2 3:30s 1 S +R HK 1948 1952 - O Su>=28 3:30s 0 - +R HK 1949 1953 - Ap Su>=1 3:30 1 S +R HK 1953 1964 - O Su>=31 3:30 0 - +R HK 1954 1964 - Mar Su>=18 3:30 1 S +R HK 1965 1976 - Ap Su>=16 3:30 1 S +R HK 1965 1976 - O Su>=16 3:30 0 - +R HK 1973 o - D 30 3:30 1 S +R HK 1979 o - May 13 3:30 1 S +R HK 1979 o - O 21 3:30 0 - +R f 1946 o - May 15 0 1 D +R f 1946 o - O 1 0 0 S +R f 1947 o - Ap 15 0 1 D +R f 1947 o - N 1 0 0 S +R f 1948 1951 - May 1 0 1 D +R f 1948 1951 - O 1 0 0 S +R f 1952 o - Mar 1 0 1 D +R f 1952 1954 - N 1 0 0 S +R f 1953 1959 - Ap 1 0 1 D +R f 1955 1961 - O 1 0 0 S +R f 1960 1961 - Jun 1 0 1 D +R f 1974 1975 - Ap 1 0 1 D +R f 1974 1975 - O 1 0 0 S +R f 1979 o - Jul 1 0 1 D +R f 1979 o - O 1 0 0 S +R _ 1942 1943 - Ap 30 23 1 - +R _ 1942 o - N 17 23 0 - +R _ 1943 o - S 30 23 0 S +R _ 1946 o - Ap 30 23s 1 D +R _ 1946 o - S 30 23s 0 S +R _ 1947 o - Ap 19 23s 1 D +R _ 1947 o - N 30 23s 0 S +R _ 1948 o - May 2 23s 1 D +R _ 1948 o - O 31 23s 0 S +R _ 1949 1950 - Ap Sa>=1 23s 1 D +R _ 1949 1950 - O lastSa 23s 0 S +R _ 1951 o - Mar 31 23s 1 D +R _ 1951 o - O 28 23s 0 S +R _ 1952 1953 - Ap Sa>=1 23s 1 D +R _ 1952 o - N 1 23s 0 S +R _ 1953 1954 - O lastSa 23s 0 S +R _ 1954 1956 - Mar Sa>=17 23s 1 D +R _ 1955 o - N 5 23s 0 S +R _ 1956 1964 - N Su>=1 3:30 0 S +R _ 1957 1964 - Mar Su>=18 3:30 1 D +R _ 1965 1973 - Ap Su>=16 3:30 1 D +R _ 1965 1966 - O Su>=16 2:30 0 S +R _ 1967 1976 - O Su>=16 3:30 0 S +R _ 1973 o - D 30 3:30 1 D +R _ 1975 1976 - Ap Su>=16 3:30 1 D +R _ 1979 o - May 13 3:30 1 D +R _ 1979 o - O Su>=16 3:30 0 S +R CY 1975 o - Ap 13 0 1 S +R CY 1975 o - O 12 0 0 - +R CY 1976 o - May 15 0 1 S +R CY 1976 o - O 11 0 0 - +R CY 1977 1980 - Ap Su>=1 0 1 S +R CY 1977 o - S 25 0 0 - +R CY 1978 o - O 2 0 0 - +R CY 1979 1997 - S lastSu 0 0 - +R CY 1981 1998 - Mar lastSu 0 1 S +R i 1910 o - Ja 1 0 0 - +R i 1977 o - Mar 21 23 1 - +R i 1977 o - O 20 24 0 - +R i 1978 o - Mar 24 24 1 - +R i 1978 o - Au 5 1 0 - +R i 1979 o - May 26 24 1 - +R i 1979 o - S 18 24 0 - +R i 1980 o - Mar 20 24 1 - +R i 1980 o - S 22 24 0 - +R i 1991 o - May 2 24 1 - +R i 1992 1995 - Mar 21 24 1 - +R i 1991 1995 - S 21 24 0 - +R i 1996 o - Mar 20 24 1 - +R i 1996 o - S 20 24 0 - +R i 1997 1999 - Mar 21 24 1 - +R i 1997 1999 - S 21 24 0 - +R i 2000 o - Mar 20 24 1 - +R i 2000 o - S 20 24 0 - +R i 2001 2003 - Mar 21 24 1 - +R i 2001 2003 - S 21 24 0 - +R i 2004 o - Mar 20 24 1 - +R i 2004 o - S 20 24 0 - +R i 2005 o - Mar 21 24 1 - +R i 2005 o - S 21 24 0 - +R i 2008 o - Mar 20 24 1 - +R i 2008 o - S 20 24 0 - +R i 2009 2011 - Mar 21 24 1 - +R i 2009 2011 - S 21 24 0 - +R i 2012 o - Mar 20 24 1 - +R i 2012 o - S 20 24 0 - +R i 2013 2015 - Mar 21 24 1 - +R i 2013 2015 - S 21 24 0 - +R i 2016 o - Mar 20 24 1 - +R i 2016 o - S 20 24 0 - +R i 2017 2019 - Mar 21 24 1 - +R i 2017 2019 - S 21 24 0 - +R i 2020 o - Mar 20 24 1 - +R i 2020 o - S 20 24 0 - +R i 2021 2022 - Mar 21 24 1 - +R i 2021 2022 - S 21 24 0 - +R IQ 1982 o - May 1 0 1 - +R IQ 1982 1984 - O 1 0 0 - +R IQ 1983 o - Mar 31 0 1 - +R IQ 1984 1985 - Ap 1 0 1 - +R IQ 1985 1990 - S lastSu 1s 0 - +R IQ 1986 1990 - Mar lastSu 1s 1 - +R IQ 1991 2007 - Ap 1 3s 1 - +R IQ 1991 2007 - O 1 3s 0 - +R Z 1940 o - May 31 24u 1 D +R Z 1940 o - S 30 24u 0 S +R Z 1940 o - N 16 24u 1 D +R Z 1942 1946 - O 31 24u 0 S +R Z 1943 1944 - Mar 31 24u 1 D +R Z 1945 1946 - Ap 15 24u 1 D +R Z 1948 o - May 22 24u 2 DD +R Z 1948 o - Au 31 24u 1 D +R Z 1948 1949 - O 31 24u 0 S +R Z 1949 o - Ap 30 24u 1 D +R Z 1950 o - Ap 15 24u 1 D +R Z 1950 o - S 14 24u 0 S +R Z 1951 o - Mar 31 24u 1 D +R Z 1951 o - N 10 24u 0 S +R Z 1952 o - Ap 19 24u 1 D +R Z 1952 o - O 18 24u 0 S +R Z 1953 o - Ap 11 24u 1 D +R Z 1953 o - S 12 24u 0 S +R Z 1954 o - Jun 12 24u 1 D +R Z 1954 o - S 11 24u 0 S +R Z 1955 o - Jun 11 24u 1 D +R Z 1955 o - S 10 24u 0 S +R Z 1956 o - Jun 2 24u 1 D +R Z 1956 o - S 29 24u 0 S +R Z 1957 o - Ap 27 24u 1 D +R Z 1957 o - S 21 24u 0 S +R Z 1974 o - Jul 6 24 1 D +R Z 1974 o - O 12 24 0 S +R Z 1975 o - Ap 19 24 1 D +R Z 1975 o - Au 30 24 0 S +R Z 1980 o - Au 2 24s 1 D +R Z 1980 o - S 13 24s 0 S +R Z 1984 o - May 5 24s 1 D +R Z 1984 o - Au 25 24s 0 S +R Z 1985 o - Ap 13 24 1 D +R Z 1985 o - Au 31 24 0 S +R Z 1986 o - May 17 24 1 D +R Z 1986 o - S 6 24 0 S +R Z 1987 o - Ap 14 24 1 D +R Z 1987 o - S 12 24 0 S +R Z 1988 o - Ap 9 24 1 D +R Z 1988 o - S 3 24 0 S +R Z 1989 o - Ap 29 24 1 D +R Z 1989 o - S 2 24 0 S +R Z 1990 o - Mar 24 24 1 D +R Z 1990 o - Au 25 24 0 S +R Z 1991 o - Mar 23 24 1 D +R Z 1991 o - Au 31 24 0 S +R Z 1992 o - Mar 28 24 1 D +R Z 1992 o - S 5 24 0 S +R Z 1993 o - Ap 2 0 1 D +R Z 1993 o - S 5 0 0 S +R Z 1994 o - Ap 1 0 1 D +R Z 1994 o - Au 28 0 0 S +R Z 1995 o - Mar 31 0 1 D +R Z 1995 o - S 3 0 0 S +R Z 1996 o - Mar 14 24 1 D +R Z 1996 o - S 15 24 0 S +R Z 1997 o - Mar 20 24 1 D +R Z 1997 o - S 13 24 0 S +R Z 1998 o - Mar 20 0 1 D +R Z 1998 o - S 6 0 0 S +R Z 1999 o - Ap 2 2 1 D +R Z 1999 o - S 3 2 0 S +R Z 2000 o - Ap 14 2 1 D +R Z 2000 o - O 6 1 0 S +R Z 2001 o - Ap 9 1 1 D +R Z 2001 o - S 24 1 0 S +R Z 2002 o - Mar 29 1 1 D +R Z 2002 o - O 7 1 0 S +R Z 2003 o - Mar 28 1 1 D +R Z 2003 o - O 3 1 0 S +R Z 2004 o - Ap 7 1 1 D +R Z 2004 o - S 22 1 0 S +R Z 2005 2012 - Ap F<=1 2 1 D +R Z 2005 o - O 9 2 0 S +R Z 2006 o - O 1 2 0 S +R Z 2007 o - S 16 2 0 S +R Z 2008 o - O 5 2 0 S +R Z 2009 o - S 27 2 0 S +R Z 2010 o - S 12 2 0 S +R Z 2011 o - O 2 2 0 S +R Z 2012 o - S 23 2 0 S +R Z 2013 ma - Mar F>=23 2 1 D +R Z 2013 ma - O lastSu 2 0 S +R JP 1948 o - May Sa>=1 24 1 D +R JP 1948 1951 - S Sa>=8 25 0 S +R JP 1949 o - Ap Sa>=1 24 1 D +R JP 1950 1951 - May Sa>=1 24 1 D +R J 1973 o - Jun 6 0 1 S +R J 1973 1975 - O 1 0 0 - +R J 1974 1977 - May 1 0 1 S +R J 1976 o - N 1 0 0 - +R J 1977 o - O 1 0 0 - +R J 1978 o - Ap 30 0 1 S +R J 1978 o - S 30 0 0 - +R J 1985 o - Ap 1 0 1 S +R J 1985 o - O 1 0 0 - +R J 1986 1988 - Ap F>=1 0 1 S +R J 1986 1990 - O F>=1 0 0 - +R J 1989 o - May 8 0 1 S +R J 1990 o - Ap 27 0 1 S +R J 1991 o - Ap 17 0 1 S +R J 1991 o - S 27 0 0 - +R J 1992 o - Ap 10 0 1 S +R J 1992 1993 - O F>=1 0 0 - +R J 1993 1998 - Ap F>=1 0 1 S +R J 1994 o - S F>=15 0 0 - +R J 1995 1998 - S F>=15 0s 0 - +R J 1999 o - Jul 1 0s 1 S +R J 1999 2002 - S lastF 0s 0 - +R J 2000 2001 - Mar lastTh 0s 1 S +R J 2002 2012 - Mar lastTh 24 1 S +R J 2003 o - O 24 0s 0 - +R J 2004 o - O 15 0s 0 - +R J 2005 o - S lastF 0s 0 - +R J 2006 2011 - O lastF 0s 0 - +R J 2013 o - D 20 0 0 - +R J 2014 2021 - Mar lastTh 24 1 S +R J 2014 2022 - O lastF 0s 0 - +R J 2022 o - F lastTh 24 1 S +R KG 1992 1996 - Ap Su>=7 0s 1 - +R KG 1992 1996 - S lastSu 0 0 - +R KG 1997 2005 - Mar lastSu 2:30 1 - +R KG 1997 2004 - O lastSu 2:30 0 - +R KR 1948 o - Jun 1 0 1 D +R KR 1948 o - S 12 24 0 S +R KR 1949 o - Ap 3 0 1 D +R KR 1949 1951 - S Sa>=7 24 0 S +R KR 1950 o - Ap 1 0 1 D +R KR 1951 o - May 6 0 1 D +R KR 1955 o - May 5 0 1 D +R KR 1955 o - S 8 24 0 S +R KR 1956 o - May 20 0 1 D +R KR 1956 o - S 29 24 0 S +R KR 1957 1960 - May Su>=1 0 1 D +R KR 1957 1960 - S Sa>=17 24 0 S +R KR 1987 1988 - May Su>=8 2 1 D +R KR 1987 1988 - O Su>=8 3 0 S +R l 1920 o - Mar 28 0 1 S +R l 1920 o - O 25 0 0 - +R l 1921 o - Ap 3 0 1 S +R l 1921 o - O 3 0 0 - +R l 1922 o - Mar 26 0 1 S +R l 1922 o - O 8 0 0 - +R l 1923 o - Ap 22 0 1 S +R l 1923 o - S 16 0 0 - +R l 1957 1961 - May 1 0 1 S +R l 1957 1961 - O 1 0 0 - +R l 1972 o - Jun 22 0 1 S +R l 1972 1977 - O 1 0 0 - +R l 1973 1977 - May 1 0 1 S +R l 1978 o - Ap 30 0 1 S +R l 1978 o - S 30 0 0 - +R l 1984 1987 - May 1 0 1 S +R l 1984 1991 - O 16 0 0 - +R l 1988 o - Jun 1 0 1 S +R l 1989 o - May 10 0 1 S +R l 1990 1992 - May 1 0 1 S +R l 1992 o - O 4 0 0 - +R l 1993 ma - Mar lastSu 0 1 S +R l 1993 1998 - S lastSu 0 0 - +R l 1999 ma - O lastSu 0 0 - +R NB 1935 1941 - S 14 0 0:20 - +R NB 1935 1941 - D 14 0 0 - +R X 1983 1984 - Ap 1 0 1 - +R X 1983 o - O 1 0 0 - +R X 1985 1998 - Mar lastSu 0 1 - +R X 1984 1998 - S lastSu 0 0 - +R X 2001 o - Ap lastSa 2 1 - +R X 2001 2006 - S lastSa 2 0 - +R X 2002 2006 - Mar lastSa 2 1 - +R X 2015 2016 - Mar lastSa 2 1 - +R X 2015 2016 - S lastSa 0 0 - +R PK 2002 o - Ap Su>=2 0 1 S +R PK 2002 o - O Su>=2 0 0 - +R PK 2008 o - Jun 1 0 1 S +R PK 2008 2009 - N 1 0 0 - +R PK 2009 o - Ap 15 0 1 S +R P 1999 2005 - Ap F>=15 0 1 S +R P 1999 2003 - O F>=15 0 0 - +R P 2004 o - O 1 1 0 - +R P 2005 o - O 4 2 0 - +R P 2006 2007 - Ap 1 0 1 S +R P 2006 o - S 22 0 0 - +R P 2007 o - S 13 2 0 - +R P 2008 2009 - Mar lastF 0 1 S +R P 2008 o - S 1 0 0 - +R P 2009 o - S 4 1 0 - +R P 2010 o - Mar 26 0 1 S +R P 2010 o - Au 11 0 0 - +R P 2011 o - Ap 1 0:1 1 S +R P 2011 o - Au 1 0 0 - +R P 2011 o - Au 30 0 1 S +R P 2011 o - S 30 0 0 - +R P 2012 2014 - Mar lastTh 24 1 S +R P 2012 o - S 21 1 0 - +R P 2013 o - S 27 0 0 - +R P 2014 o - O 24 0 0 - +R P 2015 o - Mar 28 0 1 S +R P 2015 o - O 23 1 0 - +R P 2016 2018 - Mar Sa<=30 1 1 S +R P 2016 2018 - O Sa<=30 1 0 - +R P 2019 o - Mar 29 0 1 S +R P 2019 o - O Sa<=30 0 0 - +R P 2020 2021 - Mar Sa<=30 0 1 S +R P 2020 o - O 24 1 0 - +R P 2021 o - O 29 1 0 - +R P 2022 o - Mar 27 0 1 S +R P 2022 2035 - O Sa<=30 2 0 - +R P 2023 o - Ap 29 2 1 S +R P 2024 o - Ap 20 2 1 S +R P 2025 o - Ap 12 2 1 S +R P 2026 2054 - Mar Sa<=30 2 1 S +R P 2036 o - O 18 2 0 - +R P 2037 o - O 10 2 0 - +R P 2038 o - S 25 2 0 - +R P 2039 o - S 17 2 0 - +R P 2040 o - S 1 2 0 - +R P 2040 o - O 20 2 1 S +R P 2040 2067 - O Sa<=30 2 0 - +R P 2041 o - Au 24 2 0 - +R P 2041 o - O 5 2 1 S +R P 2042 o - Au 16 2 0 - +R P 2042 o - S 27 2 1 S +R P 2043 o - Au 1 2 0 - +R P 2043 o - S 19 2 1 S +R P 2044 o - Jul 23 2 0 - +R P 2044 o - S 3 2 1 S +R P 2045 o - Jul 15 2 0 - +R P 2045 o - Au 26 2 1 S +R P 2046 o - Jun 30 2 0 - +R P 2046 o - Au 18 2 1 S +R P 2047 o - Jun 22 2 0 - +R P 2047 o - Au 3 2 1 S +R P 2048 o - Jun 6 2 0 - +R P 2048 o - Jul 25 2 1 S +R P 2049 o - May 29 2 0 - +R P 2049 o - Jul 10 2 1 S +R P 2050 o - May 21 2 0 - +R P 2050 o - Jul 2 2 1 S +R P 2051 o - May 6 2 0 - +R P 2051 o - Jun 24 2 1 S +R P 2052 o - Ap 27 2 0 - +R P 2052 o - Jun 8 2 1 S +R P 2053 o - Ap 12 2 0 - +R P 2053 o - May 31 2 1 S +R P 2054 o - Ap 4 2 0 - +R P 2054 o - May 23 2 1 S +R P 2055 o - May 8 2 1 S +R P 2056 o - Ap 29 2 1 S +R P 2057 o - Ap 14 2 1 S +R P 2058 o - Ap 6 2 1 S +R P 2059 ma - Mar Sa<=30 2 1 S +R P 2068 o - O 20 2 0 - +R P 2069 o - O 12 2 0 - +R P 2070 o - O 4 2 0 - +R P 2071 o - S 19 2 0 - +R P 2072 o - S 10 2 0 - +R P 2072 o - O 22 2 1 S +R P 2072 ma - O Sa<=30 2 0 - +R P 2073 o - S 2 2 0 - +R P 2073 o - O 14 2 1 S +R P 2074 o - Au 18 2 0 - +R P 2074 o - O 6 2 1 S +R P 2075 o - Au 10 2 0 - +R P 2075 o - S 21 2 1 S +R P 2076 o - Jul 25 2 0 - +R P 2076 o - S 12 2 1 S +R P 2077 o - Jul 17 2 0 - +R P 2077 o - S 4 2 1 S +R P 2078 o - Jul 9 2 0 - +R P 2078 o - Au 20 2 1 S +R P 2079 o - Jun 24 2 0 - +R P 2079 o - Au 12 2 1 S +R P 2080 o - Jun 15 2 0 - +R P 2080 o - Jul 27 2 1 S +R P 2081 o - Jun 7 2 0 - +R P 2081 o - Jul 19 2 1 S +R P 2082 o - May 23 2 0 - +R P 2082 o - Jul 11 2 1 S +R P 2083 o - May 15 2 0 - +R P 2083 o - Jun 26 2 1 S +R P 2084 o - Ap 29 2 0 - +R P 2084 o - Jun 17 2 1 S +R P 2085 o - Ap 21 2 0 - +R P 2085 o - Jun 9 2 1 S +R P 2086 o - Ap 13 2 0 - +R P 2086 o - May 25 2 1 S +R PH 1936 o - O 31 24 1 D +R PH 1937 o - Ja 15 24 0 S +R PH 1941 o - D 15 24 1 D +R PH 1945 o - N 30 24 0 S +R PH 1954 o - Ap 11 24 1 D +R PH 1954 o - Jun 4 24 0 S +R PH 1977 o - Mar 27 24 1 D +R PH 1977 o - S 21 24 0 S +R PH 1990 o - May 21 0 1 D +R PH 1990 o - Jul 28 24 0 S +R S 1920 1923 - Ap Su>=15 2 1 S +R S 1920 1923 - O Su>=1 2 0 - +R S 1962 o - Ap 29 2 1 S +R S 1962 o - O 1 2 0 - +R S 1963 1965 - May 1 2 1 S +R S 1963 o - S 30 2 0 - +R S 1964 o - O 1 2 0 - +R S 1965 o - S 30 2 0 - +R S 1966 o - Ap 24 2 1 S +R S 1966 1976 - O 1 2 0 - +R S 1967 1978 - May 1 2 1 S +R S 1977 1978 - S 1 2 0 - +R S 1983 1984 - Ap 9 2 1 S +R S 1983 1984 - O 1 2 0 - +R S 1986 o - F 16 2 1 S +R S 1986 o - O 9 2 0 - +R S 1987 o - Mar 1 2 1 S +R S 1987 1988 - O 31 2 0 - +R S 1988 o - Mar 15 2 1 S +R S 1989 o - Mar 31 2 1 S +R S 1989 o - O 1 2 0 - +R S 1990 o - Ap 1 2 1 S +R S 1990 o - S 30 2 0 - +R S 1991 o - Ap 1 0 1 S +R S 1991 1992 - O 1 0 0 - +R S 1992 o - Ap 8 0 1 S +R S 1993 o - Mar 26 0 1 S +R S 1993 o - S 25 0 0 - +R S 1994 1996 - Ap 1 0 1 S +R S 1994 2005 - O 1 0 0 - +R S 1997 1998 - Mar lastM 0 1 S +R S 1999 2006 - Ap 1 0 1 S +R S 2006 o - S 22 0 0 - +R S 2007 o - Mar lastF 0 1 S +R S 2007 o - N F>=1 0 0 - +R S 2008 o - Ap F>=1 0 1 S +R S 2008 o - N 1 0 0 - +R S 2009 o - Mar lastF 0 1 S +R S 2010 2011 - Ap F>=1 0 1 S +R S 2012 2022 - Mar lastF 0 1 S +R S 2009 2022 - O lastF 0 0 - +R AU 1917 o - Ja 1 2s 1 D +R AU 1917 o - Mar lastSu 2s 0 S +R AU 1942 o - Ja 1 2s 1 D +R AU 1942 o - Mar lastSu 2s 0 S +R AU 1942 o - S 27 2s 1 D +R AU 1943 1944 - Mar lastSu 2s 0 S +R AU 1943 o - O 3 2s 1 D +R AW 1974 o - O lastSu 2s 1 D +R AW 1975 o - Mar Su>=1 2s 0 S +R AW 1983 o - O lastSu 2s 1 D +R AW 1984 o - Mar Su>=1 2s 0 S +R AW 1991 o - N 17 2s 1 D +R AW 1992 o - Mar Su>=1 2s 0 S +R AW 2006 o - D 3 2s 1 D +R AW 2007 2009 - Mar lastSu 2s 0 S +R AW 2007 2008 - O lastSu 2s 1 D +R AQ 1971 o - O lastSu 2s 1 D +R AQ 1972 o - F lastSu 2s 0 S +R AQ 1989 1991 - O lastSu 2s 1 D +R AQ 1990 1992 - Mar Su>=1 2s 0 S +R Ho 1992 1993 - O lastSu 2s 1 D +R Ho 1993 1994 - Mar Su>=1 2s 0 S +R AS 1971 1985 - O lastSu 2s 1 D +R AS 1986 o - O 19 2s 1 D +R AS 1987 2007 - O lastSu 2s 1 D +R AS 1972 o - F 27 2s 0 S +R AS 1973 1985 - Mar Su>=1 2s 0 S +R AS 1986 1990 - Mar Su>=15 2s 0 S +R AS 1991 o - Mar 3 2s 0 S +R AS 1992 o - Mar 22 2s 0 S +R AS 1993 o - Mar 7 2s 0 S +R AS 1994 o - Mar 20 2s 0 S +R AS 1995 2005 - Mar lastSu 2s 0 S +R AS 2006 o - Ap 2 2s 0 S +R AS 2007 o - Mar lastSu 2s 0 S +R AS 2008 ma - Ap Su>=1 2s 0 S +R AS 2008 ma - O Su>=1 2s 1 D +R AT 1916 o - O Su>=1 2s 1 D +R AT 1917 o - Mar lastSu 2s 0 S +R AT 1917 1918 - O Su>=22 2s 1 D +R AT 1918 1919 - Mar Su>=1 2s 0 S +R AT 1967 o - O Su>=1 2s 1 D +R AT 1968 o - Mar Su>=29 2s 0 S +R AT 1968 1985 - O lastSu 2s 1 D +R AT 1969 1971 - Mar Su>=8 2s 0 S +R AT 1972 o - F lastSu 2s 0 S +R AT 1973 1981 - Mar Su>=1 2s 0 S +R AT 1982 1983 - Mar lastSu 2s 0 S +R AT 1984 1986 - Mar Su>=1 2s 0 S +R AT 1986 o - O Su>=15 2s 1 D +R AT 1987 1990 - Mar Su>=15 2s 0 S +R AT 1987 o - O Su>=22 2s 1 D +R AT 1988 1990 - O lastSu 2s 1 D +R AT 1991 1999 - O Su>=1 2s 1 D +R AT 1991 2005 - Mar lastSu 2s 0 S +R AT 2000 o - Au lastSu 2s 1 D +R AT 2001 ma - O Su>=1 2s 1 D +R AT 2006 o - Ap Su>=1 2s 0 S +R AT 2007 o - Mar lastSu 2s 0 S +R AT 2008 ma - Ap Su>=1 2s 0 S +R AV 1971 1985 - O lastSu 2s 1 D +R AV 1972 o - F lastSu 2s 0 S +R AV 1973 1985 - Mar Su>=1 2s 0 S +R AV 1986 1990 - Mar Su>=15 2s 0 S +R AV 1986 1987 - O Su>=15 2s 1 D +R AV 1988 1999 - O lastSu 2s 1 D +R AV 1991 1994 - Mar Su>=1 2s 0 S +R AV 1995 2005 - Mar lastSu 2s 0 S +R AV 2000 o - Au lastSu 2s 1 D +R AV 2001 2007 - O lastSu 2s 1 D +R AV 2006 o - Ap Su>=1 2s 0 S +R AV 2007 o - Mar lastSu 2s 0 S +R AV 2008 ma - Ap Su>=1 2s 0 S +R AV 2008 ma - O Su>=1 2s 1 D +R AN 1971 1985 - O lastSu 2s 1 D +R AN 1972 o - F 27 2s 0 S +R AN 1973 1981 - Mar Su>=1 2s 0 S +R AN 1982 o - Ap Su>=1 2s 0 S +R AN 1983 1985 - Mar Su>=1 2s 0 S +R AN 1986 1989 - Mar Su>=15 2s 0 S +R AN 1986 o - O 19 2s 1 D +R AN 1987 1999 - O lastSu 2s 1 D +R AN 1990 1995 - Mar Su>=1 2s 0 S +R AN 1996 2005 - Mar lastSu 2s 0 S +R AN 2000 o - Au lastSu 2s 1 D +R AN 2001 2007 - O lastSu 2s 1 D +R AN 2006 o - Ap Su>=1 2s 0 S +R AN 2007 o - Mar lastSu 2s 0 S +R AN 2008 ma - Ap Su>=1 2s 0 S +R AN 2008 ma - O Su>=1 2s 1 D +R LH 1981 1984 - O lastSu 2 1 - +R LH 1982 1985 - Mar Su>=1 2 0 - +R LH 1985 o - O lastSu 2 0:30 - +R LH 1986 1989 - Mar Su>=15 2 0 - +R LH 1986 o - O 19 2 0:30 - +R LH 1987 1999 - O lastSu 2 0:30 - +R LH 1990 1995 - Mar Su>=1 2 0 - +R LH 1996 2005 - Mar lastSu 2 0 - +R LH 2000 o - Au lastSu 2 0:30 - +R LH 2001 2007 - O lastSu 2 0:30 - +R LH 2006 o - Ap Su>=1 2 0 - +R LH 2007 o - Mar lastSu 2 0 - +R LH 2008 ma - Ap Su>=1 2 0 - +R LH 2008 ma - O Su>=1 2 0:30 - +R FJ 1998 1999 - N Su>=1 2 1 - +R FJ 1999 2000 - F lastSu 3 0 - +R FJ 2009 o - N 29 2 1 - +R FJ 2010 o - Mar lastSu 3 0 - +R FJ 2010 2013 - O Su>=21 2 1 - +R FJ 2011 o - Mar Su>=1 3 0 - +R FJ 2012 2013 - Ja Su>=18 3 0 - +R FJ 2014 o - Ja Su>=18 2 0 - +R FJ 2014 2018 - N Su>=1 2 1 - +R FJ 2015 2021 - Ja Su>=12 3 0 - +R FJ 2019 o - N Su>=8 2 1 - +R FJ 2020 o - D 20 2 1 - +R Gu 1959 o - Jun 27 2 1 D +R Gu 1961 o - Ja 29 2 0 S +R Gu 1967 o - S 1 2 1 D +R Gu 1969 o - Ja 26 0:1 0 S +R Gu 1969 o - Jun 22 2 1 D +R Gu 1969 o - Au 31 2 0 S +R Gu 1970 1971 - Ap lastSu 2 1 D +R Gu 1970 1971 - S Su>=1 2 0 S +R Gu 1973 o - D 16 2 1 D +R Gu 1974 o - F 24 2 0 S +R Gu 1976 o - May 26 2 1 D +R Gu 1976 o - Au 22 2:1 0 S +R Gu 1977 o - Ap 24 2 1 D +R Gu 1977 o - Au 28 2 0 S +R NC 1977 1978 - D Su>=1 0 1 - +R NC 1978 1979 - F 27 0 0 - +R NC 1996 o - D 1 2s 1 - +R NC 1997 o - Mar 2 2s 0 - +R NZ 1927 o - N 6 2 1 S +R NZ 1928 o - Mar 4 2 0 M +R NZ 1928 1933 - O Su>=8 2 0:30 S +R NZ 1929 1933 - Mar Su>=15 2 0 M +R NZ 1934 1940 - Ap lastSu 2 0 M +R NZ 1934 1940 - S lastSu 2 0:30 S +R NZ 1946 o - Ja 1 0 0 S +R NZ 1974 o - N Su>=1 2s 1 D +R k 1974 o - N Su>=1 2:45s 1 - +R NZ 1975 o - F lastSu 2s 0 S +R k 1975 o - F lastSu 2:45s 0 - +R NZ 1975 1988 - O lastSu 2s 1 D +R k 1975 1988 - O lastSu 2:45s 1 - +R NZ 1976 1989 - Mar Su>=1 2s 0 S +R k 1976 1989 - Mar Su>=1 2:45s 0 - +R NZ 1989 o - O Su>=8 2s 1 D +R k 1989 o - O Su>=8 2:45s 1 - +R NZ 1990 2006 - O Su>=1 2s 1 D +R k 1990 2006 - O Su>=1 2:45s 1 - +R NZ 1990 2007 - Mar Su>=15 2s 0 S +R k 1990 2007 - Mar Su>=15 2:45s 0 - +R NZ 2007 ma - S lastSu 2s 1 D +R k 2007 ma - S lastSu 2:45s 1 - +R NZ 2008 ma - Ap Su>=1 2s 0 S +R k 2008 ma - Ap Su>=1 2:45s 0 - +R CK 1978 o - N 12 0 0:30 - +R CK 1979 1991 - Mar Su>=1 0 0 - +R CK 1979 1990 - O lastSu 0 0:30 - +R WS 2010 o - S lastSu 0 1 - +R WS 2011 o - Ap Sa>=1 4 0 - +R WS 2011 o - S lastSa 3 1 - +R WS 2012 2021 - Ap Su>=1 4 0 - +R WS 2012 2020 - S lastSu 3 1 - +R TO 1999 o - O 7 2s 1 - +R TO 2000 o - Mar 19 2s 0 - +R TO 2000 2001 - N Su>=1 2 1 - +R TO 2001 2002 - Ja lastSu 2 0 - +R TO 2016 o - N Su>=1 2 1 - +R TO 2017 o - Ja Su>=15 3 0 - +R VU 1973 o - D 22 12u 1 - +R VU 1974 o - Mar 30 12u 0 - +R VU 1983 1991 - S Sa>=22 24 1 - +R VU 1984 1991 - Mar Sa>=22 24 0 - +R VU 1992 1993 - Ja Sa>=22 24 0 - +R VU 1992 o - O Sa>=22 24 1 - +R G 1916 o - May 21 2s 1 BST +R G 1916 o - O 1 2s 0 GMT +R G 1917 o - Ap 8 2s 1 BST +R G 1917 o - S 17 2s 0 GMT +R G 1918 o - Mar 24 2s 1 BST +R G 1918 o - S 30 2s 0 GMT +R G 1919 o - Mar 30 2s 1 BST +R G 1919 o - S 29 2s 0 GMT +R G 1920 o - Mar 28 2s 1 BST +R G 1920 o - O 25 2s 0 GMT +R G 1921 o - Ap 3 2s 1 BST +R G 1921 o - O 3 2s 0 GMT +R G 1922 o - Mar 26 2s 1 BST +R G 1922 o - O 8 2s 0 GMT +R G 1923 o - Ap Su>=16 2s 1 BST +R G 1923 1924 - S Su>=16 2s 0 GMT +R G 1924 o - Ap Su>=9 2s 1 BST +R G 1925 1926 - Ap Su>=16 2s 1 BST +R G 1925 1938 - O Su>=2 2s 0 GMT +R G 1927 o - Ap Su>=9 2s 1 BST +R G 1928 1929 - Ap Su>=16 2s 1 BST +R G 1930 o - Ap Su>=9 2s 1 BST +R G 1931 1932 - Ap Su>=16 2s 1 BST +R G 1933 o - Ap Su>=9 2s 1 BST +R G 1934 o - Ap Su>=16 2s 1 BST +R G 1935 o - Ap Su>=9 2s 1 BST +R G 1936 1937 - Ap Su>=16 2s 1 BST +R G 1938 o - Ap Su>=9 2s 1 BST +R G 1939 o - Ap Su>=16 2s 1 BST +R G 1939 o - N Su>=16 2s 0 GMT +R G 1940 o - F Su>=23 2s 1 BST +R G 1941 o - May Su>=2 1s 2 BDST +R G 1941 1943 - Au Su>=9 1s 1 BST +R G 1942 1944 - Ap Su>=2 1s 2 BDST +R G 1944 o - S Su>=16 1s 1 BST +R G 1945 o - Ap M>=2 1s 2 BDST +R G 1945 o - Jul Su>=9 1s 1 BST +R G 1945 1946 - O Su>=2 2s 0 GMT +R G 1946 o - Ap Su>=9 2s 1 BST +R G 1947 o - Mar 16 2s 1 BST +R G 1947 o - Ap 13 1s 2 BDST +R G 1947 o - Au 10 1s 1 BST +R G 1947 o - N 2 2s 0 GMT +R G 1948 o - Mar 14 2s 1 BST +R G 1948 o - O 31 2s 0 GMT +R G 1949 o - Ap 3 2s 1 BST +R G 1949 o - O 30 2s 0 GMT +R G 1950 1952 - Ap Su>=14 2s 1 BST +R G 1950 1952 - O Su>=21 2s 0 GMT +R G 1953 o - Ap Su>=16 2s 1 BST +R G 1953 1960 - O Su>=2 2s 0 GMT +R G 1954 o - Ap Su>=9 2s 1 BST +R G 1955 1956 - Ap Su>=16 2s 1 BST +R G 1957 o - Ap Su>=9 2s 1 BST +R G 1958 1959 - Ap Su>=16 2s 1 BST +R G 1960 o - Ap Su>=9 2s 1 BST +R G 1961 1963 - Mar lastSu 2s 1 BST +R G 1961 1968 - O Su>=23 2s 0 GMT +R G 1964 1967 - Mar Su>=19 2s 1 BST +R G 1968 o - F 18 2s 1 BST +R G 1972 1980 - Mar Su>=16 2s 1 BST +R G 1972 1980 - O Su>=23 2s 0 GMT +R G 1981 1995 - Mar lastSu 1u 1 BST +R G 1981 1989 - O Su>=23 1u 0 GMT +R G 1990 1995 - O Su>=22 1u 0 GMT +R IE 1971 o - O 31 2u -1 - +R IE 1972 1980 - Mar Su>=16 2u 0 - +R IE 1972 1980 - O Su>=23 2u -1 - +R IE 1981 ma - Mar lastSu 1u 0 - +R IE 1981 1989 - O Su>=23 1u -1 - +R IE 1990 1995 - O Su>=22 1u -1 - +R IE 1996 ma - O lastSu 1u -1 - +R E 1977 1980 - Ap Su>=1 1u 1 S +R E 1977 o - S lastSu 1u 0 - +R E 1978 o - O 1 1u 0 - +R E 1979 1995 - S lastSu 1u 0 - +R E 1981 ma - Mar lastSu 1u 1 S +R E 1996 ma - O lastSu 1u 0 - +R W- 1977 1980 - Ap Su>=1 1s 1 S +R W- 1977 o - S lastSu 1s 0 - +R W- 1978 o - O 1 1s 0 - +R W- 1979 1995 - S lastSu 1s 0 - +R W- 1981 ma - Mar lastSu 1s 1 S +R W- 1996 ma - O lastSu 1s 0 - +R c 1916 o - Ap 30 23 1 S +R c 1916 o - O 1 1 0 - +R c 1917 1918 - Ap M>=15 2s 1 S +R c 1917 1918 - S M>=15 2s 0 - +R c 1940 o - Ap 1 2s 1 S +R c 1942 o - N 2 2s 0 - +R c 1943 o - Mar 29 2s 1 S +R c 1943 o - O 4 2s 0 - +R c 1944 1945 - Ap M>=1 2s 1 S +R c 1944 o - O 2 2s 0 - +R c 1945 o - S 16 2s 0 - +R c 1977 1980 - Ap Su>=1 2s 1 S +R c 1977 o - S lastSu 2s 0 - +R c 1978 o - O 1 2s 0 - +R c 1979 1995 - S lastSu 2s 0 - +R c 1981 ma - Mar lastSu 2s 1 S +R c 1996 ma - O lastSu 2s 0 - +R e 1977 1980 - Ap Su>=1 0 1 S +R e 1977 o - S lastSu 0 0 - +R e 1978 o - O 1 0 0 - +R e 1979 1995 - S lastSu 0 0 - +R e 1981 ma - Mar lastSu 0 1 S +R e 1996 ma - O lastSu 0 0 - +R R 1917 o - Jul 1 23 1 MST +R R 1917 o - D 28 0 0 MMT +R R 1918 o - May 31 22 2 MDST +R R 1918 o - S 16 1 1 MST +R R 1919 o - May 31 23 2 MDST +R R 1919 o - Jul 1 0u 1 MSD +R R 1919 o - Au 16 0 0 MSK +R R 1921 o - F 14 23 1 MSD +R R 1921 o - Mar 20 23 2 +05 +R R 1921 o - S 1 0 1 MSD +R R 1921 o - O 1 0 0 - +R R 1981 1984 - Ap 1 0 1 S +R R 1981 1983 - O 1 0 0 - +R R 1984 1995 - S lastSu 2s 0 - +R R 1985 2010 - Mar lastSu 2s 1 S +R R 1996 2010 - O lastSu 2s 0 - +R q 1940 o - Jun 16 0 1 S +R q 1942 o - N 2 3 0 - +R q 1943 o - Mar 29 2 1 S +R q 1943 o - Ap 10 3 0 - +R q 1974 o - May 4 0 1 S +R q 1974 o - O 2 0 0 - +R q 1975 o - May 1 0 1 S +R q 1975 o - O 2 0 0 - +R q 1976 o - May 2 0 1 S +R q 1976 o - O 3 0 0 - +R q 1977 o - May 8 0 1 S +R q 1977 o - O 2 0 0 - +R q 1978 o - May 6 0 1 S +R q 1978 o - O 1 0 0 - +R q 1979 o - May 5 0 1 S +R q 1979 o - S 30 0 0 - +R q 1980 o - May 3 0 1 S +R q 1980 o - O 4 0 0 - +R q 1981 o - Ap 26 0 1 S +R q 1981 o - S 27 0 0 - +R q 1982 o - May 2 0 1 S +R q 1982 o - O 3 0 0 - +R q 1983 o - Ap 18 0 1 S +R q 1983 o - O 1 0 0 - +R q 1984 o - Ap 1 0 1 S +R a 1920 o - Ap 5 2s 1 S +R a 1920 o - S 13 2s 0 - +R a 1946 o - Ap 14 2s 1 S +R a 1946 o - O 7 2s 0 - +R a 1947 1948 - O Su>=1 2s 0 - +R a 1947 o - Ap 6 2s 1 S +R a 1948 o - Ap 18 2s 1 S +R a 1980 o - Ap 6 0 1 S +R a 1980 o - S 28 0 0 - +R b 1918 o - Mar 9 0s 1 S +R b 1918 1919 - O Sa>=1 23s 0 - +R b 1919 o - Mar 1 23s 1 S +R b 1920 o - F 14 23s 1 S +R b 1920 o - O 23 23s 0 - +R b 1921 o - Mar 14 23s 1 S +R b 1921 o - O 25 23s 0 - +R b 1922 o - Mar 25 23s 1 S +R b 1922 1927 - O Sa>=1 23s 0 - +R b 1923 o - Ap 21 23s 1 S +R b 1924 o - Mar 29 23s 1 S +R b 1925 o - Ap 4 23s 1 S +R b 1926 o - Ap 17 23s 1 S +R b 1927 o - Ap 9 23s 1 S +R b 1928 o - Ap 14 23s 1 S +R b 1928 1938 - O Su>=2 2s 0 - +R b 1929 o - Ap 21 2s 1 S +R b 1930 o - Ap 13 2s 1 S +R b 1931 o - Ap 19 2s 1 S +R b 1932 o - Ap 3 2s 1 S +R b 1933 o - Mar 26 2s 1 S +R b 1934 o - Ap 8 2s 1 S +R b 1935 o - Mar 31 2s 1 S +R b 1936 o - Ap 19 2s 1 S +R b 1937 o - Ap 4 2s 1 S +R b 1938 o - Mar 27 2s 1 S +R b 1939 o - Ap 16 2s 1 S +R b 1939 o - N 19 2s 0 - +R b 1940 o - F 25 2s 1 S +R b 1944 o - S 17 2s 0 - +R b 1945 o - Ap 2 2s 1 S +R b 1945 o - S 16 2s 0 - +R b 1946 o - May 19 2s 1 S +R b 1946 o - O 7 2s 0 - +R BG 1979 o - Mar 31 23 1 S +R BG 1979 o - O 1 1 0 - +R BG 1980 1982 - Ap Sa>=1 23 1 S +R BG 1980 o - S 29 1 0 - +R BG 1981 o - S 27 2 0 - +R CZ 1945 o - Ap M>=1 2s 1 S +R CZ 1945 o - O 1 2s 0 - +R CZ 1946 o - May 6 2s 1 S +R CZ 1946 1949 - O Su>=1 2s 0 - +R CZ 1947 1948 - Ap Su>=15 2s 1 S +R CZ 1949 o - Ap 9 2s 1 S +R Th 1991 1992 - Mar lastSu 2 1 D +R Th 1991 1992 - S lastSu 2 0 S +R Th 1993 2006 - Ap Su>=1 2 1 D +R Th 1993 2006 - O lastSu 2 0 S +R Th 2007 ma - Mar Su>=8 2 1 D +R Th 2007 ma - N Su>=1 2 0 S +R FI 1942 o - Ap 2 24 1 S +R FI 1942 o - O 4 1 0 - +R FI 1981 1982 - Mar lastSu 2 1 S +R FI 1981 1982 - S lastSu 3 0 - +R F 1916 o - Jun 14 23s 1 S +R F 1916 1919 - O Su>=1 23s 0 - +R F 1917 o - Mar 24 23s 1 S +R F 1918 o - Mar 9 23s 1 S +R F 1919 o - Mar 1 23s 1 S +R F 1920 o - F 14 23s 1 S +R F 1920 o - O 23 23s 0 - +R F 1921 o - Mar 14 23s 1 S +R F 1921 o - O 25 23s 0 - +R F 1922 o - Mar 25 23s 1 S +R F 1922 1938 - O Sa>=1 23s 0 - +R F 1923 o - May 26 23s 1 S +R F 1924 o - Mar 29 23s 1 S +R F 1925 o - Ap 4 23s 1 S +R F 1926 o - Ap 17 23s 1 S +R F 1927 o - Ap 9 23s 1 S +R F 1928 o - Ap 14 23s 1 S +R F 1929 o - Ap 20 23s 1 S +R F 1930 o - Ap 12 23s 1 S +R F 1931 o - Ap 18 23s 1 S +R F 1932 o - Ap 2 23s 1 S +R F 1933 o - Mar 25 23s 1 S +R F 1934 o - Ap 7 23s 1 S +R F 1935 o - Mar 30 23s 1 S +R F 1936 o - Ap 18 23s 1 S +R F 1937 o - Ap 3 23s 1 S +R F 1938 o - Mar 26 23s 1 S +R F 1939 o - Ap 15 23s 1 S +R F 1939 o - N 18 23s 0 - +R F 1940 o - F 25 2 1 S +R F 1941 o - May 5 0 2 M +R F 1941 o - O 6 0 1 S +R F 1942 o - Mar 9 0 2 M +R F 1942 o - N 2 3 1 S +R F 1943 o - Mar 29 2 2 M +R F 1943 o - O 4 3 1 S +R F 1944 o - Ap 3 2 2 M +R F 1944 o - O 8 1 1 S +R F 1945 o - Ap 2 2 2 M +R F 1945 o - S 16 3 0 - +R F 1976 o - Mar 28 1 1 S +R F 1976 o - S 26 1 0 - +R DE 1946 o - Ap 14 2s 1 S +R DE 1946 o - O 7 2s 0 - +R DE 1947 1949 - O Su>=1 2s 0 - +R DE 1947 o - Ap 6 3s 1 S +R DE 1947 o - May 11 2s 2 M +R DE 1947 o - Jun 29 3 1 S +R DE 1948 o - Ap 18 2s 1 S +R DE 1949 o - Ap 10 2s 1 S +R So 1945 o - May 24 2 2 M +R So 1945 o - S 24 3 1 S +R So 1945 o - N 18 2s 0 - +R g 1932 o - Jul 7 0 1 S +R g 1932 o - S 1 0 0 - +R g 1941 o - Ap 7 0 1 S +R g 1942 o - N 2 3 0 - +R g 1943 o - Mar 30 0 1 S +R g 1943 o - O 4 0 0 - +R g 1952 o - Jul 1 0 1 S +R g 1952 o - N 2 0 0 - +R g 1975 o - Ap 12 0s 1 S +R g 1975 o - N 26 0s 0 - +R g 1976 o - Ap 11 2s 1 S +R g 1976 o - O 10 2s 0 - +R g 1977 1978 - Ap Su>=1 2s 1 S +R g 1977 o - S 26 2s 0 - +R g 1978 o - S 24 4 0 - +R g 1979 o - Ap 1 9 1 S +R g 1979 o - S 29 2 0 - +R g 1980 o - Ap 1 0 1 S +R g 1980 o - S 28 0 0 - +R h 1918 1919 - Ap 15 2 1 S +R h 1918 1920 - S M>=15 3 0 - +R h 1920 o - Ap 5 2 1 S +R h 1945 o - May 1 23 1 S +R h 1945 o - N 1 1 0 - +R h 1946 o - Mar 31 2s 1 S +R h 1946 o - O 7 2 0 - +R h 1947 1949 - Ap Su>=4 2s 1 S +R h 1947 1949 - O Su>=1 2s 0 - +R h 1954 o - May 23 0 1 S +R h 1954 o - O 3 0 0 - +R h 1955 o - May 22 2 1 S +R h 1955 o - O 2 3 0 - +R h 1956 1957 - Jun Su>=1 2 1 S +R h 1956 1957 - S lastSu 3 0 - +R h 1980 o - Ap 6 0 1 S +R h 1980 o - S 28 1 0 - +R h 1981 1983 - Mar lastSu 0 1 S +R h 1981 1983 - S lastSu 1 0 - +R I 1916 o - Jun 3 24 1 S +R I 1916 1917 - S 30 24 0 - +R I 1917 o - Mar 31 24 1 S +R I 1918 o - Mar 9 24 1 S +R I 1918 o - O 6 24 0 - +R I 1919 o - Mar 1 24 1 S +R I 1919 o - O 4 24 0 - +R I 1920 o - Mar 20 24 1 S +R I 1920 o - S 18 24 0 - +R I 1940 o - Jun 14 24 1 S +R I 1942 o - N 2 2s 0 - +R I 1943 o - Mar 29 2s 1 S +R I 1943 o - O 4 2s 0 - +R I 1944 o - Ap 2 2s 1 S +R I 1944 o - S 17 2s 0 - +R I 1945 o - Ap 2 2 1 S +R I 1945 o - S 15 1 0 - +R I 1946 o - Mar 17 2s 1 S +R I 1946 o - O 6 2s 0 - +R I 1947 o - Mar 16 0s 1 S +R I 1947 o - O 5 0s 0 - +R I 1948 o - F 29 2s 1 S +R I 1948 o - O 3 2s 0 - +R I 1966 1968 - May Su>=22 0s 1 S +R I 1966 o - S 24 24 0 - +R I 1967 1969 - S Su>=22 0s 0 - +R I 1969 o - Jun 1 0s 1 S +R I 1970 o - May 31 0s 1 S +R I 1970 o - S lastSu 0s 0 - +R I 1971 1972 - May Su>=22 0s 1 S +R I 1971 o - S lastSu 0s 0 - +R I 1972 o - O 1 0s 0 - +R I 1973 o - Jun 3 0s 1 S +R I 1973 1974 - S lastSu 0s 0 - +R I 1974 o - May 26 0s 1 S +R I 1975 o - Jun 1 0s 1 S +R I 1975 1977 - S lastSu 0s 0 - +R I 1976 o - May 30 0s 1 S +R I 1977 1979 - May Su>=22 0s 1 S +R I 1978 o - O 1 0s 0 - +R I 1979 o - S 30 0s 0 - +R LV 1989 1996 - Mar lastSu 2s 1 S +R LV 1989 1996 - S lastSu 2s 0 - +R MT 1973 o - Mar 31 0s 1 S +R MT 1973 o - S 29 0s 0 - +R MT 1974 o - Ap 21 0s 1 S +R MT 1974 o - S 16 0s 0 - +R MT 1975 1979 - Ap Su>=15 2 1 S +R MT 1975 1980 - S Su>=15 2 0 - +R MT 1980 o - Mar 31 2 1 S +R MD 1997 ma - Mar lastSu 2 1 S +R MD 1997 ma - O lastSu 3 0 - +R O 1918 1919 - S 16 2s 0 - +R O 1919 o - Ap 15 2s 1 S +R O 1944 o - Ap 3 2s 1 S +R O 1944 o - O 4 2 0 - +R O 1945 o - Ap 29 0 1 S +R O 1945 o - N 1 0 0 - +R O 1946 o - Ap 14 0s 1 S +R O 1946 o - O 7 2s 0 - +R O 1947 o - May 4 2s 1 S +R O 1947 1949 - O Su>=1 2s 0 - +R O 1948 o - Ap 18 2s 1 S +R O 1949 o - Ap 10 2s 1 S +R O 1957 o - Jun 2 1s 1 S +R O 1957 1958 - S lastSu 1s 0 - +R O 1958 o - Mar 30 1s 1 S +R O 1959 o - May 31 1s 1 S +R O 1959 1961 - O Su>=1 1s 0 - +R O 1960 o - Ap 3 1s 1 S +R O 1961 1964 - May lastSu 1s 1 S +R O 1962 1964 - S lastSu 1s 0 - +R p 1916 o - Jun 17 23 1 S +R p 1916 o - N 1 1 0 - +R p 1917 1921 - Mar 1 0 1 S +R p 1917 1921 - O 14 24 0 - +R p 1924 o - Ap 16 23s 1 S +R p 1924 o - O 4 23s 0 - +R p 1926 o - Ap 17 23s 1 S +R p 1926 1929 - O Sa>=1 23s 0 - +R p 1927 o - Ap 9 23s 1 S +R p 1928 o - Ap 14 23s 1 S +R p 1929 o - Ap 20 23s 1 S +R p 1931 o - Ap 18 23s 1 S +R p 1931 1932 - O Sa>=1 23s 0 - +R p 1932 o - Ap 2 23s 1 S +R p 1934 o - Ap 7 23s 1 S +R p 1934 1938 - O Sa>=1 23s 0 - +R p 1935 o - Mar 30 23s 1 S +R p 1936 o - Ap 18 23s 1 S +R p 1937 o - Ap 3 23s 1 S +R p 1938 o - Mar 26 23s 1 S +R p 1939 o - Ap 15 23s 1 S +R p 1939 o - N 18 23s 0 - +R p 1940 o - F 24 23s 1 S +R p 1940 o - O 7 23s 0 - +R p 1941 o - Ap 5 23s 1 S +R p 1941 o - O 5 23s 0 - +R p 1942 1945 - Mar Sa>=8 23s 1 S +R p 1942 o - Ap 25 22s 2 M +R p 1942 o - Au 15 22s 1 S +R p 1942 1945 - O Sa>=24 23s 0 - +R p 1943 o - Ap 17 22s 2 M +R p 1943 1945 - Au Sa>=25 22s 1 S +R p 1944 1945 - Ap Sa>=21 22s 2 M +R p 1946 o - Ap Sa>=1 23s 1 S +R p 1946 o - O Sa>=1 23s 0 - +R p 1947 1966 - Ap Su>=1 2s 1 S +R p 1947 1965 - O Su>=1 2s 0 - +R p 1976 o - S lastSu 1 0 - +R p 1977 o - Mar lastSu 0s 1 S +R p 1977 o - S lastSu 0s 0 - +R p 1978 1980 - Ap Su>=1 1s 1 S +R p 1978 o - O 1 1s 0 - +R p 1979 1980 - S lastSu 1s 0 - +R p 1981 1986 - Mar lastSu 0s 1 S +R p 1981 1985 - S lastSu 0s 0 - +R z 1932 o - May 21 0s 1 S +R z 1932 1939 - O Su>=1 0s 0 - +R z 1933 1939 - Ap Su>=2 0s 1 S +R z 1979 o - May 27 0 1 S +R z 1979 o - S lastSu 0 0 - +R z 1980 o - Ap 5 23 1 S +R z 1980 o - S lastSu 1 0 - +R z 1991 1993 - Mar lastSu 0s 1 S +R z 1991 1993 - S lastSu 0s 0 - +R s 1918 o - Ap 15 23 1 S +R s 1918 1919 - O 6 24s 0 - +R s 1919 o - Ap 6 23 1 S +R s 1924 o - Ap 16 23 1 S +R s 1924 o - O 4 24s 0 - +R s 1926 o - Ap 17 23 1 S +R s 1926 1929 - O Sa>=1 24s 0 - +R s 1927 o - Ap 9 23 1 S +R s 1928 o - Ap 15 0 1 S +R s 1929 o - Ap 20 23 1 S +R s 1937 o - Jun 16 23 1 S +R s 1937 o - O 2 24s 0 - +R s 1938 o - Ap 2 23 1 S +R s 1938 o - Ap 30 23 2 M +R s 1938 o - O 2 24 1 S +R s 1939 o - O 7 24s 0 - +R s 1942 o - May 2 23 1 S +R s 1942 o - S 1 1 0 - +R s 1943 1946 - Ap Sa>=13 23 1 S +R s 1943 1944 - O Su>=1 1 0 - +R s 1945 1946 - S lastSu 1 0 - +R s 1949 o - Ap 30 23 1 S +R s 1949 o - O 2 1 0 - +R s 1974 1975 - Ap Sa>=12 23 1 S +R s 1974 1975 - O Su>=1 1 0 - +R s 1976 o - Mar 27 23 1 S +R s 1976 1977 - S lastSu 1 0 - +R s 1977 o - Ap 2 23 1 S +R s 1978 o - Ap 2 2s 1 S +R s 1978 o - O 1 2s 0 - +R Sp 1967 o - Jun 3 12 1 S +R Sp 1967 o - O 1 0 0 - +R Sp 1974 o - Jun 24 0 1 S +R Sp 1974 o - S 1 0 0 - +R Sp 1976 1977 - May 1 0 1 S +R Sp 1976 o - Au 1 0 0 - +R Sp 1977 o - S 28 0 0 - +R Sp 1978 o - Jun 1 0 1 S +R Sp 1978 o - Au 4 0 0 - +R CH 1941 1942 - May M>=1 1 1 S +R CH 1941 1942 - O M>=1 2 0 - +R T 1916 o - May 1 0 1 S +R T 1916 o - O 1 0 0 - +R T 1920 o - Mar 28 0 1 S +R T 1920 o - O 25 0 0 - +R T 1921 o - Ap 3 0 1 S +R T 1921 o - O 3 0 0 - +R T 1922 o - Mar 26 0 1 S +R T 1922 o - O 8 0 0 - +R T 1924 o - May 13 0 1 S +R T 1924 1925 - O 1 0 0 - +R T 1925 o - May 1 0 1 S +R T 1940 o - Jul 1 0 1 S +R T 1940 o - O 6 0 0 - +R T 1940 o - D 1 0 1 S +R T 1941 o - S 21 0 0 - +R T 1942 o - Ap 1 0 1 S +R T 1945 o - O 8 0 0 - +R T 1946 o - Jun 1 0 1 S +R T 1946 o - O 1 0 0 - +R T 1947 1948 - Ap Su>=16 0 1 S +R T 1947 1951 - O Su>=2 0 0 - +R T 1949 o - Ap 10 0 1 S +R T 1950 o - Ap 16 0 1 S +R T 1951 o - Ap 22 0 1 S +R T 1962 o - Jul 15 0 1 S +R T 1963 o - O 30 0 0 - +R T 1964 o - May 15 0 1 S +R T 1964 o - O 1 0 0 - +R T 1973 o - Jun 3 1 1 S +R T 1973 1976 - O Su>=31 2 0 - +R T 1974 o - Mar 31 2 1 S +R T 1975 o - Mar 22 2 1 S +R T 1976 o - Mar 21 2 1 S +R T 1977 1978 - Ap Su>=1 2 1 S +R T 1977 1978 - O Su>=15 2 0 - +R T 1978 o - Jun 29 0 0 - +R T 1983 o - Jul 31 2 1 S +R T 1983 o - O 2 2 0 - +R T 1985 o - Ap 20 1s 1 S +R T 1985 o - S 28 1s 0 - +R T 1986 1993 - Mar lastSu 1s 1 S +R T 1986 1995 - S lastSu 1s 0 - +R T 1994 o - Mar 20 1s 1 S +R T 1995 2006 - Mar lastSu 1s 1 S +R T 1996 2006 - O lastSu 1s 0 - +R u 1918 1919 - Mar lastSu 2 1 D +R u 1918 1919 - O lastSu 2 0 S +R u 1942 o - F 9 2 1 W +R u 1945 o - Au 14 23u 1 P +R u 1945 o - S 30 2 0 S +R u 1967 2006 - O lastSu 2 0 S +R u 1967 1973 - Ap lastSu 2 1 D +R u 1974 o - Ja 6 2 1 D +R u 1975 o - F lastSu 2 1 D +R u 1976 1986 - Ap lastSu 2 1 D +R u 1987 2006 - Ap Su>=1 2 1 D +R u 2007 ma - Mar Su>=8 2 1 D +R u 2007 ma - N Su>=1 2 0 S +R NY 1920 o - Mar lastSu 2 1 D +R NY 1920 o - O lastSu 2 0 S +R NY 1921 1966 - Ap lastSu 2 1 D +R NY 1921 1954 - S lastSu 2 0 S +R NY 1955 1966 - O lastSu 2 0 S +R Ch 1920 o - Jun 13 2 1 D +R Ch 1920 1921 - O lastSu 2 0 S +R Ch 1921 o - Mar lastSu 2 1 D +R Ch 1922 1966 - Ap lastSu 2 1 D +R Ch 1922 1954 - S lastSu 2 0 S +R Ch 1955 1966 - O lastSu 2 0 S +R De 1920 1921 - Mar lastSu 2 1 D +R De 1920 o - O lastSu 2 0 S +R De 1921 o - May 22 2 0 S +R De 1965 1966 - Ap lastSu 2 1 D +R De 1965 1966 - O lastSu 2 0 S +R CA 1948 o - Mar 14 2:1 1 D +R CA 1949 o - Ja 1 2 0 S +R CA 1950 1966 - Ap lastSu 1 1 D +R CA 1950 1961 - S lastSu 2 0 S +R CA 1962 1966 - O lastSu 2 0 S +R In 1941 o - Jun 22 2 1 D +R In 1941 1954 - S lastSu 2 0 S +R In 1946 1954 - Ap lastSu 2 1 D +R Ma 1951 o - Ap lastSu 2 1 D +R Ma 1951 o - S lastSu 2 0 S +R Ma 1954 1960 - Ap lastSu 2 1 D +R Ma 1954 1960 - S lastSu 2 0 S +R V 1946 o - Ap lastSu 2 1 D +R V 1946 o - S lastSu 2 0 S +R V 1953 1954 - Ap lastSu 2 1 D +R V 1953 1959 - S lastSu 2 0 S +R V 1955 o - May 1 0 1 D +R V 1956 1963 - Ap lastSu 2 1 D +R V 1960 o - O lastSu 2 0 S +R V 1961 o - S lastSu 2 0 S +R V 1962 1963 - O lastSu 2 0 S +R Pe 1955 o - May 1 0 1 D +R Pe 1955 1960 - S lastSu 2 0 S +R Pe 1956 1963 - Ap lastSu 2 1 D +R Pe 1961 1963 - O lastSu 2 0 S +R Pi 1955 o - May 1 0 1 D +R Pi 1955 1960 - S lastSu 2 0 S +R Pi 1956 1964 - Ap lastSu 2 1 D +R Pi 1961 1964 - O lastSu 2 0 S +R St 1947 1961 - Ap lastSu 2 1 D +R St 1947 1954 - S lastSu 2 0 S +R St 1955 1956 - O lastSu 2 0 S +R St 1957 1958 - S lastSu 2 0 S +R St 1959 1961 - O lastSu 2 0 S +R Pu 1946 1960 - Ap lastSu 2 1 D +R Pu 1946 1954 - S lastSu 2 0 S +R Pu 1955 1956 - O lastSu 2 0 S +R Pu 1957 1960 - S lastSu 2 0 S +R v 1921 o - May 1 2 1 D +R v 1921 o - S 1 2 0 S +R v 1941 o - Ap lastSu 2 1 D +R v 1941 o - S lastSu 2 0 S +R v 1946 o - Ap lastSu 0:1 1 D +R v 1946 o - Jun 2 2 0 S +R v 1950 1961 - Ap lastSu 2 1 D +R v 1950 1955 - S lastSu 2 0 S +R v 1956 1961 - O lastSu 2 0 S +R Dt 1948 o - Ap lastSu 2 1 D +R Dt 1948 o - S lastSu 2 0 S +R Me 1946 o - Ap lastSu 2 1 D +R Me 1946 o - S lastSu 2 0 S +R Me 1966 o - Ap lastSu 2 1 D +R Me 1966 o - O lastSu 2 0 S +R C 1918 o - Ap 14 2 1 D +R C 1918 o - O 27 2 0 S +R C 1942 o - F 9 2 1 W +R C 1945 o - Au 14 23u 1 P +R C 1945 o - S 30 2 0 S +R C 1974 1986 - Ap lastSu 2 1 D +R C 1974 2006 - O lastSu 2 0 S +R C 1987 2006 - Ap Su>=1 2 1 D +R C 2007 ma - Mar Su>=8 2 1 D +R C 2007 ma - N Su>=1 2 0 S +R j 1917 o - Ap 8 2 1 D +R j 1917 o - S 17 2 0 S +R j 1919 o - May 5 23 1 D +R j 1919 o - Au 12 23 0 S +R j 1920 1935 - May Su>=1 23 1 D +R j 1920 1935 - O lastSu 23 0 S +R j 1936 1941 - May M>=9 0 1 D +R j 1936 1941 - O M>=2 0 0 S +R j 1946 1950 - May Su>=8 2 1 D +R j 1946 1950 - O Su>=2 2 0 S +R j 1951 1986 - Ap lastSu 2 1 D +R j 1951 1959 - S lastSu 2 0 S +R j 1960 1986 - O lastSu 2 0 S +R j 1987 o - Ap Su>=1 0:1 1 D +R j 1987 2006 - O lastSu 0:1 0 S +R j 1988 o - Ap Su>=1 0:1 2 DD +R j 1989 2006 - Ap Su>=1 0:1 1 D +R j 2007 2011 - Mar Su>=8 0:1 1 D +R j 2007 2010 - N Su>=1 0:1 0 S +R H 1916 o - Ap 1 0 1 D +R H 1916 o - O 1 0 0 S +R H 1920 o - May 9 0 1 D +R H 1920 o - Au 29 0 0 S +R H 1921 o - May 6 0 1 D +R H 1921 1922 - S 5 0 0 S +R H 1922 o - Ap 30 0 1 D +R H 1923 1925 - May Su>=1 0 1 D +R H 1923 o - S 4 0 0 S +R H 1924 o - S 15 0 0 S +R H 1925 o - S 28 0 0 S +R H 1926 o - May 16 0 1 D +R H 1926 o - S 13 0 0 S +R H 1927 o - May 1 0 1 D +R H 1927 o - S 26 0 0 S +R H 1928 1931 - May Su>=8 0 1 D +R H 1928 o - S 9 0 0 S +R H 1929 o - S 3 0 0 S +R H 1930 o - S 15 0 0 S +R H 1931 1932 - S M>=24 0 0 S +R H 1932 o - May 1 0 1 D +R H 1933 o - Ap 30 0 1 D +R H 1933 o - O 2 0 0 S +R H 1934 o - May 20 0 1 D +R H 1934 o - S 16 0 0 S +R H 1935 o - Jun 2 0 1 D +R H 1935 o - S 30 0 0 S +R H 1936 o - Jun 1 0 1 D +R H 1936 o - S 14 0 0 S +R H 1937 1938 - May Su>=1 0 1 D +R H 1937 1941 - S M>=24 0 0 S +R H 1939 o - May 28 0 1 D +R H 1940 1941 - May Su>=1 0 1 D +R H 1946 1949 - Ap lastSu 2 1 D +R H 1946 1949 - S lastSu 2 0 S +R H 1951 1954 - Ap lastSu 2 1 D +R H 1951 1954 - S lastSu 2 0 S +R H 1956 1959 - Ap lastSu 2 1 D +R H 1956 1959 - S lastSu 2 0 S +R H 1962 1973 - Ap lastSu 2 1 D +R H 1962 1973 - O lastSu 2 0 S +R o 1933 1935 - Jun Su>=8 1 1 D +R o 1933 1935 - S Su>=8 1 0 S +R o 1936 1938 - Jun Su>=1 1 1 D +R o 1936 1938 - S Su>=1 1 0 S +R o 1939 o - May 27 1 1 D +R o 1939 1941 - S Sa>=21 1 0 S +R o 1940 o - May 19 1 1 D +R o 1941 o - May 4 1 1 D +R o 1946 1972 - Ap lastSu 2 1 D +R o 1946 1956 - S lastSu 2 0 S +R o 1957 1972 - O lastSu 2 0 S +R o 1993 2006 - Ap Su>=1 0:1 1 D +R o 1993 2006 - O lastSu 0:1 0 S +R t 1919 o - Mar 30 23:30 1 D +R t 1919 o - O 26 0 0 S +R t 1920 o - May 2 2 1 D +R t 1920 o - S 26 0 0 S +R t 1921 o - May 15 2 1 D +R t 1921 o - S 15 2 0 S +R t 1922 1923 - May Su>=8 2 1 D +R t 1922 1926 - S Su>=15 2 0 S +R t 1924 1927 - May Su>=1 2 1 D +R t 1927 1937 - S Su>=25 2 0 S +R t 1928 1937 - Ap Su>=25 2 1 D +R t 1938 1940 - Ap lastSu 2 1 D +R t 1938 1939 - S lastSu 2 0 S +R t 1945 1948 - S lastSu 2 0 S +R t 1946 1973 - Ap lastSu 2 1 D +R t 1949 1950 - N lastSu 2 0 S +R t 1951 1956 - S lastSu 2 0 S +R t 1957 1973 - O lastSu 2 0 S +R W 1916 o - Ap 23 0 1 D +R W 1916 o - S 17 0 0 S +R W 1918 o - Ap 14 2 1 D +R W 1918 o - O 27 2 0 S +R W 1937 o - May 16 2 1 D +R W 1937 o - S 26 2 0 S +R W 1942 o - F 9 2 1 W +R W 1945 o - Au 14 23u 1 P +R W 1945 o - S lastSu 2 0 S +R W 1946 o - May 12 2 1 D +R W 1946 o - O 13 2 0 S +R W 1947 1949 - Ap lastSu 2 1 D +R W 1947 1949 - S lastSu 2 0 S +R W 1950 o - May 1 2 1 D +R W 1950 o - S 30 2 0 S +R W 1951 1960 - Ap lastSu 2 1 D +R W 1951 1958 - S lastSu 2 0 S +R W 1959 o - O lastSu 2 0 S +R W 1960 o - S lastSu 2 0 S +R W 1963 o - Ap lastSu 2 1 D +R W 1963 o - S 22 2 0 S +R W 1966 1986 - Ap lastSu 2s 1 D +R W 1966 2005 - O lastSu 2s 0 S +R W 1987 2005 - Ap Su>=1 2s 1 D +R r 1918 o - Ap 14 2 1 D +R r 1918 o - O 27 2 0 S +R r 1930 1934 - May Su>=1 0 1 D +R r 1930 1934 - O Su>=1 0 0 S +R r 1937 1941 - Ap Su>=8 0 1 D +R r 1937 o - O Su>=8 0 0 S +R r 1938 o - O Su>=1 0 0 S +R r 1939 1941 - O Su>=8 0 0 S +R r 1942 o - F 9 2 1 W +R r 1945 o - Au 14 23u 1 P +R r 1945 o - S lastSu 2 0 S +R r 1946 o - Ap Su>=8 2 1 D +R r 1946 o - O Su>=8 2 0 S +R r 1947 1957 - Ap lastSu 2 1 D +R r 1947 1957 - S lastSu 2 0 S +R r 1959 o - Ap lastSu 2 1 D +R r 1959 o - O lastSu 2 0 S +R Sw 1957 o - Ap lastSu 2 1 D +R Sw 1957 o - O lastSu 2 0 S +R Sw 1959 1961 - Ap lastSu 2 1 D +R Sw 1959 o - O lastSu 2 0 S +R Sw 1960 1961 - S lastSu 2 0 S +R Ed 1918 1919 - Ap Su>=8 2 1 D +R Ed 1918 o - O 27 2 0 S +R Ed 1919 o - May 27 2 0 S +R Ed 1920 1923 - Ap lastSu 2 1 D +R Ed 1920 o - O lastSu 2 0 S +R Ed 1921 1923 - S lastSu 2 0 S +R Ed 1942 o - F 9 2 1 W +R Ed 1945 o - Au 14 23u 1 P +R Ed 1945 o - S lastSu 2 0 S +R Ed 1947 o - Ap lastSu 2 1 D +R Ed 1947 o - S lastSu 2 0 S +R Ed 1972 1986 - Ap lastSu 2 1 D +R Ed 1972 2006 - O lastSu 2 0 S +R Va 1918 o - Ap 14 2 1 D +R Va 1918 o - O 27 2 0 S +R Va 1942 o - F 9 2 1 W +R Va 1945 o - Au 14 23u 1 P +R Va 1945 o - S 30 2 0 S +R Va 1946 1986 - Ap lastSu 2 1 D +R Va 1946 o - S 29 2 0 S +R Va 1947 1961 - S lastSu 2 0 S +R Va 1962 2006 - O lastSu 2 0 S +R Y 1918 o - Ap 14 2 1 D +R Y 1918 o - O 27 2 0 S +R Y 1919 o - May 25 2 1 D +R Y 1919 o - N 1 0 0 S +R Y 1942 o - F 9 2 1 W +R Y 1945 o - Au 14 23u 1 P +R Y 1945 o - S 30 2 0 S +R Y 1972 1986 - Ap lastSu 2 1 D +R Y 1972 2006 - O lastSu 2 0 S +R Y 1987 2006 - Ap Su>=1 2 1 D +R Yu 1965 o - Ap lastSu 0 2 DD +R Yu 1965 o - O lastSu 2 0 S +R m 1931 o - Ap 30 0 1 D +R m 1931 o - O 1 0 0 S +R m 1939 o - F 5 0 1 D +R m 1939 o - Jun 25 0 0 S +R m 1940 o - D 9 0 1 D +R m 1941 o - Ap 1 0 0 S +R m 1943 o - D 16 0 1 W +R m 1944 o - May 1 0 0 S +R m 1950 o - F 12 0 1 D +R m 1950 o - Jul 30 0 0 S +R m 1996 2000 - Ap Su>=1 2 1 D +R m 1996 2000 - O lastSu 2 0 S +R m 2001 o - May Su>=1 2 1 D +R m 2001 o - S lastSu 2 0 S +R m 2002 2022 - Ap Su>=1 2 1 D +R m 2002 2022 - O lastSu 2 0 S +R BB 1942 o - Ap 19 5u 1 D +R BB 1942 o - Au 31 6u 0 S +R BB 1943 o - May 2 5u 1 D +R BB 1943 o - S 5 6u 0 S +R BB 1944 o - Ap 10 5u 0:30 - +R BB 1944 o - S 10 6u 0 S +R BB 1977 o - Jun 12 2 1 D +R BB 1977 1978 - O Su>=1 2 0 S +R BB 1978 1980 - Ap Su>=15 2 1 D +R BB 1979 o - S 30 2 0 S +R BB 1980 o - S 25 2 0 S +R BZ 1918 1941 - O Sa>=1 24 0:30 -0530 +R BZ 1919 1942 - F Sa>=8 24 0 CST +R BZ 1942 o - Jun 27 24 1 CWT +R BZ 1945 o - Au 14 23u 1 CPT +R BZ 1945 o - D 15 24 0 CST +R BZ 1947 1967 - O Sa>=1 24 0:30 -0530 +R BZ 1948 1968 - F Sa>=8 24 0 CST +R BZ 1973 o - D 5 0 1 CDT +R BZ 1974 o - F 9 0 0 CST +R BZ 1982 o - D 18 0 1 CDT +R BZ 1983 o - F 12 0 0 CST +R Be 1917 o - Ap 5 24 1 - +R Be 1917 o - S 30 24 0 - +R Be 1918 o - Ap 13 24 1 - +R Be 1918 o - S 15 24 0 S +R Be 1942 o - Ja 11 2 1 D +R Be 1942 o - O 18 2 0 S +R Be 1943 o - Mar 21 2 1 D +R Be 1943 o - O 31 2 0 S +R Be 1944 1945 - Mar Su>=8 2 1 D +R Be 1944 1945 - N Su>=1 2 0 S +R Be 1947 o - May Su>=15 2 1 D +R Be 1947 o - S Su>=8 2 0 S +R Be 1948 1952 - May Su>=22 2 1 D +R Be 1948 1952 - S Su>=1 2 0 S +R Be 1956 o - May Su>=22 2 1 D +R Be 1956 o - O lastSu 2 0 S +R CR 1979 1980 - F lastSu 0 1 D +R CR 1979 1980 - Jun Su>=1 0 0 S +R CR 1991 1992 - Ja Sa>=15 0 1 D +R CR 1991 o - Jul 1 0 0 S +R CR 1992 o - Mar 15 0 0 S +R Q 1928 o - Jun 10 0 1 D +R Q 1928 o - O 10 0 0 S +R Q 1940 1942 - Jun Su>=1 0 1 D +R Q 1940 1942 - S Su>=1 0 0 S +R Q 1945 1946 - Jun Su>=1 0 1 D +R Q 1945 1946 - S Su>=1 0 0 S +R Q 1965 o - Jun 1 0 1 D +R Q 1965 o - S 30 0 0 S +R Q 1966 o - May 29 0 1 D +R Q 1966 o - O 2 0 0 S +R Q 1967 o - Ap 8 0 1 D +R Q 1967 1968 - S Su>=8 0 0 S +R Q 1968 o - Ap 14 0 1 D +R Q 1969 1977 - Ap lastSu 0 1 D +R Q 1969 1971 - O lastSu 0 0 S +R Q 1972 1974 - O 8 0 0 S +R Q 1975 1977 - O lastSu 0 0 S +R Q 1978 o - May 7 0 1 D +R Q 1978 1990 - O Su>=8 0 0 S +R Q 1979 1980 - Mar Su>=15 0 1 D +R Q 1981 1985 - May Su>=5 0 1 D +R Q 1986 1989 - Mar Su>=14 0 1 D +R Q 1990 1997 - Ap Su>=1 0 1 D +R Q 1991 1995 - O Su>=8 0s 0 S +R Q 1996 o - O 6 0s 0 S +R Q 1997 o - O 12 0s 0 S +R Q 1998 1999 - Mar lastSu 0s 1 D +R Q 1998 2003 - O lastSu 0s 0 S +R Q 2000 2003 - Ap Su>=1 0s 1 D +R Q 2004 o - Mar lastSu 0s 1 D +R Q 2006 2010 - O lastSu 0s 0 S +R Q 2007 o - Mar Su>=8 0s 1 D +R Q 2008 o - Mar Su>=15 0s 1 D +R Q 2009 2010 - Mar Su>=8 0s 1 D +R Q 2011 o - Mar Su>=15 0s 1 D +R Q 2011 o - N 13 0s 0 S +R Q 2012 o - Ap 1 0s 1 D +R Q 2012 ma - N Su>=1 0s 0 S +R Q 2013 ma - Mar Su>=8 0s 1 D +R DO 1966 o - O 30 0 1 EDT +R DO 1967 o - F 28 0 0 EST +R DO 1969 1973 - O lastSu 0 0:30 -0430 +R DO 1970 o - F 21 0 0 EST +R DO 1971 o - Ja 20 0 0 EST +R DO 1972 1974 - Ja 21 0 0 EST +R SV 1987 1988 - May Su>=1 0 1 D +R SV 1987 1988 - S lastSu 0 0 S +R GT 1973 o - N 25 0 1 D +R GT 1974 o - F 24 0 0 S +R GT 1983 o - May 21 0 1 D +R GT 1983 o - S 22 0 0 S +R GT 1991 o - Mar 23 0 1 D +R GT 1991 o - S 7 0 0 S +R GT 2006 o - Ap 30 0 1 D +R GT 2006 o - O 1 0 0 S +R HT 1983 o - May 8 0 1 D +R HT 1984 1987 - Ap lastSu 0 1 D +R HT 1983 1987 - O lastSu 0 0 S +R HT 1988 1997 - Ap Su>=1 1s 1 D +R HT 1988 1997 - O lastSu 1s 0 S +R HT 2005 2006 - Ap Su>=1 0 1 D +R HT 2005 2006 - O lastSu 0 0 S +R HT 2012 2015 - Mar Su>=8 2 1 D +R HT 2012 2015 - N Su>=1 2 0 S +R HT 2017 ma - Mar Su>=8 2 1 D +R HT 2017 ma - N Su>=1 2 0 S +R HN 1987 1988 - May Su>=1 0 1 D +R HN 1987 1988 - S lastSu 0 0 S +R HN 2006 o - May Su>=1 0 1 D +R HN 2006 o - Au M>=1 0 0 S +R NI 1979 1980 - Mar Su>=16 0 1 D +R NI 1979 1980 - Jun M>=23 0 0 S +R NI 2005 o - Ap 10 0 1 D +R NI 2005 o - O Su>=1 0 0 S +R NI 2006 o - Ap 30 2 1 D +R NI 2006 o - O Su>=1 1 0 S +R A 1930 o - D 1 0 1 - +R A 1931 o - Ap 1 0 0 - +R A 1931 o - O 15 0 1 - +R A 1932 1940 - Mar 1 0 0 - +R A 1932 1939 - N 1 0 1 - +R A 1940 o - Jul 1 0 1 - +R A 1941 o - Jun 15 0 0 - +R A 1941 o - O 15 0 1 - +R A 1943 o - Au 1 0 0 - +R A 1943 o - O 15 0 1 - +R A 1946 o - Mar 1 0 0 - +R A 1946 o - O 1 0 1 - +R A 1963 o - O 1 0 0 - +R A 1963 o - D 15 0 1 - +R A 1964 1966 - Mar 1 0 0 - +R A 1964 1966 - O 15 0 1 - +R A 1967 o - Ap 2 0 0 - +R A 1967 1968 - O Su>=1 0 1 - +R A 1968 1969 - Ap Su>=1 0 0 - +R A 1974 o - Ja 23 0 1 - +R A 1974 o - May 1 0 0 - +R A 1988 o - D 1 0 1 - +R A 1989 1993 - Mar Su>=1 0 0 - +R A 1989 1992 - O Su>=15 0 1 - +R A 1999 o - O Su>=1 0 1 - +R A 2000 o - Mar 3 0 0 - +R A 2007 o - D 30 0 1 - +R A 2008 2009 - Mar Su>=15 0 0 - +R A 2008 o - O Su>=15 0 1 - +R Sa 2008 2009 - Mar Su>=8 0 0 - +R Sa 2007 2008 - O Su>=8 0 1 - +R B 1931 o - O 3 11 1 - +R B 1932 1933 - Ap 1 0 0 - +R B 1932 o - O 3 0 1 - +R B 1949 1952 - D 1 0 1 - +R B 1950 o - Ap 16 1 0 - +R B 1951 1952 - Ap 1 0 0 - +R B 1953 o - Mar 1 0 0 - +R B 1963 o - D 9 0 1 - +R B 1964 o - Mar 1 0 0 - +R B 1965 o - Ja 31 0 1 - +R B 1965 o - Mar 31 0 0 - +R B 1965 o - D 1 0 1 - +R B 1966 1968 - Mar 1 0 0 - +R B 1966 1967 - N 1 0 1 - +R B 1985 o - N 2 0 1 - +R B 1986 o - Mar 15 0 0 - +R B 1986 o - O 25 0 1 - +R B 1987 o - F 14 0 0 - +R B 1987 o - O 25 0 1 - +R B 1988 o - F 7 0 0 - +R B 1988 o - O 16 0 1 - +R B 1989 o - Ja 29 0 0 - +R B 1989 o - O 15 0 1 - +R B 1990 o - F 11 0 0 - +R B 1990 o - O 21 0 1 - +R B 1991 o - F 17 0 0 - +R B 1991 o - O 20 0 1 - +R B 1992 o - F 9 0 0 - +R B 1992 o - O 25 0 1 - +R B 1993 o - Ja 31 0 0 - +R B 1993 1995 - O Su>=11 0 1 - +R B 1994 1995 - F Su>=15 0 0 - +R B 1996 o - F 11 0 0 - +R B 1996 o - O 6 0 1 - +R B 1997 o - F 16 0 0 - +R B 1997 o - O 6 0 1 - +R B 1998 o - Mar 1 0 0 - +R B 1998 o - O 11 0 1 - +R B 1999 o - F 21 0 0 - +R B 1999 o - O 3 0 1 - +R B 2000 o - F 27 0 0 - +R B 2000 2001 - O Su>=8 0 1 - +R B 2001 2006 - F Su>=15 0 0 - +R B 2002 o - N 3 0 1 - +R B 2003 o - O 19 0 1 - +R B 2004 o - N 2 0 1 - +R B 2005 o - O 16 0 1 - +R B 2006 o - N 5 0 1 - +R B 2007 o - F 25 0 0 - +R B 2007 o - O Su>=8 0 1 - +R B 2008 2017 - O Su>=15 0 1 - +R B 2008 2011 - F Su>=15 0 0 - +R B 2012 o - F Su>=22 0 0 - +R B 2013 2014 - F Su>=15 0 0 - +R B 2015 o - F Su>=22 0 0 - +R B 2016 2019 - F Su>=15 0 0 - +R B 2018 o - N Su>=1 0 1 - +R x 1927 1931 - S 1 0 1 - +R x 1928 1932 - Ap 1 0 0 - +R x 1968 o - N 3 4u 1 - +R x 1969 o - Mar 30 3u 0 - +R x 1969 o - N 23 4u 1 - +R x 1970 o - Mar 29 3u 0 - +R x 1971 o - Mar 14 3u 0 - +R x 1970 1972 - O Su>=9 4u 1 - +R x 1972 1986 - Mar Su>=9 3u 0 - +R x 1973 o - S 30 4u 1 - +R x 1974 1987 - O Su>=9 4u 1 - +R x 1987 o - Ap 12 3u 0 - +R x 1988 1990 - Mar Su>=9 3u 0 - +R x 1988 1989 - O Su>=9 4u 1 - +R x 1990 o - S 16 4u 1 - +R x 1991 1996 - Mar Su>=9 3u 0 - +R x 1991 1997 - O Su>=9 4u 1 - +R x 1997 o - Mar 30 3u 0 - +R x 1998 o - Mar Su>=9 3u 0 - +R x 1998 o - S 27 4u 1 - +R x 1999 o - Ap 4 3u 0 - +R x 1999 2010 - O Su>=9 4u 1 - +R x 2000 2007 - Mar Su>=9 3u 0 - +R x 2008 o - Mar 30 3u 0 - +R x 2009 o - Mar Su>=9 3u 0 - +R x 2010 o - Ap Su>=1 3u 0 - +R x 2011 o - May Su>=2 3u 0 - +R x 2011 o - Au Su>=16 4u 1 - +R x 2012 2014 - Ap Su>=23 3u 0 - +R x 2012 2014 - S Su>=2 4u 1 - +R x 2016 2018 - May Su>=9 3u 0 - +R x 2016 2018 - Au Su>=9 4u 1 - +R x 2019 ma - Ap Su>=2 3u 0 - +R x 2019 2021 - S Su>=2 4u 1 - +R x 2022 o - S Su>=9 4u 1 - +R x 2023 ma - S Su>=2 4u 1 - +R CO 1992 o - May 3 0 1 - +R CO 1993 o - F 6 24 0 - +R EC 1992 o - N 28 0 1 - +R EC 1993 o - F 5 0 0 - +R FK 1937 1938 - S lastSu 0 1 - +R FK 1938 1942 - Mar Su>=19 0 0 - +R FK 1939 o - O 1 0 1 - +R FK 1940 1942 - S lastSu 0 1 - +R FK 1943 o - Ja 1 0 0 - +R FK 1983 o - S lastSu 0 1 - +R FK 1984 1985 - Ap lastSu 0 0 - +R FK 1984 o - S 16 0 1 - +R FK 1985 2000 - S Su>=9 0 1 - +R FK 1986 2000 - Ap Su>=16 0 0 - +R FK 2001 2010 - Ap Su>=15 2 0 - +R FK 2001 2010 - S Su>=1 2 1 - +R y 1975 1988 - O 1 0 1 - +R y 1975 1978 - Mar 1 0 0 - +R y 1979 1991 - Ap 1 0 0 - +R y 1989 o - O 22 0 1 - +R y 1990 o - O 1 0 1 - +R y 1991 o - O 6 0 1 - +R y 1992 o - Mar 1 0 0 - +R y 1992 o - O 5 0 1 - +R y 1993 o - Mar 31 0 0 - +R y 1993 1995 - O 1 0 1 - +R y 1994 1995 - F lastSu 0 0 - +R y 1996 o - Mar 1 0 0 - +R y 1996 2001 - O Su>=1 0 1 - +R y 1997 o - F lastSu 0 0 - +R y 1998 2001 - Mar Su>=1 0 0 - +R y 2002 2004 - Ap Su>=1 0 0 - +R y 2002 2003 - S Su>=1 0 1 - +R y 2004 2009 - O Su>=15 0 1 - +R y 2005 2009 - Mar Su>=8 0 0 - +R y 2010 2024 - O Su>=1 0 1 - +R y 2010 2012 - Ap Su>=8 0 0 - +R y 2013 2024 - Mar Su>=22 0 0 - +R PE 1938 o - Ja 1 0 1 - +R PE 1938 o - Ap 1 0 0 - +R PE 1938 1939 - S lastSu 0 1 - +R PE 1939 1940 - Mar Su>=24 0 0 - +R PE 1986 1987 - Ja 1 0 1 - +R PE 1986 1987 - Ap 1 0 0 - +R PE 1990 o - Ja 1 0 1 - +R PE 1990 o - Ap 1 0 0 - +R PE 1994 o - Ja 1 0 1 - +R PE 1994 o - Ap 1 0 0 - +R U 1923 1925 - O 1 0 0:30 - +R U 1924 1926 - Ap 1 0 0 - +R U 1933 1938 - O lastSu 0 0:30 - +R U 1934 1941 - Mar lastSa 24 0 - +R U 1939 o - O 1 0 0:30 - +R U 1940 o - O 27 0 0:30 - +R U 1941 o - Au 1 0 0:30 - +R U 1942 o - D 14 0 0:30 - +R U 1943 o - Mar 14 0 0 - +R U 1959 o - May 24 0 0:30 - +R U 1959 o - N 15 0 0 - +R U 1960 o - Ja 17 0 1 - +R U 1960 o - Mar 6 0 0 - +R U 1965 o - Ap 4 0 1 - +R U 1965 o - S 26 0 0 - +R U 1968 o - May 27 0 0:30 - +R U 1968 o - D 1 0 0 - +R U 1970 o - Ap 25 0 1 - +R U 1970 o - Jun 14 0 0 - +R U 1972 o - Ap 23 0 1 - +R U 1972 o - Jul 16 0 0 - +R U 1974 o - Ja 13 0 1:30 - +R U 1974 o - Mar 10 0 0:30 - +R U 1974 o - S 1 0 0 - +R U 1974 o - D 22 0 1 - +R U 1975 o - Mar 30 0 0 - +R U 1976 o - D 19 0 1 - +R U 1977 o - Mar 6 0 0 - +R U 1977 o - D 4 0 1 - +R U 1978 1979 - Mar Su>=1 0 0 - +R U 1978 o - D 17 0 1 - +R U 1979 o - Ap 29 0 1 - +R U 1980 o - Mar 16 0 0 - +R U 1987 o - D 14 0 1 - +R U 1988 o - F 28 0 0 - +R U 1988 o - D 11 0 1 - +R U 1989 o - Mar 5 0 0 - +R U 1989 o - O 29 0 1 - +R U 1990 o - F 25 0 0 - +R U 1990 1991 - O Su>=21 0 1 - +R U 1991 1992 - Mar Su>=1 0 0 - +R U 1992 o - O 18 0 1 - +R U 1993 o - F 28 0 0 - +R U 2004 o - S 19 0 1 - +R U 2005 o - Mar 27 2 0 - +R U 2005 o - O 9 2 1 - +R U 2006 2015 - Mar Su>=8 2 0 - +R U 2006 2014 - O Su>=1 2 1 - +Z Africa/Abidjan -0:16:8 - LMT 1912 +0 - GMT +Z Africa/Algiers 0:12:12 - LMT 1891 Mar 16 +0:9:21 - PMT 1911 Mar 11 +0 d WE%sT 1940 F 25 2 +1 d CE%sT 1946 O 7 +0 - WET 1956 Ja 29 +1 - CET 1963 Ap 14 +0 d WE%sT 1977 O 21 +1 d CE%sT 1979 O 26 +0 d WE%sT 1981 May +1 - CET +Z Africa/Bissau -1:2:20 - LMT 1912 Ja 1 1u +-1 - %z 1975 +0 - GMT +Z Africa/Cairo 2:5:9 - LMT 1900 O +2 K EE%sT +Z Africa/Casablanca -0:30:20 - LMT 1913 O 26 +0 M %z 1984 Mar 16 +1 - %z 1986 +0 M %z 2018 O 28 3 +1 M %z +Z Africa/Ceuta -0:21:16 - LMT 1901 Ja 1 0u +0 - WET 1918 May 6 23 +0 1 WEST 1918 O 7 23 +0 - WET 1924 +0 s WE%sT 1929 +0 - WET 1967 +0 Sp WE%sT 1984 Mar 16 +1 - CET 1986 +1 E CE%sT +Z Africa/El_Aaiun -0:52:48 - LMT 1934 +-1 - %z 1976 Ap 14 +0 M %z 2018 O 28 3 +1 M %z +Z Africa/Johannesburg 1:52 - LMT 1892 F 8 +1:30 - SAST 1903 Mar +2 SA SAST +Z Africa/Juba 2:6:28 - LMT 1931 +2 SD CA%sT 2000 Ja 15 12 +3 - EAT 2021 F +2 - CAT +Z Africa/Khartoum 2:10:8 - LMT 1931 +2 SD CA%sT 2000 Ja 15 12 +3 - EAT 2017 N +2 - CAT +Z Africa/Lagos 0:13:35 - LMT 1905 Jul +0 - GMT 1908 Jul +0:13:35 - LMT 1914 +0:30 - %z 1919 S +1 - WAT +Z Africa/Maputo 2:10:18 - LMT 1909 +2 - CAT +Z Africa/Monrovia -0:43:8 - LMT 1882 +-0:43:8 - MMT 1919 Mar +-0:44:30 - MMT 1972 Ja 7 +0 - GMT +Z Africa/Nairobi 2:27:16 - LMT 1908 May +2:30 - %z 1928 Jun 30 24 +3 - EAT 1930 Ja 4 24 +2:30 - %z 1936 D 31 24 +2:45 - %z 1942 Jul 31 24 +3 - EAT +Z Africa/Ndjamena 1:0:12 - LMT 1912 +1 - WAT 1979 O 14 +1 1 WAST 1980 Mar 8 +1 - WAT +Z Africa/Sao_Tome 0:26:56 - LMT 1884 +-0:36:45 - LMT 1912 Ja 1 0u +0 - GMT 2018 Ja 1 1 +1 - WAT 2019 Ja 1 2 +0 - GMT +Z Africa/Tripoli 0:52:44 - LMT 1920 +1 L CE%sT 1959 +2 - EET 1982 +1 L CE%sT 1990 May 4 +2 - EET 1996 S 30 +1 L CE%sT 1997 O 4 +2 - EET 2012 N 10 2 +1 L CE%sT 2013 O 25 2 +2 - EET +Z Africa/Tunis 0:40:44 - LMT 1881 May 12 +0:9:21 - PMT 1911 Mar 11 +1 n CE%sT +Z Africa/Windhoek 1:8:24 - LMT 1892 F 8 +1:30 - %z 1903 Mar +2 - SAST 1942 S 20 2 +2 1 SAST 1943 Mar 21 2 +2 - SAST 1990 Mar 21 +2 NA %s +Z America/Adak 12:13:22 - LMT 1867 O 19 12:44:35 +-11:46:38 - LMT 1900 Au 20 12 +-11 - NST 1942 +-11 u N%sT 1946 +-11 - NST 1967 Ap +-11 - BST 1969 +-11 u B%sT 1983 O 30 2 +-10 u AH%sT 1983 N 30 +-10 u H%sT +Z America/Anchorage 14:0:24 - LMT 1867 O 19 14:31:37 +-9:59:36 - LMT 1900 Au 20 12 +-10 - AST 1942 +-10 u A%sT 1967 Ap +-10 - AHST 1969 +-10 u AH%sT 1983 O 30 2 +-9 u Y%sT 1983 N 30 +-9 u AK%sT +Z America/Araguaina -3:12:48 - LMT 1914 +-3 B %z 1990 S 17 +-3 - %z 1995 S 14 +-3 B %z 2003 S 24 +-3 - %z 2012 O 21 +-3 B %z 2013 S +-3 - %z +Z America/Argentina/Buenos_Aires -3:53:48 - LMT 1894 O 31 +-4:16:48 - CMT 1920 May +-4 - %z 1930 D +-4 A %z 1969 O 5 +-3 A %z 1999 O 3 +-4 A %z 2000 Mar 3 +-3 A %z +Z America/Argentina/Catamarca -4:23:8 - LMT 1894 O 31 +-4:16:48 - CMT 1920 May +-4 - %z 1930 D +-4 A %z 1969 O 5 +-3 A %z 1991 Mar 3 +-4 - %z 1991 O 20 +-3 A %z 1999 O 3 +-4 A %z 2000 Mar 3 +-3 - %z 2004 Jun +-4 - %z 2004 Jun 20 +-3 A %z 2008 O 18 +-3 - %z +Z America/Argentina/Cordoba -4:16:48 - LMT 1894 O 31 +-4:16:48 - CMT 1920 May +-4 - %z 1930 D +-4 A %z 1969 O 5 +-3 A %z 1991 Mar 3 +-4 - %z 1991 O 20 +-3 A %z 1999 O 3 +-4 A %z 2000 Mar 3 +-3 A %z +Z America/Argentina/Jujuy -4:21:12 - LMT 1894 O 31 +-4:16:48 - CMT 1920 May +-4 - %z 1930 D +-4 A %z 1969 O 5 +-3 A %z 1990 Mar 4 +-4 - %z 1990 O 28 +-4 1 %z 1991 Mar 17 +-4 - %z 1991 O 6 +-3 1 %z 1992 +-3 A %z 1999 O 3 +-4 A %z 2000 Mar 3 +-3 A %z 2008 O 18 +-3 - %z +Z America/Argentina/La_Rioja -4:27:24 - LMT 1894 O 31 +-4:16:48 - CMT 1920 May +-4 - %z 1930 D +-4 A %z 1969 O 5 +-3 A %z 1991 Mar +-4 - %z 1991 May 7 +-3 A %z 1999 O 3 +-4 A %z 2000 Mar 3 +-3 - %z 2004 Jun +-4 - %z 2004 Jun 20 +-3 A %z 2008 O 18 +-3 - %z +Z America/Argentina/Mendoza -4:35:16 - LMT 1894 O 31 +-4:16:48 - CMT 1920 May +-4 - %z 1930 D +-4 A %z 1969 O 5 +-3 A %z 1990 Mar 4 +-4 - %z 1990 O 15 +-4 1 %z 1991 Mar +-4 - %z 1991 O 15 +-4 1 %z 1992 Mar +-4 - %z 1992 O 18 +-3 A %z 1999 O 3 +-4 A %z 2000 Mar 3 +-3 - %z 2004 May 23 +-4 - %z 2004 S 26 +-3 A %z 2008 O 18 +-3 - %z +Z America/Argentina/Rio_Gallegos -4:36:52 - LMT 1894 O 31 +-4:16:48 - CMT 1920 May +-4 - %z 1930 D +-4 A %z 1969 O 5 +-3 A %z 1999 O 3 +-4 A %z 2000 Mar 3 +-3 - %z 2004 Jun +-4 - %z 2004 Jun 20 +-3 A %z 2008 O 18 +-3 - %z +Z America/Argentina/Salta -4:21:40 - LMT 1894 O 31 +-4:16:48 - CMT 1920 May +-4 - %z 1930 D +-4 A %z 1969 O 5 +-3 A %z 1991 Mar 3 +-4 - %z 1991 O 20 +-3 A %z 1999 O 3 +-4 A %z 2000 Mar 3 +-3 A %z 2008 O 18 +-3 - %z +Z America/Argentina/San_Juan -4:34:4 - LMT 1894 O 31 +-4:16:48 - CMT 1920 May +-4 - %z 1930 D +-4 A %z 1969 O 5 +-3 A %z 1991 Mar +-4 - %z 1991 May 7 +-3 A %z 1999 O 3 +-4 A %z 2000 Mar 3 +-3 - %z 2004 May 31 +-4 - %z 2004 Jul 25 +-3 A %z 2008 O 18 +-3 - %z +Z America/Argentina/San_Luis -4:25:24 - LMT 1894 O 31 +-4:16:48 - CMT 1920 May +-4 - %z 1930 D +-4 A %z 1969 O 5 +-3 A %z 1990 +-3 1 %z 1990 Mar 14 +-4 - %z 1990 O 15 +-4 1 %z 1991 Mar +-4 - %z 1991 Jun +-3 - %z 1999 O 3 +-4 1 %z 2000 Mar 3 +-3 - %z 2004 May 31 +-4 - %z 2004 Jul 25 +-3 A %z 2008 Ja 21 +-4 Sa %z 2009 O 11 +-3 - %z +Z America/Argentina/Tucuman -4:20:52 - LMT 1894 O 31 +-4:16:48 - CMT 1920 May +-4 - %z 1930 D +-4 A %z 1969 O 5 +-3 A %z 1991 Mar 3 +-4 - %z 1991 O 20 +-3 A %z 1999 O 3 +-4 A %z 2000 Mar 3 +-3 - %z 2004 Jun +-4 - %z 2004 Jun 13 +-3 A %z +Z America/Argentina/Ushuaia -4:33:12 - LMT 1894 O 31 +-4:16:48 - CMT 1920 May +-4 - %z 1930 D +-4 A %z 1969 O 5 +-3 A %z 1999 O 3 +-4 A %z 2000 Mar 3 +-3 - %z 2004 May 30 +-4 - %z 2004 Jun 20 +-3 A %z 2008 O 18 +-3 - %z +Z America/Asuncion -3:50:40 - LMT 1890 +-3:50:40 - AMT 1931 O 10 +-4 - %z 1972 O +-3 - %z 1974 Ap +-4 y %z 2024 O 15 +-3 - %z +Z America/Bahia -2:34:4 - LMT 1914 +-3 B %z 2003 S 24 +-3 - %z 2011 O 16 +-3 B %z 2012 O 21 +-3 - %z +Z America/Bahia_Banderas -7:1 - LMT 1922 Ja 1 7u +-7 - MST 1927 Jun 10 +-6 - CST 1930 N 15 +-7 m M%sT 1932 Ap +-6 - CST 1942 Ap 24 +-7 - MST 1970 +-7 m M%sT 2010 Ap 4 2 +-6 m C%sT +Z America/Barbados -3:58:29 - LMT 1911 Au 28 +-4 BB A%sT 1944 +-4 BB AST/-0330 1945 +-4 BB A%sT +Z America/Belem -3:13:56 - LMT 1914 +-3 B %z 1988 S 12 +-3 - %z +Z America/Belize -5:52:48 - LMT 1912 Ap +-6 BZ %s +Z America/Boa_Vista -4:2:40 - LMT 1914 +-4 B %z 1988 S 12 +-4 - %z 1999 S 30 +-4 B %z 2000 O 15 +-4 - %z +Z America/Bogota -4:56:16 - LMT 1884 Mar 13 +-4:56:16 - BMT 1914 N 23 +-5 CO %z +Z America/Boise -7:44:49 - LMT 1883 N 18 20u +-8 u P%sT 1923 May 13 2 +-7 u M%sT 1974 +-7 - MST 1974 F 3 2 +-7 u M%sT +Z America/Cambridge_Bay 0 - -00 1920 +-7 Y M%sT 1999 O 31 2 +-6 C C%sT 2000 O 29 2 +-5 - EST 2000 N 5 +-6 - CST 2001 Ap 1 3 +-7 C M%sT +Z America/Campo_Grande -3:38:28 - LMT 1914 +-4 B %z +Z America/Cancun -5:47:4 - LMT 1922 Ja 1 6u +-6 - CST 1981 D 26 2 +-5 - EST 1983 Ja 4 +-6 m C%sT 1997 O 26 2 +-5 m E%sT 1998 Au 2 2 +-6 m C%sT 2015 F 1 2 +-5 - EST +Z America/Caracas -4:27:44 - LMT 1890 +-4:27:40 - CMT 1912 F 12 +-4:30 - %z 1965 +-4 - %z 2007 D 9 3 +-4:30 - %z 2016 May 1 2:30 +-4 - %z +Z America/Cayenne -3:29:20 - LMT 1911 Jul +-4 - %z 1967 O +-3 - %z +Z America/Chicago -5:50:36 - LMT 1883 N 18 18u +-6 u C%sT 1920 +-6 Ch C%sT 1936 Mar 1 2 +-5 - EST 1936 N 15 2 +-6 Ch C%sT 1942 +-6 u C%sT 1946 +-6 Ch C%sT 1967 +-6 u C%sT +Z America/Chihuahua -7:4:20 - LMT 1922 Ja 1 7u +-7 - MST 1927 Jun 10 +-6 - CST 1930 N 15 +-7 m M%sT 1932 Ap +-6 - CST 1996 +-6 m C%sT 1998 +-6 - CST 1998 Ap Su>=1 3 +-7 m M%sT 2022 O 30 2 +-6 - CST +Z America/Ciudad_Juarez -7:5:56 - LMT 1922 Ja 1 7u +-7 - MST 1927 Jun 10 +-6 - CST 1930 N 15 +-7 m M%sT 1932 Ap +-6 - CST 1996 +-6 m C%sT 1998 +-6 - CST 1998 Ap Su>=1 3 +-7 m M%sT 2010 +-7 u M%sT 2022 O 30 2 +-6 - CST 2022 N 30 +-7 u M%sT +Z America/Costa_Rica -5:36:13 - LMT 1890 +-5:36:13 - SJMT 1921 Ja 15 +-6 CR C%sT +Z America/Cuiaba -3:44:20 - LMT 1914 +-4 B %z 2003 S 24 +-4 - %z 2004 O +-4 B %z +Z America/Danmarkshavn -1:14:40 - LMT 1916 Jul 28 +-3 - %z 1980 Ap 6 2 +-3 E %z 1996 +0 - GMT +Z America/Dawson -9:17:40 - LMT 1900 Au 20 +-9 Y Y%sT 1965 +-9 Yu Y%sT 1973 O 28 +-8 - PST 1980 +-8 C P%sT 2020 N +-7 - MST +Z America/Dawson_Creek -8:0:56 - LMT 1884 +-8 C P%sT 1947 +-8 Va P%sT 1972 Au 30 2 +-7 - MST +Z America/Denver -6:59:56 - LMT 1883 N 18 19u +-7 u M%sT 1920 +-7 De M%sT 1942 +-7 u M%sT 1946 +-7 De M%sT 1967 +-7 u M%sT +Z America/Detroit -5:32:11 - LMT 1905 +-6 - CST 1915 May 15 2 +-5 - EST 1942 +-5 u E%sT 1946 +-5 Dt E%sT 1967 Jun 14 0:1 +-5 u E%sT 1969 +-5 - EST 1973 +-5 u E%sT 1975 +-5 - EST 1975 Ap 27 2 +-5 u E%sT +Z America/Edmonton -7:33:52 - LMT 1906 S +-7 Ed M%sT 1987 +-7 C M%sT +Z America/Eirunepe -4:39:28 - LMT 1914 +-5 B %z 1988 S 12 +-5 - %z 1993 S 28 +-5 B %z 1994 S 22 +-5 - %z 2008 Jun 24 +-4 - %z 2013 N 10 +-5 - %z +Z America/El_Salvador -5:56:48 - LMT 1921 +-6 SV C%sT +Z America/Fort_Nelson -8:10:47 - LMT 1884 +-8 Va P%sT 1946 +-8 - PST 1947 +-8 Va P%sT 1987 +-8 C P%sT 2015 Mar 8 2 +-7 - MST +Z America/Fortaleza -2:34 - LMT 1914 +-3 B %z 1990 S 17 +-3 - %z 1999 S 30 +-3 B %z 2000 O 22 +-3 - %z 2001 S 13 +-3 B %z 2002 O +-3 - %z +Z America/Glace_Bay -3:59:48 - LMT 1902 Jun 15 +-4 C A%sT 1953 +-4 H A%sT 1954 +-4 - AST 1972 +-4 H A%sT 1974 +-4 C A%sT +Z America/Goose_Bay -4:1:40 - LMT 1884 +-3:30:52 - NST 1918 +-3:30:52 C N%sT 1919 +-3:30:52 - NST 1935 Mar 30 +-3:30 - NST 1936 +-3:30 j N%sT 1942 May 11 +-3:30 C N%sT 1946 +-3:30 j N%sT 1966 Mar 15 2 +-4 j A%sT 2011 N +-4 C A%sT +Z America/Grand_Turk -4:44:32 - LMT 1890 +-5:7:10 - KMT 1912 F +-5 - EST 1979 +-5 u E%sT 2015 Mar 8 2 +-4 - AST 2018 Mar 11 3 +-5 u E%sT +Z America/Guatemala -6:2:4 - LMT 1918 O 5 +-6 GT C%sT +Z America/Guayaquil -5:19:20 - LMT 1890 +-5:14 - QMT 1931 +-5 EC %z +Z America/Guyana -3:52:39 - LMT 1911 Au +-4 - %z 1915 Mar +-3:45 - %z 1975 Au +-3 - %z 1992 Mar 29 1 +-4 - %z +Z America/Halifax -4:14:24 - LMT 1902 Jun 15 +-4 H A%sT 1918 +-4 C A%sT 1919 +-4 H A%sT 1942 F 9 2s +-4 C A%sT 1946 +-4 H A%sT 1974 +-4 C A%sT +Z America/Havana -5:29:28 - LMT 1890 +-5:29:36 - HMT 1925 Jul 19 12 +-5 Q C%sT +Z America/Hermosillo -7:23:52 - LMT 1922 Ja 1 7u +-7 - MST 1927 Jun 10 +-6 - CST 1930 N 15 +-7 m M%sT 1932 Ap +-6 - CST 1942 Ap 24 +-7 - MST 1996 +-7 m M%sT 1999 +-7 - MST +Z America/Indiana/Indianapolis -5:44:38 - LMT 1883 N 18 18u +-6 u C%sT 1920 +-6 In C%sT 1942 +-6 u C%sT 1946 +-6 In C%sT 1955 Ap 24 2 +-5 - EST 1957 S 29 2 +-6 - CST 1958 Ap 27 2 +-5 - EST 1969 +-5 u E%sT 1971 +-5 - EST 2006 +-5 u E%sT +Z America/Indiana/Knox -5:46:30 - LMT 1883 N 18 18u +-6 u C%sT 1947 +-6 St C%sT 1962 Ap 29 2 +-5 - EST 1963 O 27 2 +-6 u C%sT 1991 O 27 2 +-5 - EST 2006 Ap 2 2 +-6 u C%sT +Z America/Indiana/Marengo -5:45:23 - LMT 1883 N 18 18u +-6 u C%sT 1951 +-6 Ma C%sT 1961 Ap 30 2 +-5 - EST 1969 +-5 u E%sT 1974 Ja 6 2 +-6 1 CDT 1974 O 27 2 +-5 u E%sT 1976 +-5 - EST 2006 +-5 u E%sT +Z America/Indiana/Petersburg -5:49:7 - LMT 1883 N 18 18u +-6 u C%sT 1955 +-6 Pi C%sT 1965 Ap 25 2 +-5 - EST 1966 O 30 2 +-6 u C%sT 1977 O 30 2 +-5 - EST 2006 Ap 2 2 +-6 u C%sT 2007 N 4 2 +-5 u E%sT +Z America/Indiana/Tell_City -5:47:3 - LMT 1883 N 18 18u +-6 u C%sT 1946 +-6 Pe C%sT 1964 Ap 26 2 +-5 - EST 1967 O 29 2 +-6 u C%sT 1969 Ap 27 2 +-5 u E%sT 1971 +-5 - EST 2006 Ap 2 2 +-6 u C%sT +Z America/Indiana/Vevay -5:40:16 - LMT 1883 N 18 18u +-6 u C%sT 1954 Ap 25 2 +-5 - EST 1969 +-5 u E%sT 1973 +-5 - EST 2006 +-5 u E%sT +Z America/Indiana/Vincennes -5:50:7 - LMT 1883 N 18 18u +-6 u C%sT 1946 +-6 V C%sT 1964 Ap 26 2 +-5 - EST 1969 +-5 u E%sT 1971 +-5 - EST 2006 Ap 2 2 +-6 u C%sT 2007 N 4 2 +-5 u E%sT +Z America/Indiana/Winamac -5:46:25 - LMT 1883 N 18 18u +-6 u C%sT 1946 +-6 Pu C%sT 1961 Ap 30 2 +-5 - EST 1969 +-5 u E%sT 1971 +-5 - EST 2006 Ap 2 2 +-6 u C%sT 2007 Mar 11 2 +-5 u E%sT +Z America/Inuvik 0 - -00 1953 +-8 Y P%sT 1979 Ap lastSu 2 +-7 Y M%sT 1980 +-7 C M%sT +Z America/Iqaluit 0 - -00 1942 Au +-5 Y E%sT 1999 O 31 2 +-6 C C%sT 2000 O 29 2 +-5 C E%sT +Z America/Jamaica -5:7:10 - LMT 1890 +-5:7:10 - KMT 1912 F +-5 - EST 1974 +-5 u E%sT 1984 +-5 - EST +Z America/Juneau 15:2:19 - LMT 1867 O 19 15:33:32 +-8:57:41 - LMT 1900 Au 20 12 +-8 - PST 1942 +-8 u P%sT 1946 +-8 - PST 1969 +-8 u P%sT 1980 Ap 27 2 +-9 u Y%sT 1980 O 26 2 +-8 u P%sT 1983 O 30 2 +-9 u Y%sT 1983 N 30 +-9 u AK%sT +Z America/Kentucky/Louisville -5:43:2 - LMT 1883 N 18 18u +-6 u C%sT 1921 +-6 v C%sT 1942 +-6 u C%sT 1946 +-6 v C%sT 1961 Jul 23 2 +-5 - EST 1968 +-5 u E%sT 1974 Ja 6 2 +-6 1 CDT 1974 O 27 2 +-5 u E%sT +Z America/Kentucky/Monticello -5:39:24 - LMT 1883 N 18 18u +-6 u C%sT 1946 +-6 - CST 1968 +-6 u C%sT 2000 O 29 2 +-5 u E%sT +Z America/La_Paz -4:32:36 - LMT 1890 +-4:32:36 - CMT 1931 O 15 +-4:32:36 1 BST 1932 Mar 21 +-4 - %z +Z America/Lima -5:8:12 - LMT 1890 +-5:8:36 - LMT 1908 Jul 28 +-5 PE %z +Z America/Los_Angeles -7:52:58 - LMT 1883 N 18 20u +-8 u P%sT 1946 +-8 CA P%sT 1967 +-8 u P%sT +Z America/Maceio -2:22:52 - LMT 1914 +-3 B %z 1990 S 17 +-3 - %z 1995 O 13 +-3 B %z 1996 S 4 +-3 - %z 1999 S 30 +-3 B %z 2000 O 22 +-3 - %z 2001 S 13 +-3 B %z 2002 O +-3 - %z +Z America/Managua -5:45:8 - LMT 1890 +-5:45:12 - MMT 1934 Jun 23 +-6 - CST 1973 May +-5 - EST 1975 F 16 +-6 NI C%sT 1992 Ja 1 4 +-5 - EST 1992 S 24 +-6 - CST 1993 +-5 - EST 1997 +-6 NI C%sT +Z America/Manaus -4:0:4 - LMT 1914 +-4 B %z 1988 S 12 +-4 - %z 1993 S 28 +-4 B %z 1994 S 22 +-4 - %z +Z America/Martinique -4:4:20 - LMT 1890 +-4:4:20 - FFMT 1911 May +-4 - AST 1980 Ap 6 +-4 1 ADT 1980 S 28 +-4 - AST +Z America/Matamoros -6:30 - LMT 1922 Ja 1 6u +-6 - CST 1988 +-6 u C%sT 1989 +-6 m C%sT 2010 +-6 u C%sT +Z America/Mazatlan -7:5:40 - LMT 1922 Ja 1 7u +-7 - MST 1927 Jun 10 +-6 - CST 1930 N 15 +-7 m M%sT 1932 Ap +-6 - CST 1942 Ap 24 +-7 - MST 1970 +-7 m M%sT +Z America/Menominee -5:50:27 - LMT 1885 S 18 12 +-6 u C%sT 1946 +-6 Me C%sT 1969 Ap 27 2 +-5 - EST 1973 Ap 29 2 +-6 u C%sT +Z America/Merida -5:58:28 - LMT 1922 Ja 1 6u +-6 - CST 1981 D 26 2 +-5 - EST 1982 N 2 2 +-6 m C%sT +Z America/Metlakatla 15:13:42 - LMT 1867 O 19 15:44:55 +-8:46:18 - LMT 1900 Au 20 12 +-8 - PST 1942 +-8 u P%sT 1946 +-8 - PST 1969 +-8 u P%sT 1983 O 30 2 +-8 - PST 2015 N 1 2 +-9 u AK%sT 2018 N 4 2 +-8 - PST 2019 Ja 20 2 +-9 u AK%sT +Z America/Mexico_City -6:36:36 - LMT 1922 Ja 1 7u +-7 - MST 1927 Jun 10 +-6 - CST 1930 N 15 +-7 m M%sT 1932 Ap +-6 m C%sT 2001 S 30 2 +-6 - CST 2002 F 20 +-6 m C%sT +Z America/Miquelon -3:44:40 - LMT 1911 Jun 15 +-4 - AST 1980 May +-3 - %z 1987 +-3 C %z +Z America/Moncton -4:19:8 - LMT 1883 D 9 +-5 - EST 1902 Jun 15 +-4 C A%sT 1933 +-4 o A%sT 1942 +-4 C A%sT 1946 +-4 o A%sT 1973 +-4 C A%sT 1993 +-4 o A%sT 2007 +-4 C A%sT +Z America/Monterrey -6:41:16 - LMT 1922 Ja 1 6u +-7 - MST 1927 Jun 10 +-6 - CST 1930 N 15 +-7 m M%sT 1932 Ap +-6 - CST 1988 +-6 u C%sT 1989 +-6 m C%sT +Z America/Montevideo -3:44:51 - LMT 1908 Jun 10 +-3:44:51 - MMT 1920 May +-4 - %z 1923 O +-3:30 U %z 1942 D 14 +-3 U %z 1960 +-3 U %z 1968 +-3 U %z 1970 +-3 U %z 1974 +-3 U %z 1974 Mar 10 +-3 U %z 1974 D 22 +-3 U %z +Z America/New_York -4:56:2 - LMT 1883 N 18 17u +-5 u E%sT 1920 +-5 NY E%sT 1942 +-5 u E%sT 1946 +-5 NY E%sT 1967 +-5 u E%sT +Z America/Nome 12:58:22 - LMT 1867 O 19 13:29:35 +-11:1:38 - LMT 1900 Au 20 12 +-11 - NST 1942 +-11 u N%sT 1946 +-11 - NST 1967 Ap +-11 - BST 1969 +-11 u B%sT 1983 O 30 2 +-9 u Y%sT 1983 N 30 +-9 u AK%sT +Z America/Noronha -2:9:40 - LMT 1914 +-2 B %z 1990 S 17 +-2 - %z 1999 S 30 +-2 B %z 2000 O 15 +-2 - %z 2001 S 13 +-2 B %z 2002 O +-2 - %z +Z America/North_Dakota/Beulah -6:47:7 - LMT 1883 N 18 19u +-7 u M%sT 2010 N 7 2 +-6 u C%sT +Z America/North_Dakota/Center -6:45:12 - LMT 1883 N 18 19u +-7 u M%sT 1992 O 25 2 +-6 u C%sT +Z America/North_Dakota/New_Salem -6:45:39 - LMT 1883 N 18 19u +-7 u M%sT 2003 O 26 2 +-6 u C%sT +Z America/Nuuk -3:26:56 - LMT 1916 Jul 28 +-3 - %z 1980 Ap 6 2 +-3 E %z 2023 Mar 26 1u +-2 - %z 2023 O 29 1u +-2 E %z +Z America/Ojinaga -6:57:40 - LMT 1922 Ja 1 7u +-7 - MST 1927 Jun 10 +-6 - CST 1930 N 15 +-7 m M%sT 1932 Ap +-6 - CST 1996 +-6 m C%sT 1998 +-6 - CST 1998 Ap Su>=1 3 +-7 m M%sT 2010 +-7 u M%sT 2022 O 30 2 +-6 - CST 2022 N 30 +-6 u C%sT +Z America/Panama -5:18:8 - LMT 1890 +-5:19:36 - CMT 1908 Ap 22 +-5 - EST +Z America/Paramaribo -3:40:40 - LMT 1911 +-3:40:52 - PMT 1935 +-3:40:36 - PMT 1945 O +-3:30 - %z 1984 O +-3 - %z +Z America/Phoenix -7:28:18 - LMT 1883 N 18 19u +-7 u M%sT 1944 Ja 1 0:1 +-7 - MST 1944 Ap 1 0:1 +-7 u M%sT 1944 O 1 0:1 +-7 - MST 1967 +-7 u M%sT 1968 Mar 21 +-7 - MST +Z America/Port-au-Prince -4:49:20 - LMT 1890 +-4:49 - PPMT 1917 Ja 24 12 +-5 HT E%sT +Z America/Porto_Velho -4:15:36 - LMT 1914 +-4 B %z 1988 S 12 +-4 - %z +Z America/Puerto_Rico -4:24:25 - LMT 1899 Mar 28 12 +-4 - AST 1942 May 3 +-4 u A%sT 1946 +-4 - AST +Z America/Punta_Arenas -4:43:40 - LMT 1890 +-4:42:45 - SMT 1910 Ja 10 +-5 - %z 1916 Jul +-4:42:45 - SMT 1918 S 10 +-4 - %z 1919 Jul +-4:42:45 - SMT 1927 S +-5 x %z 1932 S +-4 - %z 1942 Jun +-5 - %z 1942 Au +-4 - %z 1946 Au 28 24 +-5 1 %z 1947 Mar 31 24 +-5 - %z 1947 May 21 23 +-4 x %z 2016 D 4 +-3 - %z +Z America/Rankin_Inlet 0 - -00 1957 +-6 Y C%sT 2000 O 29 2 +-5 - EST 2001 Ap 1 3 +-6 C C%sT +Z America/Recife -2:19:36 - LMT 1914 +-3 B %z 1990 S 17 +-3 - %z 1999 S 30 +-3 B %z 2000 O 15 +-3 - %z 2001 S 13 +-3 B %z 2002 O +-3 - %z +Z America/Regina -6:58:36 - LMT 1905 S +-7 r M%sT 1960 Ap lastSu 2 +-6 - CST +Z America/Resolute 0 - -00 1947 Au 31 +-6 Y C%sT 2000 O 29 2 +-5 - EST 2001 Ap 1 3 +-6 C C%sT 2006 O 29 2 +-5 - EST 2007 Mar 11 3 +-6 C C%sT +Z America/Rio_Branco -4:31:12 - LMT 1914 +-5 B %z 1988 S 12 +-5 - %z 2008 Jun 24 +-4 - %z 2013 N 10 +-5 - %z +Z America/Santarem -3:38:48 - LMT 1914 +-4 B %z 1988 S 12 +-4 - %z 2008 Jun 24 +-3 - %z +Z America/Santiago -4:42:45 - LMT 1890 +-4:42:45 - SMT 1910 Ja 10 +-5 - %z 1916 Jul +-4:42:45 - SMT 1918 S 10 +-4 - %z 1919 Jul +-4:42:45 - SMT 1927 S +-5 x %z 1932 S +-4 - %z 1942 Jun +-5 - %z 1942 Au +-4 - %z 1946 Jul 14 24 +-4 1 %z 1946 Au 28 24 +-5 1 %z 1947 Mar 31 24 +-5 - %z 1947 May 21 23 +-4 x %z +Z America/Santo_Domingo -4:39:36 - LMT 1890 +-4:40 - SDMT 1933 Ap 1 12 +-5 DO %s 1974 O 27 +-4 - AST 2000 O 29 2 +-5 u E%sT 2000 D 3 1 +-4 - AST +Z America/Sao_Paulo -3:6:28 - LMT 1914 +-3 B %z 1963 O 23 +-3 1 %z 1964 +-3 B %z +Z America/Scoresbysund -1:27:52 - LMT 1916 Jul 28 +-2 - %z 1980 Ap 6 2 +-2 c %z 1981 Mar 29 +-1 E %z 2024 Mar 31 +-2 E %z +Z America/Sitka 14:58:47 - LMT 1867 O 19 15:30 +-9:1:13 - LMT 1900 Au 20 12 +-8 - PST 1942 +-8 u P%sT 1946 +-8 - PST 1969 +-8 u P%sT 1983 O 30 2 +-9 u Y%sT 1983 N 30 +-9 u AK%sT +Z America/St_Johns -3:30:52 - LMT 1884 +-3:30:52 j N%sT 1918 +-3:30:52 C N%sT 1919 +-3:30:52 j N%sT 1935 Mar 30 +-3:30 j N%sT 1942 May 11 +-3:30 C N%sT 1946 +-3:30 j N%sT 2011 N +-3:30 C N%sT +Z America/Swift_Current -7:11:20 - LMT 1905 S +-7 C M%sT 1946 Ap lastSu 2 +-7 r M%sT 1950 +-7 Sw M%sT 1972 Ap lastSu 2 +-6 - CST +Z America/Tegucigalpa -5:48:52 - LMT 1921 Ap +-6 HN C%sT +Z America/Thule -4:35:8 - LMT 1916 Jul 28 +-4 Th A%sT +Z America/Tijuana -7:48:4 - LMT 1922 Ja 1 7u +-7 - MST 1924 +-8 - PST 1927 Jun 10 +-7 - MST 1930 N 15 +-8 - PST 1931 Ap +-8 1 PDT 1931 S 30 +-8 - PST 1942 Ap 24 +-8 1 PWT 1945 Au 14 23u +-8 1 PPT 1945 N 15 +-8 - PST 1948 Ap 5 +-8 1 PDT 1949 Ja 14 +-8 - PST 1950 May +-8 1 PDT 1950 S 24 +-8 - PST 1951 Ap 29 2 +-8 1 PDT 1951 S 30 2 +-8 - PST 1952 Ap 27 2 +-8 1 PDT 1952 S 28 2 +-8 - PST 1954 +-8 CA P%sT 1961 +-8 - PST 1976 +-8 u P%sT 1996 +-8 m P%sT 2001 +-8 u P%sT 2002 F 20 +-8 m P%sT 2010 +-8 u P%sT +Z America/Toronto -5:17:32 - LMT 1895 +-5 C E%sT 1919 +-5 t E%sT 1942 F 9 2s +-5 C E%sT 1946 +-5 t E%sT 1974 +-5 C E%sT +Z America/Vancouver -8:12:28 - LMT 1884 +-8 Va P%sT 1987 +-8 C P%sT +Z America/Whitehorse -9:0:12 - LMT 1900 Au 20 +-9 Y Y%sT 1965 +-9 Yu Y%sT 1966 F 27 +-8 - PST 1980 +-8 C P%sT 2020 N +-7 - MST +Z America/Winnipeg -6:28:36 - LMT 1887 Jul 16 +-6 W C%sT 2006 +-6 C C%sT +Z America/Yakutat 14:41:5 - LMT 1867 O 19 15:12:18 +-9:18:55 - LMT 1900 Au 20 12 +-9 - YST 1942 +-9 u Y%sT 1946 +-9 - YST 1969 +-9 u Y%sT 1983 N 30 +-9 u AK%sT +Z Antarctica/Casey 0 - -00 1969 +8 - %z 2009 O 18 2 +11 - %z 2010 Mar 5 2 +8 - %z 2011 O 28 2 +11 - %z 2012 F 21 17u +8 - %z 2016 O 22 +11 - %z 2018 Mar 11 4 +8 - %z 2018 O 7 4 +11 - %z 2019 Mar 17 3 +8 - %z 2019 O 4 3 +11 - %z 2020 Mar 8 3 +8 - %z 2020 O 4 0:1 +11 - %z 2021 Mar 14 +8 - %z 2021 O 3 0:1 +11 - %z 2022 Mar 13 +8 - %z 2022 O 2 0:1 +11 - %z 2023 Mar 9 3 +8 - %z +Z Antarctica/Davis 0 - -00 1957 Ja 13 +7 - %z 1964 N +0 - -00 1969 F +7 - %z 2009 O 18 2 +5 - %z 2010 Mar 10 20u +7 - %z 2011 O 28 2 +5 - %z 2012 F 21 20u +7 - %z +Z Antarctica/Macquarie 0 - -00 1899 N +10 - AEST 1916 O 1 2 +10 1 AEDT 1917 F +10 AU AE%sT 1919 Ap 1 0s +0 - -00 1948 Mar 25 +10 AU AE%sT 1967 +10 AT AE%sT 2010 +10 1 AEDT 2011 +10 AT AE%sT +Z Antarctica/Mawson 0 - -00 1954 F 13 +6 - %z 2009 O 18 2 +5 - %z +Z Antarctica/Palmer 0 - -00 1965 +-4 A %z 1969 O 5 +-3 A %z 1982 May +-4 x %z 2016 D 4 +-3 - %z +Z Antarctica/Rothera 0 - -00 1976 D +-3 - %z +Z Antarctica/Troll 0 - -00 2005 F 12 +0 Tr %s +Z Antarctica/Vostok 0 - -00 1957 D 16 +7 - %z 1994 F +0 - -00 1994 N +7 - %z 2023 D 18 2 +5 - %z +Z Asia/Almaty 5:7:48 - LMT 1924 May 2 +5 - %z 1930 Jun 21 +6 R %z 1991 Mar 31 2s +5 R %z 1992 Ja 19 2s +6 R %z 2004 O 31 2s +6 - %z 2024 Mar +5 - %z +Z Asia/Amman 2:23:44 - LMT 1931 +2 J EE%sT 2022 O 28 0s +3 - %z +Z Asia/Anadyr 11:49:56 - LMT 1924 May 2 +12 - %z 1930 Jun 21 +13 R %z 1982 Ap 1 0s +12 R %z 1991 Mar 31 2s +11 R %z 1992 Ja 19 2s +12 R %z 2010 Mar 28 2s +11 R %z 2011 Mar 27 2s +12 - %z +Z Asia/Aqtau 3:21:4 - LMT 1924 May 2 +4 - %z 1930 Jun 21 +5 - %z 1981 O +6 - %z 1982 Ap +5 R %z 1991 Mar 31 2s +4 R %z 1992 Ja 19 2s +5 R %z 1994 S 25 2s +4 R %z 2004 O 31 2s +5 - %z +Z Asia/Aqtobe 3:48:40 - LMT 1924 May 2 +4 - %z 1930 Jun 21 +5 - %z 1981 Ap +5 1 %z 1981 O +6 - %z 1982 Ap +5 R %z 1991 Mar 31 2s +4 R %z 1992 Ja 19 2s +5 R %z 2004 O 31 2s +5 - %z +Z Asia/Ashgabat 3:53:32 - LMT 1924 May 2 +4 - %z 1930 Jun 21 +5 R %z 1991 Mar 31 2 +4 R %z 1992 Ja 19 2 +5 - %z +Z Asia/Atyrau 3:27:44 - LMT 1924 May 2 +3 - %z 1930 Jun 21 +5 - %z 1981 O +6 - %z 1982 Ap +5 R %z 1991 Mar 31 2s +4 R %z 1992 Ja 19 2s +5 R %z 1999 Mar 28 2s +4 R %z 2004 O 31 2s +5 - %z +Z Asia/Baghdad 2:57:40 - LMT 1890 +2:57:36 - BMT 1918 +3 - %z 1982 May +3 IQ %z +Z Asia/Baku 3:19:24 - LMT 1924 May 2 +3 - %z 1957 Mar +4 R %z 1991 Mar 31 2s +3 R %z 1992 S lastSu 2s +4 - %z 1996 +4 E %z 1997 +4 AZ %z +Z Asia/Bangkok 6:42:4 - LMT 1880 +6:42:4 - BMT 1920 Ap +7 - %z +Z Asia/Barnaul 5:35 - LMT 1919 D 10 +6 - %z 1930 Jun 21 +7 R %z 1991 Mar 31 2s +6 R %z 1992 Ja 19 2s +7 R %z 1995 May 28 +6 R %z 2011 Mar 27 2s +7 - %z 2014 O 26 2s +6 - %z 2016 Mar 27 2s +7 - %z +Z Asia/Beirut 2:22 - LMT 1880 +2 l EE%sT +Z Asia/Bishkek 4:58:24 - LMT 1924 May 2 +5 - %z 1930 Jun 21 +6 R %z 1991 Mar 31 2s +5 R %z 1991 Au 31 2 +5 KG %z 2005 Au 12 +6 - %z +Z Asia/Chita 7:33:52 - LMT 1919 D 15 +8 - %z 1930 Jun 21 +9 R %z 1991 Mar 31 2s +8 R %z 1992 Ja 19 2s +9 R %z 2011 Mar 27 2s +10 - %z 2014 O 26 2s +8 - %z 2016 Mar 27 2 +9 - %z +Z Asia/Colombo 5:19:24 - LMT 1880 +5:19:32 - MMT 1906 +5:30 - %z 1942 Ja 5 +5:30 0:30 %z 1942 S +5:30 1 %z 1945 O 16 2 +5:30 - %z 1996 May 25 +6:30 - %z 1996 O 26 0:30 +6 - %z 2006 Ap 15 0:30 +5:30 - %z +Z Asia/Damascus 2:25:12 - LMT 1920 +2 S EE%sT 2022 O 28 +3 - %z +Z Asia/Dhaka 6:1:40 - LMT 1890 +5:53:20 - HMT 1941 O +6:30 - %z 1942 May 15 +5:30 - %z 1942 S +6:30 - %z 1951 S 30 +6 - %z 2009 +6 BD %z +Z Asia/Dili 8:22:20 - LMT 1911 D 31 16u +8 - %z 1942 F 21 23 +9 - %z 1976 May 3 +8 - %z 2000 S 17 +9 - %z +Z Asia/Dubai 3:41:12 - LMT 1920 +4 - %z +Z Asia/Dushanbe 4:35:12 - LMT 1924 May 2 +5 - %z 1930 Jun 21 +6 R %z 1991 Mar 31 2s +5 1 %z 1991 S 9 2s +5 - %z +Z Asia/Famagusta 2:15:48 - LMT 1921 N 14 +2 CY EE%sT 1998 S +2 E EE%sT 2016 S 8 +3 - %z 2017 O 29 1u +2 E EE%sT +Z Asia/Gaza 2:17:52 - LMT 1900 O +2 Z EET/EEST 1948 May 15 +2 K EE%sT 1967 Jun 5 +2 Z I%sT 1996 +2 J EE%sT 1999 +2 P EE%sT 2008 Au 29 +2 - EET 2008 S +2 P EE%sT 2010 +2 - EET 2010 Mar 27 0:1 +2 P EE%sT 2011 Au +2 - EET 2012 +2 P EE%sT +Z Asia/Hebron 2:20:23 - LMT 1900 O +2 Z EET/EEST 1948 May 15 +2 K EE%sT 1967 Jun 5 +2 Z I%sT 1996 +2 J EE%sT 1999 +2 P EE%sT +Z Asia/Ho_Chi_Minh 7:6:30 - LMT 1906 Jul +7:6:30 - PLMT 1911 May +7 - %z 1942 D 31 23 +8 - %z 1945 Mar 14 23 +9 - %z 1945 S 1 24 +7 - %z 1947 Ap +8 - %z 1955 Jul 1 1 +7 - %z 1959 D 31 23 +8 - %z 1975 Jun 13 +7 - %z +Z Asia/Hong_Kong 7:36:42 - LMT 1904 O 29 17u +8 - HKT 1941 Jun 15 3 +8 1 HKST 1941 O 1 4 +8 0:30 HKWT 1941 D 25 +9 - JST 1945 N 18 2 +8 HK HK%sT +Z Asia/Hovd 6:6:36 - LMT 1905 Au +6 - %z 1978 +7 X %z +Z Asia/Irkutsk 6:57:5 - LMT 1880 +6:57:5 - IMT 1920 Ja 25 +7 - %z 1930 Jun 21 +8 R %z 1991 Mar 31 2s +7 R %z 1992 Ja 19 2s +8 R %z 2011 Mar 27 2s +9 - %z 2014 O 26 2s +8 - %z +Z Asia/Jakarta 7:7:12 - LMT 1867 Au 10 +7:7:12 - BMT 1923 D 31 16:40u +7:20 - %z 1932 N +7:30 - %z 1942 Mar 23 +9 - %z 1945 S 23 +7:30 - %z 1948 May +8 - %z 1950 May +7:30 - %z 1964 +7 - WIB +Z Asia/Jayapura 9:22:48 - LMT 1932 N +9 - %z 1944 S +9:30 - %z 1964 +9 - WIT +Z Asia/Jerusalem 2:20:54 - LMT 1880 +2:20:40 - JMT 1918 +2 Z I%sT +Z Asia/Kabul 4:36:48 - LMT 1890 +4 - %z 1945 +4:30 - %z +Z Asia/Kamchatka 10:34:36 - LMT 1922 N 10 +11 - %z 1930 Jun 21 +12 R %z 1991 Mar 31 2s +11 R %z 1992 Ja 19 2s +12 R %z 2010 Mar 28 2s +11 R %z 2011 Mar 27 2s +12 - %z +Z Asia/Karachi 4:28:12 - LMT 1907 +5:30 - %z 1942 S +5:30 1 %z 1945 O 15 +5:30 - %z 1951 S 30 +5 - %z 1971 Mar 26 +5 PK PK%sT +Z Asia/Kathmandu 5:41:16 - LMT 1920 +5:30 - %z 1986 +5:45 - %z +Z Asia/Khandyga 9:2:13 - LMT 1919 D 15 +8 - %z 1930 Jun 21 +9 R %z 1991 Mar 31 2s +8 R %z 1992 Ja 19 2s +9 R %z 2004 +10 R %z 2011 Mar 27 2s +11 - %z 2011 S 13 0s +10 - %z 2014 O 26 2s +9 - %z +Z Asia/Kolkata 5:53:28 - LMT 1854 Jun 28 +5:53:20 - HMT 1870 +5:21:10 - MMT 1906 +5:30 - IST 1941 O +5:30 1 %z 1942 May 15 +5:30 - IST 1942 S +5:30 1 %z 1945 O 15 +5:30 - IST +Z Asia/Krasnoyarsk 6:11:26 - LMT 1920 Ja 6 +6 - %z 1930 Jun 21 +7 R %z 1991 Mar 31 2s +6 R %z 1992 Ja 19 2s +7 R %z 2011 Mar 27 2s +8 - %z 2014 O 26 2s +7 - %z +Z Asia/Kuching 7:21:20 - LMT 1926 Mar +7:30 - %z 1933 +8 NB %z 1942 F 16 +9 - %z 1945 S 12 +8 - %z +Z Asia/Macau 7:34:10 - LMT 1904 O 30 +8 - CST 1941 D 21 23 +9 _ %z 1945 S 30 24 +8 _ C%sT +Z Asia/Magadan 10:3:12 - LMT 1924 May 2 +10 - %z 1930 Jun 21 +11 R %z 1991 Mar 31 2s +10 R %z 1992 Ja 19 2s +11 R %z 2011 Mar 27 2s +12 - %z 2014 O 26 2s +10 - %z 2016 Ap 24 2s +11 - %z +Z Asia/Makassar 7:57:36 - LMT 1920 +7:57:36 - MMT 1932 N +8 - %z 1942 F 9 +9 - %z 1945 S 23 +8 - WITA +Z Asia/Manila -15:56:8 - LMT 1844 D 31 +8:3:52 - LMT 1899 S 6 4u +8 PH P%sT 1942 F 11 24 +9 - JST 1945 Mar 4 +8 PH P%sT +Z Asia/Nicosia 2:13:28 - LMT 1921 N 14 +2 CY EE%sT 1998 S +2 E EE%sT +Z Asia/Novokuznetsk 5:48:48 - LMT 1924 May +6 - %z 1930 Jun 21 +7 R %z 1991 Mar 31 2s +6 R %z 1992 Ja 19 2s +7 R %z 2010 Mar 28 2s +6 R %z 2011 Mar 27 2s +7 - %z +Z Asia/Novosibirsk 5:31:40 - LMT 1919 D 14 6 +6 - %z 1930 Jun 21 +7 R %z 1991 Mar 31 2s +6 R %z 1992 Ja 19 2s +7 R %z 1993 May 23 +6 R %z 2011 Mar 27 2s +7 - %z 2014 O 26 2s +6 - %z 2016 Jul 24 2s +7 - %z +Z Asia/Omsk 4:53:30 - LMT 1919 N 14 +5 - %z 1930 Jun 21 +6 R %z 1991 Mar 31 2s +5 R %z 1992 Ja 19 2s +6 R %z 2011 Mar 27 2s +7 - %z 2014 O 26 2s +6 - %z +Z Asia/Oral 3:25:24 - LMT 1924 May 2 +3 - %z 1930 Jun 21 +5 - %z 1981 Ap +5 1 %z 1981 O +6 - %z 1982 Ap +5 R %z 1989 Mar 26 2s +4 R %z 1992 Ja 19 2s +5 R %z 1992 Mar 29 2s +4 R %z 2004 O 31 2s +5 - %z +Z Asia/Pontianak 7:17:20 - LMT 1908 May +7:17:20 - PMT 1932 N +7:30 - %z 1942 Ja 29 +9 - %z 1945 S 23 +7:30 - %z 1948 May +8 - %z 1950 May +7:30 - %z 1964 +8 - WITA 1988 +7 - WIB +Z Asia/Pyongyang 8:23 - LMT 1908 Ap +8:30 - KST 1912 +9 - JST 1945 Au 24 +9 - KST 2015 Au 15 +8:30 - KST 2018 May 4 23:30 +9 - KST +Z Asia/Qatar 3:26:8 - LMT 1920 +4 - %z 1972 Jun +3 - %z +Z Asia/Qostanay 4:14:28 - LMT 1924 May 2 +4 - %z 1930 Jun 21 +5 - %z 1981 Ap +5 1 %z 1981 O +6 - %z 1982 Ap +5 R %z 1991 Mar 31 2s +4 R %z 1992 Ja 19 2s +5 R %z 2004 O 31 2s +6 - %z 2024 Mar +5 - %z +Z Asia/Qyzylorda 4:21:52 - LMT 1924 May 2 +4 - %z 1930 Jun 21 +5 - %z 1981 Ap +5 1 %z 1981 O +6 - %z 1982 Ap +5 R %z 1991 Mar 31 2s +4 R %z 1991 S 29 2s +5 R %z 1992 Ja 19 2s +6 R %z 1992 Mar 29 2s +5 R %z 2004 O 31 2s +6 - %z 2018 D 21 +5 - %z +Z Asia/Riyadh 3:6:52 - LMT 1947 Mar 14 +3 - %z +Z Asia/Sakhalin 9:30:48 - LMT 1905 Au 23 +9 - %z 1945 Au 25 +11 R %z 1991 Mar 31 2s +10 R %z 1992 Ja 19 2s +11 R %z 1997 Mar lastSu 2s +10 R %z 2011 Mar 27 2s +11 - %z 2014 O 26 2s +10 - %z 2016 Mar 27 2s +11 - %z +Z Asia/Samarkand 4:27:53 - LMT 1924 May 2 +4 - %z 1930 Jun 21 +5 - %z 1981 Ap +5 1 %z 1981 O +6 - %z 1982 Ap +5 R %z 1992 +5 - %z +Z Asia/Seoul 8:27:52 - LMT 1908 Ap +8:30 - KST 1912 +9 - JST 1945 S 8 +9 KR K%sT 1954 Mar 21 +8:30 KR K%sT 1961 Au 10 +9 KR K%sT +Z Asia/Shanghai 8:5:43 - LMT 1901 +8 Sh C%sT 1949 May 28 +8 CN C%sT +Z Asia/Singapore 6:55:25 - LMT 1901 +6:55:25 - SMT 1905 Jun +7 - %z 1933 +7 0:20 %z 1936 +7:20 - %z 1941 S +7:30 - %z 1942 F 16 +9 - %z 1945 S 12 +7:30 - %z 1981 D 31 16u +8 - %z +Z Asia/Srednekolymsk 10:14:52 - LMT 1924 May 2 +10 - %z 1930 Jun 21 +11 R %z 1991 Mar 31 2s +10 R %z 1992 Ja 19 2s +11 R %z 2011 Mar 27 2s +12 - %z 2014 O 26 2s +11 - %z +Z Asia/Taipei 8:6 - LMT 1896 +8 - CST 1937 O +9 - JST 1945 S 21 1 +8 f C%sT +Z Asia/Tashkent 4:37:11 - LMT 1924 May 2 +5 - %z 1930 Jun 21 +6 R %z 1991 Mar 31 2 +5 R %z 1992 +5 - %z +Z Asia/Tbilisi 2:59:11 - LMT 1880 +2:59:11 - TBMT 1924 May 2 +3 - %z 1957 Mar +4 R %z 1991 Mar 31 2s +3 R %z 1992 +3 e %z 1994 S lastSu +4 e %z 1996 O lastSu +4 1 %z 1997 Mar lastSu +4 e %z 2004 Jun 27 +3 R %z 2005 Mar lastSu 2 +4 - %z +Z Asia/Tehran 3:25:44 - LMT 1916 +3:25:44 - TMT 1935 Jun 13 +3:30 i %z 1977 O 20 24 +4 i %z 1979 +3:30 i %z +Z Asia/Thimphu 5:58:36 - LMT 1947 Au 15 +5:30 - %z 1987 O +6 - %z +Z Asia/Tokyo 9:18:59 - LMT 1887 D 31 15u +9 JP J%sT +Z Asia/Tomsk 5:39:51 - LMT 1919 D 22 +6 - %z 1930 Jun 21 +7 R %z 1991 Mar 31 2s +6 R %z 1992 Ja 19 2s +7 R %z 2002 May 1 3 +6 R %z 2011 Mar 27 2s +7 - %z 2014 O 26 2s +6 - %z 2016 May 29 2s +7 - %z +Z Asia/Ulaanbaatar 7:7:32 - LMT 1905 Au +7 - %z 1978 +8 X %z +Z Asia/Urumqi 5:50:20 - LMT 1928 +6 - %z +Z Asia/Ust-Nera 9:32:54 - LMT 1919 D 15 +8 - %z 1930 Jun 21 +9 R %z 1981 Ap +11 R %z 1991 Mar 31 2s +10 R %z 1992 Ja 19 2s +11 R %z 2011 Mar 27 2s +12 - %z 2011 S 13 0s +11 - %z 2014 O 26 2s +10 - %z +Z Asia/Vladivostok 8:47:31 - LMT 1922 N 15 +9 - %z 1930 Jun 21 +10 R %z 1991 Mar 31 2s +9 R %z 1992 Ja 19 2s +10 R %z 2011 Mar 27 2s +11 - %z 2014 O 26 2s +10 - %z +Z Asia/Yakutsk 8:38:58 - LMT 1919 D 15 +8 - %z 1930 Jun 21 +9 R %z 1991 Mar 31 2s +8 R %z 1992 Ja 19 2s +9 R %z 2011 Mar 27 2s +10 - %z 2014 O 26 2s +9 - %z +Z Asia/Yangon 6:24:47 - LMT 1880 +6:24:47 - RMT 1920 +6:30 - %z 1942 May +9 - %z 1945 May 3 +6:30 - %z +Z Asia/Yekaterinburg 4:2:33 - LMT 1916 Jul 3 +3:45:5 - PMT 1919 Jul 15 4 +4 - %z 1930 Jun 21 +5 R %z 1991 Mar 31 2s +4 R %z 1992 Ja 19 2s +5 R %z 2011 Mar 27 2s +6 - %z 2014 O 26 2s +5 - %z +Z Asia/Yerevan 2:58 - LMT 1924 May 2 +3 - %z 1957 Mar +4 R %z 1991 Mar 31 2s +3 R %z 1995 S 24 2s +4 - %z 1997 +4 R %z 2011 +4 AM %z +Z Atlantic/Azores -1:42:40 - LMT 1884 +-1:54:32 - HMT 1912 Ja 1 2u +-2 p %z 1966 O 2 2s +-1 - %z 1982 Mar 28 0s +-1 p %z 1986 +-1 E %z 1992 D 27 1s +0 E WE%sT 1993 Jun 17 1u +-1 E %z +Z Atlantic/Bermuda -4:19:18 - LMT 1890 +-4:19:18 Be BMT/BST 1930 Ja 1 2 +-4 Be A%sT 1974 Ap 28 2 +-4 C A%sT 1976 +-4 u A%sT +Z Atlantic/Canary -1:1:36 - LMT 1922 Mar +-1 - %z 1946 S 30 1 +0 - WET 1980 Ap 6 0s +0 1 WEST 1980 S 28 1u +0 E WE%sT +Z Atlantic/Cape_Verde -1:34:4 - LMT 1912 Ja 1 2u +-2 - %z 1942 S +-2 1 %z 1945 O 15 +-2 - %z 1975 N 25 2 +-1 - %z +Z Atlantic/Faroe -0:27:4 - LMT 1908 Ja 11 +0 - WET 1981 +0 E WE%sT +Z Atlantic/Madeira -1:7:36 - LMT 1884 +-1:7:36 - FMT 1912 Ja 1 1u +-1 p %z 1966 O 2 2s +0 - WET 1982 Ap 4 +0 p WE%sT 1986 Jul 31 +0 E WE%sT +Z Atlantic/South_Georgia -2:26:8 - LMT 1890 +-2 - %z +Z Atlantic/Stanley -3:51:24 - LMT 1890 +-3:51:24 - SMT 1912 Mar 12 +-4 FK %z 1983 May +-3 FK %z 1985 S 15 +-4 FK %z 2010 S 5 2 +-3 - %z +Z Australia/Adelaide 9:14:20 - LMT 1895 F +9 - ACST 1899 May +9:30 AU AC%sT 1971 +9:30 AS AC%sT +Z Australia/Brisbane 10:12:8 - LMT 1895 +10 AU AE%sT 1971 +10 AQ AE%sT +Z Australia/Broken_Hill 9:25:48 - LMT 1895 F +10 - AEST 1896 Au 23 +9 - ACST 1899 May +9:30 AU AC%sT 1971 +9:30 AN AC%sT 2000 +9:30 AS AC%sT +Z Australia/Darwin 8:43:20 - LMT 1895 F +9 - ACST 1899 May +9:30 AU AC%sT +Z Australia/Eucla 8:35:28 - LMT 1895 D +8:45 AU %z 1943 Jul +8:45 AW %z +Z Australia/Hobart 9:49:16 - LMT 1895 S +10 AT AE%sT 1919 O 24 +10 AU AE%sT 1967 +10 AT AE%sT +Z Australia/Lindeman 9:55:56 - LMT 1895 +10 AU AE%sT 1971 +10 AQ AE%sT 1992 Jul +10 Ho AE%sT +Z Australia/Lord_Howe 10:36:20 - LMT 1895 F +10 - AEST 1981 Mar +10:30 LH %z 1985 Jul +10:30 LH %z +Z Australia/Melbourne 9:39:52 - LMT 1895 F +10 AU AE%sT 1971 +10 AV AE%sT +Z Australia/Perth 7:43:24 - LMT 1895 D +8 AU AW%sT 1943 Jul +8 AW AW%sT +Z Australia/Sydney 10:4:52 - LMT 1895 F +10 AU AE%sT 1971 +10 AN AE%sT +Z Etc/GMT 0 - GMT +Z Etc/GMT+1 -1 - %z +Z Etc/GMT+10 -10 - %z +Z Etc/GMT+11 -11 - %z +Z Etc/GMT+12 -12 - %z +Z Etc/GMT+2 -2 - %z +Z Etc/GMT+3 -3 - %z +Z Etc/GMT+4 -4 - %z +Z Etc/GMT+5 -5 - %z +Z Etc/GMT+6 -6 - %z +Z Etc/GMT+7 -7 - %z +Z Etc/GMT+8 -8 - %z +Z Etc/GMT+9 -9 - %z +Z Etc/GMT-1 1 - %z +Z Etc/GMT-10 10 - %z +Z Etc/GMT-11 11 - %z +Z Etc/GMT-12 12 - %z +Z Etc/GMT-13 13 - %z +Z Etc/GMT-14 14 - %z +Z Etc/GMT-2 2 - %z +Z Etc/GMT-3 3 - %z +Z Etc/GMT-4 4 - %z +Z Etc/GMT-5 5 - %z +Z Etc/GMT-6 6 - %z +Z Etc/GMT-7 7 - %z +Z Etc/GMT-8 8 - %z +Z Etc/GMT-9 9 - %z +Z Etc/UTC 0 - UTC +Z Europe/Andorra 0:6:4 - LMT 1901 +0 - WET 1946 S 30 +1 - CET 1985 Mar 31 2 +1 E CE%sT +Z Europe/Astrakhan 3:12:12 - LMT 1924 May +3 - %z 1930 Jun 21 +4 R %z 1989 Mar 26 2s +3 R %z 1991 Mar 31 2s +4 - %z 1992 Mar 29 2s +3 R %z 2011 Mar 27 2s +4 - %z 2014 O 26 2s +3 - %z 2016 Mar 27 2s +4 - %z +Z Europe/Athens 1:34:52 - LMT 1895 S 14 +1:34:52 - AMT 1916 Jul 28 0:1 +2 g EE%sT 1941 Ap 30 +1 g CE%sT 1944 Ap 4 +2 g EE%sT 1981 +2 E EE%sT +Z Europe/Belgrade 1:22 - LMT 1884 +1 - CET 1941 Ap 18 23 +1 c CE%sT 1945 +1 - CET 1945 May 8 2s +1 1 CEST 1945 S 16 2s +1 - CET 1982 N 27 +1 E CE%sT +Z Europe/Berlin 0:53:28 - LMT 1893 Ap +1 c CE%sT 1945 May 24 2 +1 So CE%sT 1946 +1 DE CE%sT 1980 +1 E CE%sT +Z Europe/Brussels 0:17:30 - LMT 1880 +0:17:30 - BMT 1892 May 1 0:17:30 +0 - WET 1914 N 8 +1 - CET 1916 May +1 c CE%sT 1918 N 11 11u +0 b WE%sT 1940 May 20 2s +1 c CE%sT 1944 S 3 +1 b CE%sT 1977 +1 E CE%sT +Z Europe/Bucharest 1:44:24 - LMT 1891 O +1:44:24 - BMT 1931 Jul 24 +2 z EE%sT 1981 Mar 29 2s +2 c EE%sT 1991 +2 z EE%sT 1994 +2 e EE%sT 1997 +2 E EE%sT +Z Europe/Budapest 1:16:20 - LMT 1890 N +1 c CE%sT 1918 +1 h CE%sT 1941 Ap 7 23 +1 c CE%sT 1945 +1 h CE%sT 1984 +1 E CE%sT +Z Europe/Chisinau 1:55:20 - LMT 1880 +1:55 - CMT 1918 F 15 +1:44:24 - BMT 1931 Jul 24 +2 z EE%sT 1940 Au 15 +2 1 EEST 1941 Jul 17 +1 c CE%sT 1944 Au 24 +3 R MSK/MSD 1990 May 6 2 +2 R EE%sT 1992 +2 e EE%sT 1997 +2 MD EE%sT +Z Europe/Dublin -0:25:21 - LMT 1880 Au 2 +-0:25:21 - DMT 1916 May 21 2s +-0:25:21 1 IST 1916 O 1 2s +0 G %s 1921 D 6 +0 G GMT/IST 1940 F 25 2s +0 1 IST 1946 O 6 2s +0 - GMT 1947 Mar 16 2s +0 1 IST 1947 N 2 2s +0 - GMT 1948 Ap 18 2s +0 G GMT/IST 1968 O 27 +1 IE IST/GMT +Z Europe/Gibraltar -0:21:24 - LMT 1880 Au 2 +0 G %s 1957 Ap 14 2 +1 - CET 1982 +1 E CE%sT +Z Europe/Helsinki 1:39:49 - LMT 1878 May 31 +1:39:49 - HMT 1921 May +2 FI EE%sT 1983 +2 E EE%sT +Z Europe/Istanbul 1:55:52 - LMT 1880 +1:56:56 - IMT 1910 O +2 T EE%sT 1978 Jun 29 +3 T %z 1984 N 1 2 +2 T EE%sT 2007 +2 E EE%sT 2011 Mar 27 1u +2 - EET 2011 Mar 28 1u +2 E EE%sT 2014 Mar 30 1u +2 - EET 2014 Mar 31 1u +2 E EE%sT 2015 O 25 1u +2 1 EEST 2015 N 8 1u +2 E EE%sT 2016 S 7 +3 - %z +Z Europe/Kaliningrad 1:22 - LMT 1893 Ap +1 c CE%sT 1945 Ap 10 +2 O EE%sT 1946 Ap 7 +3 R MSK/MSD 1989 Mar 26 2s +2 R EE%sT 2011 Mar 27 2s +3 - %z 2014 O 26 2s +2 - EET +Z Europe/Kirov 3:18:48 - LMT 1919 Jul 1 0u +3 - %z 1930 Jun 21 +4 R %z 1989 Mar 26 2s +3 R MSK/MSD 1991 Mar 31 2s +4 - %z 1992 Mar 29 2s +3 R MSK/MSD 2011 Mar 27 2s +4 - MSK 2014 O 26 2s +3 - MSK +Z Europe/Kyiv 2:2:4 - LMT 1880 +2:2:4 - KMT 1924 May 2 +2 - EET 1930 Jun 21 +3 - MSK 1941 S 20 +1 c CE%sT 1943 N 6 +3 R MSK/MSD 1990 Jul 1 2 +2 1 EEST 1991 S 29 3 +2 c EE%sT 1996 May 13 +2 E EE%sT +Z Europe/Lisbon -0:36:45 - LMT 1884 +-0:36:45 - LMT 1912 Ja 1 0u +0 p WE%sT 1966 O 2 2s +1 - CET 1976 S 26 1 +0 p WE%sT 1986 +0 E WE%sT 1992 S 27 1u +1 E CE%sT 1996 Mar 31 1u +0 E WE%sT +Z Europe/London -0:1:15 - LMT 1847 D +0 G %s 1968 O 27 +1 - BST 1971 O 31 2u +0 G %s 1996 +0 E GMT/BST +Z Europe/Madrid -0:14:44 - LMT 1901 Ja 1 0u +0 s WE%sT 1940 Mar 16 23 +1 s CE%sT 1979 +1 E CE%sT +Z Europe/Malta 0:58:4 - LMT 1893 N 2 +1 I CE%sT 1973 Mar 31 +1 MT CE%sT 1981 +1 E CE%sT +Z Europe/Minsk 1:50:16 - LMT 1880 +1:50 - MMT 1924 May 2 +2 - EET 1930 Jun 21 +3 - MSK 1941 Jun 28 +1 c CE%sT 1944 Jul 3 +3 R MSK/MSD 1990 +3 - MSK 1991 Mar 31 2s +2 R EE%sT 2011 Mar 27 2s +3 - %z +Z Europe/Moscow 2:30:17 - LMT 1880 +2:30:17 - MMT 1916 Jul 3 +2:31:19 R %s 1919 Jul 1 0u +3 R %s 1921 O +3 R MSK/MSD 1922 O +2 - EET 1930 Jun 21 +3 R MSK/MSD 1991 Mar 31 2s +2 R EE%sT 1992 Ja 19 2s +3 R MSK/MSD 2011 Mar 27 2s +4 - MSK 2014 O 26 2s +3 - MSK +Z Europe/Paris 0:9:21 - LMT 1891 Mar 16 +0:9:21 - PMT 1911 Mar 11 +0 F WE%sT 1940 Jun 14 23 +1 c CE%sT 1944 Au 25 +0 F WE%sT 1945 S 16 3 +1 F CE%sT 1977 +1 E CE%sT +Z Europe/Prague 0:57:44 - LMT 1850 +0:57:44 - PMT 1891 O +1 c CE%sT 1945 May 9 +1 CZ CE%sT 1946 D 1 3 +1 -1 GMT 1947 F 23 2 +1 CZ CE%sT 1979 +1 E CE%sT +Z Europe/Riga 1:36:34 - LMT 1880 +1:36:34 - RMT 1918 Ap 15 2 +1:36:34 1 LST 1918 S 16 3 +1:36:34 - RMT 1919 Ap 1 2 +1:36:34 1 LST 1919 May 22 3 +1:36:34 - RMT 1926 May 11 +2 - EET 1940 Au 5 +3 - MSK 1941 Jul +1 c CE%sT 1944 O 13 +3 R MSK/MSD 1989 Mar lastSu 2s +2 1 EEST 1989 S lastSu 2s +2 LV EE%sT 1997 Ja 21 +2 E EE%sT 2000 F 29 +2 - EET 2001 Ja 2 +2 E EE%sT +Z Europe/Rome 0:49:56 - LMT 1866 D 12 +0:49:56 - RMT 1893 O 31 23u +1 I CE%sT 1943 S 10 +1 c CE%sT 1944 Jun 4 +1 I CE%sT 1980 +1 E CE%sT +Z Europe/Samara 3:20:20 - LMT 1919 Jul 1 0u +3 - %z 1930 Jun 21 +4 - %z 1935 Ja 27 +4 R %z 1989 Mar 26 2s +3 R %z 1991 Mar 31 2s +2 R %z 1991 S 29 2s +3 - %z 1991 O 20 3 +4 R %z 2010 Mar 28 2s +3 R %z 2011 Mar 27 2s +4 - %z +Z Europe/Saratov 3:4:18 - LMT 1919 Jul 1 0u +3 - %z 1930 Jun 21 +4 R %z 1988 Mar 27 2s +3 R %z 1991 Mar 31 2s +4 - %z 1992 Mar 29 2s +3 R %z 2011 Mar 27 2s +4 - %z 2014 O 26 2s +3 - %z 2016 D 4 2s +4 - %z +Z Europe/Simferopol 2:16:24 - LMT 1880 +2:16 - SMT 1924 May 2 +2 - EET 1930 Jun 21 +3 - MSK 1941 N +1 c CE%sT 1944 Ap 13 +3 R MSK/MSD 1990 +3 - MSK 1990 Jul 1 2 +2 - EET 1992 Mar 20 +2 c EE%sT 1994 May +3 c MSK/MSD 1996 Mar 31 0s +3 1 MSD 1996 O 27 3s +3 - MSK 1997 Mar lastSu 1u +2 E EE%sT 2014 Mar 30 2 +4 - MSK 2014 O 26 2s +3 - MSK +Z Europe/Sofia 1:33:16 - LMT 1880 +1:56:56 - IMT 1894 N 30 +2 - EET 1942 N 2 3 +1 c CE%sT 1945 +1 - CET 1945 Ap 2 3 +2 - EET 1979 Mar 31 23 +2 BG EE%sT 1982 S 26 3 +2 c EE%sT 1991 +2 e EE%sT 1997 +2 E EE%sT +Z Europe/Tallinn 1:39 - LMT 1880 +1:39 - TMT 1918 F +1 c CE%sT 1919 Jul +1:39 - TMT 1921 May +2 - EET 1940 Au 6 +3 - MSK 1941 S 15 +1 c CE%sT 1944 S 22 +3 R MSK/MSD 1989 Mar 26 2s +2 1 EEST 1989 S 24 2s +2 c EE%sT 1998 S 22 +2 E EE%sT 1999 O 31 4 +2 - EET 2002 F 21 +2 E EE%sT +Z Europe/Tirane 1:19:20 - LMT 1914 +1 - CET 1940 Jun 16 +1 q CE%sT 1984 Jul +1 E CE%sT +Z Europe/Ulyanovsk 3:13:36 - LMT 1919 Jul 1 0u +3 - %z 1930 Jun 21 +4 R %z 1989 Mar 26 2s +3 R %z 1991 Mar 31 2s +2 R %z 1992 Ja 19 2s +3 R %z 2011 Mar 27 2s +4 - %z 2014 O 26 2s +3 - %z 2016 Mar 27 2s +4 - %z +Z Europe/Vienna 1:5:21 - LMT 1893 Ap +1 c CE%sT 1920 +1 a CE%sT 1940 Ap 1 2s +1 c CE%sT 1945 Ap 2 2s +1 1 CEST 1945 Ap 12 2s +1 - CET 1946 +1 a CE%sT 1981 +1 E CE%sT +Z Europe/Vilnius 1:41:16 - LMT 1880 +1:24 - WMT 1917 +1:35:36 - KMT 1919 O 10 +1 - CET 1920 Jul 12 +2 - EET 1920 O 9 +1 - CET 1940 Au 3 +3 - MSK 1941 Jun 24 +1 c CE%sT 1944 Au +3 R MSK/MSD 1989 Mar 26 2s +2 R EE%sT 1991 S 29 2s +2 c EE%sT 1998 +2 - EET 1998 Mar 29 1u +1 E CE%sT 1999 O 31 1u +2 - EET 2003 +2 E EE%sT +Z Europe/Volgograd 2:57:40 - LMT 1920 Ja 3 +3 - %z 1930 Jun 21 +4 - %z 1961 N 11 +4 R %z 1988 Mar 27 2s +3 R MSK/MSD 1991 Mar 31 2s +4 - %z 1992 Mar 29 2s +3 R MSK/MSD 2011 Mar 27 2s +4 - MSK 2014 O 26 2s +3 - MSK 2018 O 28 2s +4 - %z 2020 D 27 2s +3 - MSK +Z Europe/Warsaw 1:24 - LMT 1880 +1:24 - WMT 1915 Au 5 +1 c CE%sT 1918 S 16 3 +2 O EE%sT 1922 Jun +1 O CE%sT 1940 Jun 23 2 +1 c CE%sT 1944 O +1 O CE%sT 1977 +1 W- CE%sT 1988 +1 E CE%sT +Z Europe/Zurich 0:34:8 - LMT 1853 Jul 16 +0:29:46 - BMT 1894 Jun +1 CH CE%sT 1981 +1 E CE%sT +Z Factory 0 - -00 +Z Indian/Chagos 4:49:40 - LMT 1907 +5 - %z 1996 +6 - %z +Z Indian/Maldives 4:54 - LMT 1880 +4:54 - MMT 1960 +5 - %z +Z Indian/Mauritius 3:50 - LMT 1907 +4 MU %z +Z Pacific/Apia 12:33:4 - LMT 1892 Jul 5 +-11:26:56 - LMT 1911 +-11:30 - %z 1950 +-11 WS %z 2011 D 29 24 +13 WS %z +Z Pacific/Auckland 11:39:4 - LMT 1868 N 2 +11:30 NZ NZ%sT 1946 +12 NZ NZ%sT +Z Pacific/Bougainville 10:22:16 - LMT 1880 +9:48:32 - PMMT 1895 +10 - %z 1942 Jul +9 - %z 1945 Au 21 +10 - %z 2014 D 28 2 +11 - %z +Z Pacific/Chatham 12:13:48 - LMT 1868 N 2 +12:15 - %z 1946 +12:45 k %z +Z Pacific/Easter -7:17:28 - LMT 1890 +-7:17:28 - EMT 1932 S +-7 x %z 1982 Mar 14 3u +-6 x %z +Z Pacific/Efate 11:13:16 - LMT 1912 Ja 13 +11 VU %z +Z Pacific/Fakaofo -11:24:56 - LMT 1901 +-11 - %z 2011 D 30 +13 - %z +Z Pacific/Fiji 11:55:44 - LMT 1915 O 26 +12 FJ %z +Z Pacific/Galapagos -5:58:24 - LMT 1931 +-5 - %z 1986 +-6 EC %z +Z Pacific/Gambier -8:59:48 - LMT 1912 O +-9 - %z +Z Pacific/Guadalcanal 10:39:48 - LMT 1912 O +11 - %z +Z Pacific/Guam -14:21 - LMT 1844 D 31 +9:39 - LMT 1901 +10 - GST 1941 D 10 +9 - %z 1944 Jul 31 +10 Gu G%sT 2000 D 23 +10 - ChST +Z Pacific/Honolulu -10:31:26 - LMT 1896 Ja 13 12 +-10:30 - HST 1933 Ap 30 2 +-10:30 1 HDT 1933 May 21 12 +-10:30 u H%sT 1947 Jun 8 2 +-10 - HST +Z Pacific/Kanton 0 - -00 1937 Au 31 +-12 - %z 1979 O +-11 - %z 1994 D 31 +13 - %z +Z Pacific/Kiritimati -10:29:20 - LMT 1901 +-10:40 - %z 1979 O +-10 - %z 1994 D 31 +14 - %z +Z Pacific/Kosrae -13:8:4 - LMT 1844 D 31 +10:51:56 - LMT 1901 +11 - %z 1914 O +9 - %z 1919 F +11 - %z 1937 +10 - %z 1941 Ap +9 - %z 1945 Au +11 - %z 1969 O +12 - %z 1999 +11 - %z +Z Pacific/Kwajalein 11:9:20 - LMT 1901 +11 - %z 1937 +10 - %z 1941 Ap +9 - %z 1944 F 6 +11 - %z 1969 O +-12 - %z 1993 Au 20 24 +12 - %z +Z Pacific/Marquesas -9:18 - LMT 1912 O +-9:30 - %z +Z Pacific/Nauru 11:7:40 - LMT 1921 Ja 15 +11:30 - %z 1942 Au 29 +9 - %z 1945 S 8 +11:30 - %z 1979 F 10 2 +12 - %z +Z Pacific/Niue -11:19:40 - LMT 1952 O 16 +-11:20 - %z 1964 Jul +-11 - %z +Z Pacific/Norfolk 11:11:52 - LMT 1901 +11:12 - %z 1951 +11:30 - %z 1974 O 27 2s +11:30 1 %z 1975 Mar 2 2s +11:30 - %z 2015 O 4 2s +11 - %z 2019 Jul +11 AN %z +Z Pacific/Noumea 11:5:48 - LMT 1912 Ja 13 +11 NC %z +Z Pacific/Pago_Pago 12:37:12 - LMT 1892 Jul 5 +-11:22:48 - LMT 1911 +-11 - SST +Z Pacific/Palau -15:2:4 - LMT 1844 D 31 +8:57:56 - LMT 1901 +9 - %z +Z Pacific/Pitcairn -8:40:20 - LMT 1901 +-8:30 - %z 1998 Ap 27 +-8 - %z +Z Pacific/Port_Moresby 9:48:40 - LMT 1880 +9:48:32 - PMMT 1895 +10 - %z +Z Pacific/Rarotonga 13:20:56 - LMT 1899 D 26 +-10:39:4 - LMT 1952 O 16 +-10:30 - %z 1978 N 12 +-10 CK %z +Z Pacific/Tahiti -9:58:16 - LMT 1912 O +-10 - %z +Z Pacific/Tarawa 11:32:4 - LMT 1901 +12 - %z +Z Pacific/Tongatapu 12:19:12 - LMT 1945 S 10 +12:20 - %z 1961 +13 - %z 1999 +13 TO %z +L Etc/GMT GMT +L Australia/Sydney Australia/ACT +L Australia/Lord_Howe Australia/LHI +L Australia/Sydney Australia/NSW +L Australia/Darwin Australia/North +L Australia/Brisbane Australia/Queensland +L Australia/Adelaide Australia/South +L Australia/Hobart Australia/Tasmania +L Australia/Melbourne Australia/Victoria +L Australia/Perth Australia/West +L Australia/Broken_Hill Australia/Yancowinna +L America/Rio_Branco Brazil/Acre +L America/Noronha Brazil/DeNoronha +L America/Sao_Paulo Brazil/East +L America/Manaus Brazil/West +L Europe/Brussels CET +L America/Chicago CST6CDT +L America/Halifax Canada/Atlantic +L America/Winnipeg Canada/Central +L America/Toronto Canada/Eastern +L America/Edmonton Canada/Mountain +L America/St_Johns Canada/Newfoundland +L America/Vancouver Canada/Pacific +L America/Regina Canada/Saskatchewan +L America/Whitehorse Canada/Yukon +L America/Santiago Chile/Continental +L Pacific/Easter Chile/EasterIsland +L America/Havana Cuba +L Europe/Athens EET +L America/Panama EST +L America/New_York EST5EDT +L Africa/Cairo Egypt +L Europe/Dublin Eire +L Etc/GMT Etc/GMT+0 +L Etc/GMT Etc/GMT-0 +L Etc/GMT Etc/GMT0 +L Etc/GMT Etc/Greenwich +L Etc/UTC Etc/UCT +L Etc/UTC Etc/Universal +L Etc/UTC Etc/Zulu +L Europe/London GB +L Europe/London GB-Eire +L Etc/GMT GMT+0 +L Etc/GMT GMT-0 +L Etc/GMT GMT0 +L Etc/GMT Greenwich +L Asia/Hong_Kong Hongkong +L Africa/Abidjan Iceland +L Asia/Tehran Iran +L Asia/Jerusalem Israel +L America/Jamaica Jamaica +L Asia/Tokyo Japan +L Pacific/Kwajalein Kwajalein +L Africa/Tripoli Libya +L Europe/Brussels MET +L America/Phoenix MST +L America/Denver MST7MDT +L America/Tijuana Mexico/BajaNorte +L America/Mazatlan Mexico/BajaSur +L America/Mexico_City Mexico/General +L Pacific/Auckland NZ +L Pacific/Chatham NZ-CHAT +L America/Denver Navajo +L Asia/Shanghai PRC +L Europe/Warsaw Poland +L Europe/Lisbon Portugal +L Asia/Taipei ROC +L Asia/Seoul ROK +L Asia/Singapore Singapore +L Europe/Istanbul Turkey +L Etc/UTC UCT +L America/Anchorage US/Alaska +L America/Adak US/Aleutian +L America/Phoenix US/Arizona +L America/Chicago US/Central +L America/Indiana/Indianapolis US/East-Indiana +L America/New_York US/Eastern +L Pacific/Honolulu US/Hawaii +L America/Indiana/Knox US/Indiana-Starke +L America/Detroit US/Michigan +L America/Denver US/Mountain +L America/Los_Angeles US/Pacific +L Pacific/Pago_Pago US/Samoa +L Etc/UTC UTC +L Etc/UTC Universal +L Europe/Moscow W-SU +L Etc/UTC Zulu +L America/Argentina/Buenos_Aires America/Buenos_Aires +L America/Argentina/Catamarca America/Catamarca +L America/Argentina/Cordoba America/Cordoba +L America/Indiana/Indianapolis America/Indianapolis +L America/Argentina/Jujuy America/Jujuy +L America/Indiana/Knox America/Knox_IN +L America/Kentucky/Louisville America/Louisville +L America/Argentina/Mendoza America/Mendoza +L America/Puerto_Rico America/Virgin +L Pacific/Pago_Pago Pacific/Samoa +L Africa/Abidjan Africa/Accra +L Africa/Nairobi Africa/Addis_Ababa +L Africa/Nairobi Africa/Asmara +L Africa/Abidjan Africa/Bamako +L Africa/Lagos Africa/Bangui +L Africa/Abidjan Africa/Banjul +L Africa/Maputo Africa/Blantyre +L Africa/Lagos Africa/Brazzaville +L Africa/Maputo Africa/Bujumbura +L Africa/Abidjan Africa/Conakry +L Africa/Abidjan Africa/Dakar +L Africa/Nairobi Africa/Dar_es_Salaam +L Africa/Nairobi Africa/Djibouti +L Africa/Lagos Africa/Douala +L Africa/Abidjan Africa/Freetown +L Africa/Maputo Africa/Gaborone +L Africa/Maputo Africa/Harare +L Africa/Nairobi Africa/Kampala +L Africa/Maputo Africa/Kigali +L Africa/Lagos Africa/Kinshasa +L Africa/Lagos Africa/Libreville +L Africa/Abidjan Africa/Lome +L Africa/Lagos Africa/Luanda +L Africa/Maputo Africa/Lubumbashi +L Africa/Maputo Africa/Lusaka +L Africa/Lagos Africa/Malabo +L Africa/Johannesburg Africa/Maseru +L Africa/Johannesburg Africa/Mbabane +L Africa/Nairobi Africa/Mogadishu +L Africa/Lagos Africa/Niamey +L Africa/Abidjan Africa/Nouakchott +L Africa/Abidjan Africa/Ouagadougou +L Africa/Lagos Africa/Porto-Novo +L America/Puerto_Rico America/Anguilla +L America/Puerto_Rico America/Antigua +L America/Puerto_Rico America/Aruba +L America/Panama America/Atikokan +L America/Puerto_Rico America/Blanc-Sablon +L America/Panama America/Cayman +L America/Phoenix America/Creston +L America/Puerto_Rico America/Curacao +L America/Puerto_Rico America/Dominica +L America/Puerto_Rico America/Grenada +L America/Puerto_Rico America/Guadeloupe +L America/Puerto_Rico America/Kralendijk +L America/Puerto_Rico America/Lower_Princes +L America/Puerto_Rico America/Marigot +L America/Puerto_Rico America/Montserrat +L America/Toronto America/Nassau +L America/Puerto_Rico America/Port_of_Spain +L America/Puerto_Rico America/St_Barthelemy +L America/Puerto_Rico America/St_Kitts +L America/Puerto_Rico America/St_Lucia +L America/Puerto_Rico America/St_Thomas +L America/Puerto_Rico America/St_Vincent +L America/Puerto_Rico America/Tortola +L Pacific/Port_Moresby Antarctica/DumontDUrville +L Pacific/Auckland Antarctica/McMurdo +L Asia/Riyadh Antarctica/Syowa +L Europe/Berlin Arctic/Longyearbyen +L Asia/Riyadh Asia/Aden +L Asia/Qatar Asia/Bahrain +L Asia/Kuching Asia/Brunei +L Asia/Singapore Asia/Kuala_Lumpur +L Asia/Riyadh Asia/Kuwait +L Asia/Dubai Asia/Muscat +L Asia/Bangkok Asia/Phnom_Penh +L Asia/Bangkok Asia/Vientiane +L Africa/Abidjan Atlantic/Reykjavik +L Africa/Abidjan Atlantic/St_Helena +L Europe/Brussels Europe/Amsterdam +L Europe/Prague Europe/Bratislava +L Europe/Zurich Europe/Busingen +L Europe/Berlin Europe/Copenhagen +L Europe/London Europe/Guernsey +L Europe/London Europe/Isle_of_Man +L Europe/London Europe/Jersey +L Europe/Belgrade Europe/Ljubljana +L Europe/Brussels Europe/Luxembourg +L Europe/Helsinki Europe/Mariehamn +L Europe/Paris Europe/Monaco +L Europe/Berlin Europe/Oslo +L Europe/Belgrade Europe/Podgorica +L Europe/Rome Europe/San_Marino +L Europe/Belgrade Europe/Sarajevo +L Europe/Belgrade Europe/Skopje +L Europe/Berlin Europe/Stockholm +L Europe/Zurich Europe/Vaduz +L Europe/Rome Europe/Vatican +L Europe/Belgrade Europe/Zagreb +L Africa/Nairobi Indian/Antananarivo +L Asia/Bangkok Indian/Christmas +L Asia/Yangon Indian/Cocos +L Africa/Nairobi Indian/Comoro +L Indian/Maldives Indian/Kerguelen +L Asia/Dubai Indian/Mahe +L Africa/Nairobi Indian/Mayotte +L Asia/Dubai Indian/Reunion +L Pacific/Port_Moresby Pacific/Chuuk +L Pacific/Tarawa Pacific/Funafuti +L Pacific/Tarawa Pacific/Majuro +L Pacific/Pago_Pago Pacific/Midway +L Pacific/Guadalcanal Pacific/Pohnpei +L Pacific/Guam Pacific/Saipan +L Pacific/Tarawa Pacific/Wake +L Pacific/Tarawa Pacific/Wallis +L Africa/Abidjan Africa/Timbuktu +L America/Argentina/Catamarca America/Argentina/ComodRivadavia +L America/Adak America/Atka +L America/Panama America/Coral_Harbour +L America/Tijuana America/Ensenada +L America/Indiana/Indianapolis America/Fort_Wayne +L America/Toronto America/Montreal +L America/Toronto America/Nipigon +L America/Iqaluit America/Pangnirtung +L America/Rio_Branco America/Porto_Acre +L America/Winnipeg America/Rainy_River +L America/Argentina/Cordoba America/Rosario +L America/Tijuana America/Santa_Isabel +L America/Denver America/Shiprock +L America/Toronto America/Thunder_Bay +L America/Edmonton America/Yellowknife +L Pacific/Auckland Antarctica/South_Pole +L Asia/Ulaanbaatar Asia/Choibalsan +L Asia/Shanghai Asia/Chongqing +L Asia/Shanghai Asia/Harbin +L Asia/Urumqi Asia/Kashgar +L Asia/Jerusalem Asia/Tel_Aviv +L Europe/Berlin Atlantic/Jan_Mayen +L Australia/Sydney Australia/Canberra +L Australia/Hobart Australia/Currie +L Europe/London Europe/Belfast +L Europe/Chisinau Europe/Tiraspol +L Europe/Kyiv Europe/Uzhgorod +L Europe/Kyiv Europe/Zaporozhye +L Pacific/Kanton Pacific/Enderbury +L Pacific/Honolulu Pacific/Johnston +L Pacific/Port_Moresby Pacific/Yap +L Europe/Lisbon WET +L Africa/Nairobi Africa/Asmera +L America/Nuuk America/Godthab +L Asia/Ashgabat Asia/Ashkhabad +L Asia/Kolkata Asia/Calcutta +L Asia/Shanghai Asia/Chungking +L Asia/Dhaka Asia/Dacca +L Europe/Istanbul Asia/Istanbul +L Asia/Kathmandu Asia/Katmandu +L Asia/Macau Asia/Macao +L Asia/Yangon Asia/Rangoon +L Asia/Ho_Chi_Minh Asia/Saigon +L Asia/Thimphu Asia/Thimbu +L Asia/Makassar Asia/Ujung_Pandang +L Asia/Ulaanbaatar Asia/Ulan_Bator +L Atlantic/Faroe Atlantic/Faeroe +L Europe/Kyiv Europe/Kiev +L Asia/Nicosia Europe/Nicosia +L Pacific/Honolulu HST +L America/Los_Angeles PST8PDT +L Pacific/Guadalcanal Pacific/Ponape +L Pacific/Port_Moresby Pacific/Truk diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/zone.tab b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/zone.tab new file mode 100644 index 0000000000000000000000000000000000000000..d2be66359f3bed4cdd0e3d5a99342379f3d00b7c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/zone.tab @@ -0,0 +1,447 @@ +# tzdb timezone descriptions (deprecated version) +# +# This file is in the public domain, so clarified as of +# 2009-05-17 by Arthur David Olson. +# +# From Paul Eggert (2021-09-20): +# This file is intended as a backward-compatibility aid for older programs. +# New programs should use zone1970.tab. This file is like zone1970.tab (see +# zone1970.tab's comments), but with the following additional restrictions: +# +# 1. This file contains only ASCII characters. +# 2. The first data column contains exactly one country code. +# +# Because of (2), each row stands for an area that is the intersection +# of a region identified by a country code and of a timezone where civil +# clocks have agreed since 1970; this is a narrower definition than +# that of zone1970.tab. +# +# Unlike zone1970.tab, a row's third column can be a Link from +# 'backward' instead of a Zone. +# +# This table is intended as an aid for users, to help them select timezones +# appropriate for their practical needs. It is not intended to take or +# endorse any position on legal or territorial claims. +# +#country- +#code coordinates TZ comments +AD +4230+00131 Europe/Andorra +AE +2518+05518 Asia/Dubai +AF +3431+06912 Asia/Kabul +AG +1703-06148 America/Antigua +AI +1812-06304 America/Anguilla +AL +4120+01950 Europe/Tirane +AM +4011+04430 Asia/Yerevan +AO -0848+01314 Africa/Luanda +AQ -7750+16636 Antarctica/McMurdo New Zealand time - McMurdo, South Pole +AQ -6617+11031 Antarctica/Casey Casey +AQ -6835+07758 Antarctica/Davis Davis +AQ -6640+14001 Antarctica/DumontDUrville Dumont-d'Urville +AQ -6736+06253 Antarctica/Mawson Mawson +AQ -6448-06406 Antarctica/Palmer Palmer +AQ -6734-06808 Antarctica/Rothera Rothera +AQ -690022+0393524 Antarctica/Syowa Syowa +AQ -720041+0023206 Antarctica/Troll Troll +AQ -7824+10654 Antarctica/Vostok Vostok +AR -3436-05827 America/Argentina/Buenos_Aires Buenos Aires (BA, CF) +AR -3124-06411 America/Argentina/Cordoba Argentina (most areas: CB, CC, CN, ER, FM, MN, SE, SF) +AR -2447-06525 America/Argentina/Salta Salta (SA, LP, NQ, RN) +AR -2411-06518 America/Argentina/Jujuy Jujuy (JY) +AR -2649-06513 America/Argentina/Tucuman Tucuman (TM) +AR -2828-06547 America/Argentina/Catamarca Catamarca (CT), Chubut (CH) +AR -2926-06651 America/Argentina/La_Rioja La Rioja (LR) +AR -3132-06831 America/Argentina/San_Juan San Juan (SJ) +AR -3253-06849 America/Argentina/Mendoza Mendoza (MZ) +AR -3319-06621 America/Argentina/San_Luis San Luis (SL) +AR -5138-06913 America/Argentina/Rio_Gallegos Santa Cruz (SC) +AR -5448-06818 America/Argentina/Ushuaia Tierra del Fuego (TF) +AS -1416-17042 Pacific/Pago_Pago +AT +4813+01620 Europe/Vienna +AU -3133+15905 Australia/Lord_Howe Lord Howe Island +AU -5430+15857 Antarctica/Macquarie Macquarie Island +AU -4253+14719 Australia/Hobart Tasmania +AU -3749+14458 Australia/Melbourne Victoria +AU -3352+15113 Australia/Sydney New South Wales (most areas) +AU -3157+14127 Australia/Broken_Hill New South Wales (Yancowinna) +AU -2728+15302 Australia/Brisbane Queensland (most areas) +AU -2016+14900 Australia/Lindeman Queensland (Whitsunday Islands) +AU -3455+13835 Australia/Adelaide South Australia +AU -1228+13050 Australia/Darwin Northern Territory +AU -3157+11551 Australia/Perth Western Australia (most areas) +AU -3143+12852 Australia/Eucla Western Australia (Eucla) +AW +1230-06958 America/Aruba +AX +6006+01957 Europe/Mariehamn +AZ +4023+04951 Asia/Baku +BA +4352+01825 Europe/Sarajevo +BB +1306-05937 America/Barbados +BD +2343+09025 Asia/Dhaka +BE +5050+00420 Europe/Brussels +BF +1222-00131 Africa/Ouagadougou +BG +4241+02319 Europe/Sofia +BH +2623+05035 Asia/Bahrain +BI -0323+02922 Africa/Bujumbura +BJ +0629+00237 Africa/Porto-Novo +BL +1753-06251 America/St_Barthelemy +BM +3217-06446 Atlantic/Bermuda +BN +0456+11455 Asia/Brunei +BO -1630-06809 America/La_Paz +BQ +120903-0681636 America/Kralendijk +BR -0351-03225 America/Noronha Atlantic islands +BR -0127-04829 America/Belem Para (east), Amapa +BR -0343-03830 America/Fortaleza Brazil (northeast: MA, PI, CE, RN, PB) +BR -0803-03454 America/Recife Pernambuco +BR -0712-04812 America/Araguaina Tocantins +BR -0940-03543 America/Maceio Alagoas, Sergipe +BR -1259-03831 America/Bahia Bahia +BR -2332-04637 America/Sao_Paulo Brazil (southeast: GO, DF, MG, ES, RJ, SP, PR, SC, RS) +BR -2027-05437 America/Campo_Grande Mato Grosso do Sul +BR -1535-05605 America/Cuiaba Mato Grosso +BR -0226-05452 America/Santarem Para (west) +BR -0846-06354 America/Porto_Velho Rondonia +BR +0249-06040 America/Boa_Vista Roraima +BR -0308-06001 America/Manaus Amazonas (east) +BR -0640-06952 America/Eirunepe Amazonas (west) +BR -0958-06748 America/Rio_Branco Acre +BS +2505-07721 America/Nassau +BT +2728+08939 Asia/Thimphu +BW -2439+02555 Africa/Gaborone +BY +5354+02734 Europe/Minsk +BZ +1730-08812 America/Belize +CA +4734-05243 America/St_Johns Newfoundland, Labrador (SE) +CA +4439-06336 America/Halifax Atlantic - NS (most areas), PE +CA +4612-05957 America/Glace_Bay Atlantic - NS (Cape Breton) +CA +4606-06447 America/Moncton Atlantic - New Brunswick +CA +5320-06025 America/Goose_Bay Atlantic - Labrador (most areas) +CA +5125-05707 America/Blanc-Sablon AST - QC (Lower North Shore) +CA +4339-07923 America/Toronto Eastern - ON & QC (most areas) +CA +6344-06828 America/Iqaluit Eastern - NU (most areas) +CA +484531-0913718 America/Atikokan EST - ON (Atikokan), NU (Coral H) +CA +4953-09709 America/Winnipeg Central - ON (west), Manitoba +CA +744144-0944945 America/Resolute Central - NU (Resolute) +CA +624900-0920459 America/Rankin_Inlet Central - NU (central) +CA +5024-10439 America/Regina CST - SK (most areas) +CA +5017-10750 America/Swift_Current CST - SK (midwest) +CA +5333-11328 America/Edmonton Mountain - AB, BC(E), NT(E), SK(W) +CA +690650-1050310 America/Cambridge_Bay Mountain - NU (west) +CA +682059-1334300 America/Inuvik Mountain - NT (west) +CA +4906-11631 America/Creston MST - BC (Creston) +CA +5546-12014 America/Dawson_Creek MST - BC (Dawson Cr, Ft St John) +CA +5848-12242 America/Fort_Nelson MST - BC (Ft Nelson) +CA +6043-13503 America/Whitehorse MST - Yukon (east) +CA +6404-13925 America/Dawson MST - Yukon (west) +CA +4916-12307 America/Vancouver Pacific - BC (most areas) +CC -1210+09655 Indian/Cocos +CD -0418+01518 Africa/Kinshasa Dem. Rep. of Congo (west) +CD -1140+02728 Africa/Lubumbashi Dem. Rep. of Congo (east) +CF +0422+01835 Africa/Bangui +CG -0416+01517 Africa/Brazzaville +CH +4723+00832 Europe/Zurich +CI +0519-00402 Africa/Abidjan +CK -2114-15946 Pacific/Rarotonga +CL -3327-07040 America/Santiago most of Chile +CL -5309-07055 America/Punta_Arenas Region of Magallanes +CL -2709-10926 Pacific/Easter Easter Island +CM +0403+00942 Africa/Douala +CN +3114+12128 Asia/Shanghai Beijing Time +CN +4348+08735 Asia/Urumqi Xinjiang Time +CO +0436-07405 America/Bogota +CR +0956-08405 America/Costa_Rica +CU +2308-08222 America/Havana +CV +1455-02331 Atlantic/Cape_Verde +CW +1211-06900 America/Curacao +CX -1025+10543 Indian/Christmas +CY +3510+03322 Asia/Nicosia most of Cyprus +CY +3507+03357 Asia/Famagusta Northern Cyprus +CZ +5005+01426 Europe/Prague +DE +5230+01322 Europe/Berlin most of Germany +DE +4742+00841 Europe/Busingen Busingen +DJ +1136+04309 Africa/Djibouti +DK +5540+01235 Europe/Copenhagen +DM +1518-06124 America/Dominica +DO +1828-06954 America/Santo_Domingo +DZ +3647+00303 Africa/Algiers +EC -0210-07950 America/Guayaquil Ecuador (mainland) +EC -0054-08936 Pacific/Galapagos Galapagos Islands +EE +5925+02445 Europe/Tallinn +EG +3003+03115 Africa/Cairo +EH +2709-01312 Africa/El_Aaiun +ER +1520+03853 Africa/Asmara +ES +4024-00341 Europe/Madrid Spain (mainland) +ES +3553-00519 Africa/Ceuta Ceuta, Melilla +ES +2806-01524 Atlantic/Canary Canary Islands +ET +0902+03842 Africa/Addis_Ababa +FI +6010+02458 Europe/Helsinki +FJ -1808+17825 Pacific/Fiji +FK -5142-05751 Atlantic/Stanley +FM +0725+15147 Pacific/Chuuk Chuuk/Truk, Yap +FM +0658+15813 Pacific/Pohnpei Pohnpei/Ponape +FM +0519+16259 Pacific/Kosrae Kosrae +FO +6201-00646 Atlantic/Faroe +FR +4852+00220 Europe/Paris +GA +0023+00927 Africa/Libreville +GB +513030-0000731 Europe/London +GD +1203-06145 America/Grenada +GE +4143+04449 Asia/Tbilisi +GF +0456-05220 America/Cayenne +GG +492717-0023210 Europe/Guernsey +GH +0533-00013 Africa/Accra +GI +3608-00521 Europe/Gibraltar +GL +6411-05144 America/Nuuk most of Greenland +GL +7646-01840 America/Danmarkshavn National Park (east coast) +GL +7029-02158 America/Scoresbysund Scoresbysund/Ittoqqortoormiit +GL +7634-06847 America/Thule Thule/Pituffik +GM +1328-01639 Africa/Banjul +GN +0931-01343 Africa/Conakry +GP +1614-06132 America/Guadeloupe +GQ +0345+00847 Africa/Malabo +GR +3758+02343 Europe/Athens +GS -5416-03632 Atlantic/South_Georgia +GT +1438-09031 America/Guatemala +GU +1328+14445 Pacific/Guam +GW +1151-01535 Africa/Bissau +GY +0648-05810 America/Guyana +HK +2217+11409 Asia/Hong_Kong +HN +1406-08713 America/Tegucigalpa +HR +4548+01558 Europe/Zagreb +HT +1832-07220 America/Port-au-Prince +HU +4730+01905 Europe/Budapest +ID -0610+10648 Asia/Jakarta Java, Sumatra +ID -0002+10920 Asia/Pontianak Borneo (west, central) +ID -0507+11924 Asia/Makassar Borneo (east, south), Sulawesi/Celebes, Bali, Nusa Tengarra, Timor (west) +ID -0232+14042 Asia/Jayapura New Guinea (West Papua / Irian Jaya), Malukus/Moluccas +IE +5320-00615 Europe/Dublin +IL +314650+0351326 Asia/Jerusalem +IM +5409-00428 Europe/Isle_of_Man +IN +2232+08822 Asia/Kolkata +IO -0720+07225 Indian/Chagos +IQ +3321+04425 Asia/Baghdad +IR +3540+05126 Asia/Tehran +IS +6409-02151 Atlantic/Reykjavik +IT +4154+01229 Europe/Rome +JE +491101-0020624 Europe/Jersey +JM +175805-0764736 America/Jamaica +JO +3157+03556 Asia/Amman +JP +353916+1394441 Asia/Tokyo +KE -0117+03649 Africa/Nairobi +KG +4254+07436 Asia/Bishkek +KH +1133+10455 Asia/Phnom_Penh +KI +0125+17300 Pacific/Tarawa Gilbert Islands +KI -0247-17143 Pacific/Kanton Phoenix Islands +KI +0152-15720 Pacific/Kiritimati Line Islands +KM -1141+04316 Indian/Comoro +KN +1718-06243 America/St_Kitts +KP +3901+12545 Asia/Pyongyang +KR +3733+12658 Asia/Seoul +KW +2920+04759 Asia/Kuwait +KY +1918-08123 America/Cayman +KZ +4315+07657 Asia/Almaty most of Kazakhstan +KZ +4448+06528 Asia/Qyzylorda Qyzylorda/Kyzylorda/Kzyl-Orda +KZ +5312+06337 Asia/Qostanay Qostanay/Kostanay/Kustanay +KZ +5017+05710 Asia/Aqtobe Aqtobe/Aktobe +KZ +4431+05016 Asia/Aqtau Mangghystau/Mankistau +KZ +4707+05156 Asia/Atyrau Atyrau/Atirau/Gur'yev +KZ +5113+05121 Asia/Oral West Kazakhstan +LA +1758+10236 Asia/Vientiane +LB +3353+03530 Asia/Beirut +LC +1401-06100 America/St_Lucia +LI +4709+00931 Europe/Vaduz +LK +0656+07951 Asia/Colombo +LR +0618-01047 Africa/Monrovia +LS -2928+02730 Africa/Maseru +LT +5441+02519 Europe/Vilnius +LU +4936+00609 Europe/Luxembourg +LV +5657+02406 Europe/Riga +LY +3254+01311 Africa/Tripoli +MA +3339-00735 Africa/Casablanca +MC +4342+00723 Europe/Monaco +MD +4700+02850 Europe/Chisinau +ME +4226+01916 Europe/Podgorica +MF +1804-06305 America/Marigot +MG -1855+04731 Indian/Antananarivo +MH +0709+17112 Pacific/Majuro most of Marshall Islands +MH +0905+16720 Pacific/Kwajalein Kwajalein +MK +4159+02126 Europe/Skopje +ML +1239-00800 Africa/Bamako +MM +1647+09610 Asia/Yangon +MN +4755+10653 Asia/Ulaanbaatar most of Mongolia +MN +4801+09139 Asia/Hovd Bayan-Olgii, Hovd, Uvs +MO +221150+1133230 Asia/Macau +MP +1512+14545 Pacific/Saipan +MQ +1436-06105 America/Martinique +MR +1806-01557 Africa/Nouakchott +MS +1643-06213 America/Montserrat +MT +3554+01431 Europe/Malta +MU -2010+05730 Indian/Mauritius +MV +0410+07330 Indian/Maldives +MW -1547+03500 Africa/Blantyre +MX +1924-09909 America/Mexico_City Central Mexico +MX +2105-08646 America/Cancun Quintana Roo +MX +2058-08937 America/Merida Campeche, Yucatan +MX +2540-10019 America/Monterrey Durango; Coahuila, Nuevo Leon, Tamaulipas (most areas) +MX +2550-09730 America/Matamoros Coahuila, Nuevo Leon, Tamaulipas (US border) +MX +2838-10605 America/Chihuahua Chihuahua (most areas) +MX +3144-10629 America/Ciudad_Juarez Chihuahua (US border - west) +MX +2934-10425 America/Ojinaga Chihuahua (US border - east) +MX +2313-10625 America/Mazatlan Baja California Sur, Nayarit (most areas), Sinaloa +MX +2048-10515 America/Bahia_Banderas Bahia de Banderas +MX +2904-11058 America/Hermosillo Sonora +MX +3232-11701 America/Tijuana Baja California +MY +0310+10142 Asia/Kuala_Lumpur Malaysia (peninsula) +MY +0133+11020 Asia/Kuching Sabah, Sarawak +MZ -2558+03235 Africa/Maputo +NA -2234+01706 Africa/Windhoek +NC -2216+16627 Pacific/Noumea +NE +1331+00207 Africa/Niamey +NF -2903+16758 Pacific/Norfolk +NG +0627+00324 Africa/Lagos +NI +1209-08617 America/Managua +NL +5222+00454 Europe/Amsterdam +NO +5955+01045 Europe/Oslo +NP +2743+08519 Asia/Kathmandu +NR -0031+16655 Pacific/Nauru +NU -1901-16955 Pacific/Niue +NZ -3652+17446 Pacific/Auckland most of New Zealand +NZ -4357-17633 Pacific/Chatham Chatham Islands +OM +2336+05835 Asia/Muscat +PA +0858-07932 America/Panama +PE -1203-07703 America/Lima +PF -1732-14934 Pacific/Tahiti Society Islands +PF -0900-13930 Pacific/Marquesas Marquesas Islands +PF -2308-13457 Pacific/Gambier Gambier Islands +PG -0930+14710 Pacific/Port_Moresby most of Papua New Guinea +PG -0613+15534 Pacific/Bougainville Bougainville +PH +143512+1205804 Asia/Manila +PK +2452+06703 Asia/Karachi +PL +5215+02100 Europe/Warsaw +PM +4703-05620 America/Miquelon +PN -2504-13005 Pacific/Pitcairn +PR +182806-0660622 America/Puerto_Rico +PS +3130+03428 Asia/Gaza Gaza Strip +PS +313200+0350542 Asia/Hebron West Bank +PT +3843-00908 Europe/Lisbon Portugal (mainland) +PT +3238-01654 Atlantic/Madeira Madeira Islands +PT +3744-02540 Atlantic/Azores Azores +PW +0720+13429 Pacific/Palau +PY -2516-05740 America/Asuncion +QA +2517+05132 Asia/Qatar +RE -2052+05528 Indian/Reunion +RO +4426+02606 Europe/Bucharest +RS +4450+02030 Europe/Belgrade +RU +5443+02030 Europe/Kaliningrad MSK-01 - Kaliningrad +RU +554521+0373704 Europe/Moscow MSK+00 - Moscow area +# The obsolescent zone.tab format cannot represent Europe/Simferopol well. +# Put it in RU section and list as UA. See "territorial claims" above. +# Programs should use zone1970.tab instead; see above. +UA +4457+03406 Europe/Simferopol Crimea +RU +5836+04939 Europe/Kirov MSK+00 - Kirov +RU +4844+04425 Europe/Volgograd MSK+00 - Volgograd +RU +4621+04803 Europe/Astrakhan MSK+01 - Astrakhan +RU +5134+04602 Europe/Saratov MSK+01 - Saratov +RU +5420+04824 Europe/Ulyanovsk MSK+01 - Ulyanovsk +RU +5312+05009 Europe/Samara MSK+01 - Samara, Udmurtia +RU +5651+06036 Asia/Yekaterinburg MSK+02 - Urals +RU +5500+07324 Asia/Omsk MSK+03 - Omsk +RU +5502+08255 Asia/Novosibirsk MSK+04 - Novosibirsk +RU +5322+08345 Asia/Barnaul MSK+04 - Altai +RU +5630+08458 Asia/Tomsk MSK+04 - Tomsk +RU +5345+08707 Asia/Novokuznetsk MSK+04 - Kemerovo +RU +5601+09250 Asia/Krasnoyarsk MSK+04 - Krasnoyarsk area +RU +5216+10420 Asia/Irkutsk MSK+05 - Irkutsk, Buryatia +RU +5203+11328 Asia/Chita MSK+06 - Zabaykalsky +RU +6200+12940 Asia/Yakutsk MSK+06 - Lena River +RU +623923+1353314 Asia/Khandyga MSK+06 - Tomponsky, Ust-Maysky +RU +4310+13156 Asia/Vladivostok MSK+07 - Amur River +RU +643337+1431336 Asia/Ust-Nera MSK+07 - Oymyakonsky +RU +5934+15048 Asia/Magadan MSK+08 - Magadan +RU +4658+14242 Asia/Sakhalin MSK+08 - Sakhalin Island +RU +6728+15343 Asia/Srednekolymsk MSK+08 - Sakha (E), N Kuril Is +RU +5301+15839 Asia/Kamchatka MSK+09 - Kamchatka +RU +6445+17729 Asia/Anadyr MSK+09 - Bering Sea +RW -0157+03004 Africa/Kigali +SA +2438+04643 Asia/Riyadh +SB -0932+16012 Pacific/Guadalcanal +SC -0440+05528 Indian/Mahe +SD +1536+03232 Africa/Khartoum +SE +5920+01803 Europe/Stockholm +SG +0117+10351 Asia/Singapore +SH -1555-00542 Atlantic/St_Helena +SI +4603+01431 Europe/Ljubljana +SJ +7800+01600 Arctic/Longyearbyen +SK +4809+01707 Europe/Bratislava +SL +0830-01315 Africa/Freetown +SM +4355+01228 Europe/San_Marino +SN +1440-01726 Africa/Dakar +SO +0204+04522 Africa/Mogadishu +SR +0550-05510 America/Paramaribo +SS +0451+03137 Africa/Juba +ST +0020+00644 Africa/Sao_Tome +SV +1342-08912 America/El_Salvador +SX +180305-0630250 America/Lower_Princes +SY +3330+03618 Asia/Damascus +SZ -2618+03106 Africa/Mbabane +TC +2128-07108 America/Grand_Turk +TD +1207+01503 Africa/Ndjamena +TF -492110+0701303 Indian/Kerguelen +TG +0608+00113 Africa/Lome +TH +1345+10031 Asia/Bangkok +TJ +3835+06848 Asia/Dushanbe +TK -0922-17114 Pacific/Fakaofo +TL -0833+12535 Asia/Dili +TM +3757+05823 Asia/Ashgabat +TN +3648+01011 Africa/Tunis +TO -210800-1751200 Pacific/Tongatapu +TR +4101+02858 Europe/Istanbul +TT +1039-06131 America/Port_of_Spain +TV -0831+17913 Pacific/Funafuti +TW +2503+12130 Asia/Taipei +TZ -0648+03917 Africa/Dar_es_Salaam +UA +5026+03031 Europe/Kyiv most of Ukraine +UG +0019+03225 Africa/Kampala +UM +2813-17722 Pacific/Midway Midway Islands +UM +1917+16637 Pacific/Wake Wake Island +US +404251-0740023 America/New_York Eastern (most areas) +US +421953-0830245 America/Detroit Eastern - MI (most areas) +US +381515-0854534 America/Kentucky/Louisville Eastern - KY (Louisville area) +US +364947-0845057 America/Kentucky/Monticello Eastern - KY (Wayne) +US +394606-0860929 America/Indiana/Indianapolis Eastern - IN (most areas) +US +384038-0873143 America/Indiana/Vincennes Eastern - IN (Da, Du, K, Mn) +US +410305-0863611 America/Indiana/Winamac Eastern - IN (Pulaski) +US +382232-0862041 America/Indiana/Marengo Eastern - IN (Crawford) +US +382931-0871643 America/Indiana/Petersburg Eastern - IN (Pike) +US +384452-0850402 America/Indiana/Vevay Eastern - IN (Switzerland) +US +415100-0873900 America/Chicago Central (most areas) +US +375711-0864541 America/Indiana/Tell_City Central - IN (Perry) +US +411745-0863730 America/Indiana/Knox Central - IN (Starke) +US +450628-0873651 America/Menominee Central - MI (Wisconsin border) +US +470659-1011757 America/North_Dakota/Center Central - ND (Oliver) +US +465042-1012439 America/North_Dakota/New_Salem Central - ND (Morton rural) +US +471551-1014640 America/North_Dakota/Beulah Central - ND (Mercer) +US +394421-1045903 America/Denver Mountain (most areas) +US +433649-1161209 America/Boise Mountain - ID (south), OR (east) +US +332654-1120424 America/Phoenix MST - AZ (except Navajo) +US +340308-1181434 America/Los_Angeles Pacific +US +611305-1495401 America/Anchorage Alaska (most areas) +US +581807-1342511 America/Juneau Alaska - Juneau area +US +571035-1351807 America/Sitka Alaska - Sitka area +US +550737-1313435 America/Metlakatla Alaska - Annette Island +US +593249-1394338 America/Yakutat Alaska - Yakutat +US +643004-1652423 America/Nome Alaska (west) +US +515248-1763929 America/Adak Alaska - western Aleutians +US +211825-1575130 Pacific/Honolulu Hawaii +UY -345433-0561245 America/Montevideo +UZ +3940+06648 Asia/Samarkand Uzbekistan (west) +UZ +4120+06918 Asia/Tashkent Uzbekistan (east) +VA +415408+0122711 Europe/Vatican +VC +1309-06114 America/St_Vincent +VE +1030-06656 America/Caracas +VG +1827-06437 America/Tortola +VI +1821-06456 America/St_Thomas +VN +1045+10640 Asia/Ho_Chi_Minh +VU -1740+16825 Pacific/Efate +WF -1318-17610 Pacific/Wallis +WS -1350-17144 Pacific/Apia +YE +1245+04512 Asia/Aden +YT -1247+04514 Indian/Mayotte +ZA -2615+02800 Africa/Johannesburg +ZM -1525+02817 Africa/Lusaka +ZW -1750+03103 Africa/Harare diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/zone1970.tab b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/zone1970.tab new file mode 100644 index 0000000000000000000000000000000000000000..5ded0565ebf31764012b71420e4da1192ac5378c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/zone1970.tab @@ -0,0 +1,374 @@ +# tzdb timezone descriptions +# +# This file is in the public domain. +# +# From Paul Eggert (2018-06-27): +# This file contains a table where each row stands for a timezone where +# civil timestamps have agreed since 1970. Columns are separated by +# a single tab. Lines beginning with '#' are comments. All text uses +# UTF-8 encoding. The columns of the table are as follows: +# +# 1. The countries that overlap the timezone, as a comma-separated list +# of ISO 3166 2-character country codes. See the file 'iso3166.tab'. +# 2. Latitude and longitude of the timezone's principal location +# in ISO 6709 sign-degrees-minutes-seconds format, +# either ±DDMM±DDDMM or ±DDMMSS±DDDMMSS, +# first latitude (+ is north), then longitude (+ is east). +# 3. Timezone name used in value of TZ environment variable. +# Please see the theory.html file for how these names are chosen. +# If multiple timezones overlap a country, each has a row in the +# table, with each column 1 containing the country code. +# 4. Comments; present if and only if countries have multiple timezones, +# and useful only for those countries. For example, the comments +# for the row with countries CH,DE,LI and name Europe/Zurich +# are useful only for DE, since CH and LI have no other timezones. +# +# If a timezone covers multiple countries, the most-populous city is used, +# and that country is listed first in column 1; any other countries +# are listed alphabetically by country code. The table is sorted +# first by country code, then (if possible) by an order within the +# country that (1) makes some geographical sense, and (2) puts the +# most populous timezones first, where that does not contradict (1). +# +# This table is intended as an aid for users, to help them select timezones +# appropriate for their practical needs. It is not intended to take or +# endorse any position on legal or territorial claims. +# +#country- +#codes coordinates TZ comments +AD +4230+00131 Europe/Andorra +AE,OM,RE,SC,TF +2518+05518 Asia/Dubai Crozet +AF +3431+06912 Asia/Kabul +AL +4120+01950 Europe/Tirane +AM +4011+04430 Asia/Yerevan +AQ -6617+11031 Antarctica/Casey Casey +AQ -6835+07758 Antarctica/Davis Davis +AQ -6736+06253 Antarctica/Mawson Mawson +AQ -6448-06406 Antarctica/Palmer Palmer +AQ -6734-06808 Antarctica/Rothera Rothera +AQ -720041+0023206 Antarctica/Troll Troll +AQ -7824+10654 Antarctica/Vostok Vostok +AR -3436-05827 America/Argentina/Buenos_Aires Buenos Aires (BA, CF) +AR -3124-06411 America/Argentina/Cordoba most areas: CB, CC, CN, ER, FM, MN, SE, SF +AR -2447-06525 America/Argentina/Salta Salta (SA, LP, NQ, RN) +AR -2411-06518 America/Argentina/Jujuy Jujuy (JY) +AR -2649-06513 America/Argentina/Tucuman Tucumán (TM) +AR -2828-06547 America/Argentina/Catamarca Catamarca (CT), Chubut (CH) +AR -2926-06651 America/Argentina/La_Rioja La Rioja (LR) +AR -3132-06831 America/Argentina/San_Juan San Juan (SJ) +AR -3253-06849 America/Argentina/Mendoza Mendoza (MZ) +AR -3319-06621 America/Argentina/San_Luis San Luis (SL) +AR -5138-06913 America/Argentina/Rio_Gallegos Santa Cruz (SC) +AR -5448-06818 America/Argentina/Ushuaia Tierra del Fuego (TF) +AS,UM -1416-17042 Pacific/Pago_Pago Midway +AT +4813+01620 Europe/Vienna +AU -3133+15905 Australia/Lord_Howe Lord Howe Island +AU -5430+15857 Antarctica/Macquarie Macquarie Island +AU -4253+14719 Australia/Hobart Tasmania +AU -3749+14458 Australia/Melbourne Victoria +AU -3352+15113 Australia/Sydney New South Wales (most areas) +AU -3157+14127 Australia/Broken_Hill New South Wales (Yancowinna) +AU -2728+15302 Australia/Brisbane Queensland (most areas) +AU -2016+14900 Australia/Lindeman Queensland (Whitsunday Islands) +AU -3455+13835 Australia/Adelaide South Australia +AU -1228+13050 Australia/Darwin Northern Territory +AU -3157+11551 Australia/Perth Western Australia (most areas) +AU -3143+12852 Australia/Eucla Western Australia (Eucla) +AZ +4023+04951 Asia/Baku +BB +1306-05937 America/Barbados +BD +2343+09025 Asia/Dhaka +BE,LU,NL +5050+00420 Europe/Brussels +BG +4241+02319 Europe/Sofia +BM +3217-06446 Atlantic/Bermuda +BO -1630-06809 America/La_Paz +BR -0351-03225 America/Noronha Atlantic islands +BR -0127-04829 America/Belem Pará (east), Amapá +BR -0343-03830 America/Fortaleza Brazil (northeast: MA, PI, CE, RN, PB) +BR -0803-03454 America/Recife Pernambuco +BR -0712-04812 America/Araguaina Tocantins +BR -0940-03543 America/Maceio Alagoas, Sergipe +BR -1259-03831 America/Bahia Bahia +BR -2332-04637 America/Sao_Paulo Brazil (southeast: GO, DF, MG, ES, RJ, SP, PR, SC, RS) +BR -2027-05437 America/Campo_Grande Mato Grosso do Sul +BR -1535-05605 America/Cuiaba Mato Grosso +BR -0226-05452 America/Santarem Pará (west) +BR -0846-06354 America/Porto_Velho Rondônia +BR +0249-06040 America/Boa_Vista Roraima +BR -0308-06001 America/Manaus Amazonas (east) +BR -0640-06952 America/Eirunepe Amazonas (west) +BR -0958-06748 America/Rio_Branco Acre +BT +2728+08939 Asia/Thimphu +BY +5354+02734 Europe/Minsk +BZ +1730-08812 America/Belize +CA +4734-05243 America/St_Johns Newfoundland, Labrador (SE) +CA +4439-06336 America/Halifax Atlantic - NS (most areas), PE +CA +4612-05957 America/Glace_Bay Atlantic - NS (Cape Breton) +CA +4606-06447 America/Moncton Atlantic - New Brunswick +CA +5320-06025 America/Goose_Bay Atlantic - Labrador (most areas) +CA,BS +4339-07923 America/Toronto Eastern - ON & QC (most areas) +CA +6344-06828 America/Iqaluit Eastern - NU (most areas) +CA +4953-09709 America/Winnipeg Central - ON (west), Manitoba +CA +744144-0944945 America/Resolute Central - NU (Resolute) +CA +624900-0920459 America/Rankin_Inlet Central - NU (central) +CA +5024-10439 America/Regina CST - SK (most areas) +CA +5017-10750 America/Swift_Current CST - SK (midwest) +CA +5333-11328 America/Edmonton Mountain - AB, BC(E), NT(E), SK(W) +CA +690650-1050310 America/Cambridge_Bay Mountain - NU (west) +CA +682059-1334300 America/Inuvik Mountain - NT (west) +CA +5546-12014 America/Dawson_Creek MST - BC (Dawson Cr, Ft St John) +CA +5848-12242 America/Fort_Nelson MST - BC (Ft Nelson) +CA +6043-13503 America/Whitehorse MST - Yukon (east) +CA +6404-13925 America/Dawson MST - Yukon (west) +CA +4916-12307 America/Vancouver Pacific - BC (most areas) +CH,DE,LI +4723+00832 Europe/Zurich Büsingen +CI,BF,GH,GM,GN,IS,ML,MR,SH,SL,SN,TG +0519-00402 Africa/Abidjan +CK -2114-15946 Pacific/Rarotonga +CL -3327-07040 America/Santiago most of Chile +CL -5309-07055 America/Punta_Arenas Region of Magallanes +CL -2709-10926 Pacific/Easter Easter Island +CN +3114+12128 Asia/Shanghai Beijing Time +CN +4348+08735 Asia/Urumqi Xinjiang Time +CO +0436-07405 America/Bogota +CR +0956-08405 America/Costa_Rica +CU +2308-08222 America/Havana +CV +1455-02331 Atlantic/Cape_Verde +CY +3510+03322 Asia/Nicosia most of Cyprus +CY +3507+03357 Asia/Famagusta Northern Cyprus +CZ,SK +5005+01426 Europe/Prague +DE,DK,NO,SE,SJ +5230+01322 Europe/Berlin most of Germany +DO +1828-06954 America/Santo_Domingo +DZ +3647+00303 Africa/Algiers +EC -0210-07950 America/Guayaquil Ecuador (mainland) +EC -0054-08936 Pacific/Galapagos Galápagos Islands +EE +5925+02445 Europe/Tallinn +EG +3003+03115 Africa/Cairo +EH +2709-01312 Africa/El_Aaiun +ES +4024-00341 Europe/Madrid Spain (mainland) +ES +3553-00519 Africa/Ceuta Ceuta, Melilla +ES +2806-01524 Atlantic/Canary Canary Islands +FI,AX +6010+02458 Europe/Helsinki +FJ -1808+17825 Pacific/Fiji +FK -5142-05751 Atlantic/Stanley +FM +0519+16259 Pacific/Kosrae Kosrae +FO +6201-00646 Atlantic/Faroe +FR,MC +4852+00220 Europe/Paris +GB,GG,IM,JE +513030-0000731 Europe/London +GE +4143+04449 Asia/Tbilisi +GF +0456-05220 America/Cayenne +GI +3608-00521 Europe/Gibraltar +GL +6411-05144 America/Nuuk most of Greenland +GL +7646-01840 America/Danmarkshavn National Park (east coast) +GL +7029-02158 America/Scoresbysund Scoresbysund/Ittoqqortoormiit +GL +7634-06847 America/Thule Thule/Pituffik +GR +3758+02343 Europe/Athens +GS -5416-03632 Atlantic/South_Georgia +GT +1438-09031 America/Guatemala +GU,MP +1328+14445 Pacific/Guam +GW +1151-01535 Africa/Bissau +GY +0648-05810 America/Guyana +HK +2217+11409 Asia/Hong_Kong +HN +1406-08713 America/Tegucigalpa +HT +1832-07220 America/Port-au-Prince +HU +4730+01905 Europe/Budapest +ID -0610+10648 Asia/Jakarta Java, Sumatra +ID -0002+10920 Asia/Pontianak Borneo (west, central) +ID -0507+11924 Asia/Makassar Borneo (east, south), Sulawesi/Celebes, Bali, Nusa Tengarra, Timor (west) +ID -0232+14042 Asia/Jayapura New Guinea (West Papua / Irian Jaya), Malukus/Moluccas +IE +5320-00615 Europe/Dublin +IL +314650+0351326 Asia/Jerusalem +IN +2232+08822 Asia/Kolkata +IO -0720+07225 Indian/Chagos +IQ +3321+04425 Asia/Baghdad +IR +3540+05126 Asia/Tehran +IT,SM,VA +4154+01229 Europe/Rome +JM +175805-0764736 America/Jamaica +JO +3157+03556 Asia/Amman +JP,AU +353916+1394441 Asia/Tokyo Eyre Bird Observatory +KE,DJ,ER,ET,KM,MG,SO,TZ,UG,YT -0117+03649 Africa/Nairobi +KG +4254+07436 Asia/Bishkek +KI,MH,TV,UM,WF +0125+17300 Pacific/Tarawa Gilberts, Marshalls, Wake +KI -0247-17143 Pacific/Kanton Phoenix Islands +KI +0152-15720 Pacific/Kiritimati Line Islands +KP +3901+12545 Asia/Pyongyang +KR +3733+12658 Asia/Seoul +KZ +4315+07657 Asia/Almaty most of Kazakhstan +KZ +4448+06528 Asia/Qyzylorda Qyzylorda/Kyzylorda/Kzyl-Orda +KZ +5312+06337 Asia/Qostanay Qostanay/Kostanay/Kustanay +KZ +5017+05710 Asia/Aqtobe Aqtöbe/Aktobe +KZ +4431+05016 Asia/Aqtau Mangghystaū/Mankistau +KZ +4707+05156 Asia/Atyrau Atyraū/Atirau/Gur'yev +KZ +5113+05121 Asia/Oral West Kazakhstan +LB +3353+03530 Asia/Beirut +LK +0656+07951 Asia/Colombo +LR +0618-01047 Africa/Monrovia +LT +5441+02519 Europe/Vilnius +LV +5657+02406 Europe/Riga +LY +3254+01311 Africa/Tripoli +MA +3339-00735 Africa/Casablanca +MD +4700+02850 Europe/Chisinau +MH +0905+16720 Pacific/Kwajalein Kwajalein +MM,CC +1647+09610 Asia/Yangon +MN +4755+10653 Asia/Ulaanbaatar most of Mongolia +MN +4801+09139 Asia/Hovd Bayan-Ölgii, Hovd, Uvs +MO +221150+1133230 Asia/Macau +MQ +1436-06105 America/Martinique +MT +3554+01431 Europe/Malta +MU -2010+05730 Indian/Mauritius +MV,TF +0410+07330 Indian/Maldives Kerguelen, St Paul I, Amsterdam I +MX +1924-09909 America/Mexico_City Central Mexico +MX +2105-08646 America/Cancun Quintana Roo +MX +2058-08937 America/Merida Campeche, Yucatán +MX +2540-10019 America/Monterrey Durango; Coahuila, Nuevo León, Tamaulipas (most areas) +MX +2550-09730 America/Matamoros Coahuila, Nuevo León, Tamaulipas (US border) +MX +2838-10605 America/Chihuahua Chihuahua (most areas) +MX +3144-10629 America/Ciudad_Juarez Chihuahua (US border - west) +MX +2934-10425 America/Ojinaga Chihuahua (US border - east) +MX +2313-10625 America/Mazatlan Baja California Sur, Nayarit (most areas), Sinaloa +MX +2048-10515 America/Bahia_Banderas Bahía de Banderas +MX +2904-11058 America/Hermosillo Sonora +MX +3232-11701 America/Tijuana Baja California +MY,BN +0133+11020 Asia/Kuching Sabah, Sarawak +MZ,BI,BW,CD,MW,RW,ZM,ZW -2558+03235 Africa/Maputo Central Africa Time +NA -2234+01706 Africa/Windhoek +NC -2216+16627 Pacific/Noumea +NF -2903+16758 Pacific/Norfolk +NG,AO,BJ,CD,CF,CG,CM,GA,GQ,NE +0627+00324 Africa/Lagos West Africa Time +NI +1209-08617 America/Managua +NP +2743+08519 Asia/Kathmandu +NR -0031+16655 Pacific/Nauru +NU -1901-16955 Pacific/Niue +NZ,AQ -3652+17446 Pacific/Auckland New Zealand time +NZ -4357-17633 Pacific/Chatham Chatham Islands +PA,CA,KY +0858-07932 America/Panama EST - ON (Atikokan), NU (Coral H) +PE -1203-07703 America/Lima +PF -1732-14934 Pacific/Tahiti Society Islands +PF -0900-13930 Pacific/Marquesas Marquesas Islands +PF -2308-13457 Pacific/Gambier Gambier Islands +PG,AQ,FM -0930+14710 Pacific/Port_Moresby Papua New Guinea (most areas), Chuuk, Yap, Dumont d'Urville +PG -0613+15534 Pacific/Bougainville Bougainville +PH +143512+1205804 Asia/Manila +PK +2452+06703 Asia/Karachi +PL +5215+02100 Europe/Warsaw +PM +4703-05620 America/Miquelon +PN -2504-13005 Pacific/Pitcairn +PR,AG,CA,AI,AW,BL,BQ,CW,DM,GD,GP,KN,LC,MF,MS,SX,TT,VC,VG,VI +182806-0660622 America/Puerto_Rico AST - QC (Lower North Shore) +PS +3130+03428 Asia/Gaza Gaza Strip +PS +313200+0350542 Asia/Hebron West Bank +PT +3843-00908 Europe/Lisbon Portugal (mainland) +PT +3238-01654 Atlantic/Madeira Madeira Islands +PT +3744-02540 Atlantic/Azores Azores +PW +0720+13429 Pacific/Palau +PY -2516-05740 America/Asuncion +QA,BH +2517+05132 Asia/Qatar +RO +4426+02606 Europe/Bucharest +RS,BA,HR,ME,MK,SI +4450+02030 Europe/Belgrade +RU +5443+02030 Europe/Kaliningrad MSK-01 - Kaliningrad +RU +554521+0373704 Europe/Moscow MSK+00 - Moscow area +# Mention RU and UA alphabetically. See "territorial claims" above. +RU,UA +4457+03406 Europe/Simferopol Crimea +RU +5836+04939 Europe/Kirov MSK+00 - Kirov +RU +4844+04425 Europe/Volgograd MSK+00 - Volgograd +RU +4621+04803 Europe/Astrakhan MSK+01 - Astrakhan +RU +5134+04602 Europe/Saratov MSK+01 - Saratov +RU +5420+04824 Europe/Ulyanovsk MSK+01 - Ulyanovsk +RU +5312+05009 Europe/Samara MSK+01 - Samara, Udmurtia +RU +5651+06036 Asia/Yekaterinburg MSK+02 - Urals +RU +5500+07324 Asia/Omsk MSK+03 - Omsk +RU +5502+08255 Asia/Novosibirsk MSK+04 - Novosibirsk +RU +5322+08345 Asia/Barnaul MSK+04 - Altai +RU +5630+08458 Asia/Tomsk MSK+04 - Tomsk +RU +5345+08707 Asia/Novokuznetsk MSK+04 - Kemerovo +RU +5601+09250 Asia/Krasnoyarsk MSK+04 - Krasnoyarsk area +RU +5216+10420 Asia/Irkutsk MSK+05 - Irkutsk, Buryatia +RU +5203+11328 Asia/Chita MSK+06 - Zabaykalsky +RU +6200+12940 Asia/Yakutsk MSK+06 - Lena River +RU +623923+1353314 Asia/Khandyga MSK+06 - Tomponsky, Ust-Maysky +RU +4310+13156 Asia/Vladivostok MSK+07 - Amur River +RU +643337+1431336 Asia/Ust-Nera MSK+07 - Oymyakonsky +RU +5934+15048 Asia/Magadan MSK+08 - Magadan +RU +4658+14242 Asia/Sakhalin MSK+08 - Sakhalin Island +RU +6728+15343 Asia/Srednekolymsk MSK+08 - Sakha (E), N Kuril Is +RU +5301+15839 Asia/Kamchatka MSK+09 - Kamchatka +RU +6445+17729 Asia/Anadyr MSK+09 - Bering Sea +SA,AQ,KW,YE +2438+04643 Asia/Riyadh Syowa +SB,FM -0932+16012 Pacific/Guadalcanal Pohnpei +SD +1536+03232 Africa/Khartoum +SG,AQ,MY +0117+10351 Asia/Singapore peninsular Malaysia, Concordia +SR +0550-05510 America/Paramaribo +SS +0451+03137 Africa/Juba +ST +0020+00644 Africa/Sao_Tome +SV +1342-08912 America/El_Salvador +SY +3330+03618 Asia/Damascus +TC +2128-07108 America/Grand_Turk +TD +1207+01503 Africa/Ndjamena +TH,CX,KH,LA,VN +1345+10031 Asia/Bangkok north Vietnam +TJ +3835+06848 Asia/Dushanbe +TK -0922-17114 Pacific/Fakaofo +TL -0833+12535 Asia/Dili +TM +3757+05823 Asia/Ashgabat +TN +3648+01011 Africa/Tunis +TO -210800-1751200 Pacific/Tongatapu +TR +4101+02858 Europe/Istanbul +TW +2503+12130 Asia/Taipei +UA +5026+03031 Europe/Kyiv most of Ukraine +US +404251-0740023 America/New_York Eastern (most areas) +US +421953-0830245 America/Detroit Eastern - MI (most areas) +US +381515-0854534 America/Kentucky/Louisville Eastern - KY (Louisville area) +US +364947-0845057 America/Kentucky/Monticello Eastern - KY (Wayne) +US +394606-0860929 America/Indiana/Indianapolis Eastern - IN (most areas) +US +384038-0873143 America/Indiana/Vincennes Eastern - IN (Da, Du, K, Mn) +US +410305-0863611 America/Indiana/Winamac Eastern - IN (Pulaski) +US +382232-0862041 America/Indiana/Marengo Eastern - IN (Crawford) +US +382931-0871643 America/Indiana/Petersburg Eastern - IN (Pike) +US +384452-0850402 America/Indiana/Vevay Eastern - IN (Switzerland) +US +415100-0873900 America/Chicago Central (most areas) +US +375711-0864541 America/Indiana/Tell_City Central - IN (Perry) +US +411745-0863730 America/Indiana/Knox Central - IN (Starke) +US +450628-0873651 America/Menominee Central - MI (Wisconsin border) +US +470659-1011757 America/North_Dakota/Center Central - ND (Oliver) +US +465042-1012439 America/North_Dakota/New_Salem Central - ND (Morton rural) +US +471551-1014640 America/North_Dakota/Beulah Central - ND (Mercer) +US +394421-1045903 America/Denver Mountain (most areas) +US +433649-1161209 America/Boise Mountain - ID (south), OR (east) +US,CA +332654-1120424 America/Phoenix MST - AZ (most areas), Creston BC +US +340308-1181434 America/Los_Angeles Pacific +US +611305-1495401 America/Anchorage Alaska (most areas) +US +581807-1342511 America/Juneau Alaska - Juneau area +US +571035-1351807 America/Sitka Alaska - Sitka area +US +550737-1313435 America/Metlakatla Alaska - Annette Island +US +593249-1394338 America/Yakutat Alaska - Yakutat +US +643004-1652423 America/Nome Alaska (west) +US +515248-1763929 America/Adak Alaska - western Aleutians +US +211825-1575130 Pacific/Honolulu Hawaii +UY -345433-0561245 America/Montevideo +UZ +3940+06648 Asia/Samarkand Uzbekistan (west) +UZ +4120+06918 Asia/Tashkent Uzbekistan (east) +VE +1030-06656 America/Caracas +VN +1045+10640 Asia/Ho_Chi_Minh south Vietnam +VU -1740+16825 Pacific/Efate +WS -1350-17144 Pacific/Apia +ZA,LS,SZ -2615+02800 Africa/Johannesburg +# +# The next section contains experimental tab-separated comments for +# use by user agents like tzselect that identify continents and oceans. +# +# For example, the comment "#@AQAntarctica/" means the country code +# AQ is in the continent Antarctica regardless of the Zone name, +# so Pacific/Auckland should be listed under Antarctica as well as +# under the Pacific because its line's country codes include AQ. +# +# If more than one country code is affected each is listed separated +# by commas, e.g., #@IS,SHAtlantic/". If a country code is in +# more than one continent or ocean, each is listed separated by +# commas, e.g., the second column of "#@CY,TRAsia/,Europe/". +# +# These experimental comments are present only for country codes where +# the continent or ocean is not already obvious from the Zone name. +# For example, there is no such comment for RU since it already +# corresponds to Zone names starting with both "Europe/" and "Asia/". +# +#@AQ Antarctica/ +#@IS,SH Atlantic/ +#@CY,TR Asia/,Europe/ +#@SJ Arctic/ +#@CC,CX,KM,MG,YT Indian/ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/zonenow.tab b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/zonenow.tab new file mode 100644 index 0000000000000000000000000000000000000000..d2c1e48584f8f8f5d7a0e48ead4e557490d4197f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/tzdata/zoneinfo/zonenow.tab @@ -0,0 +1,296 @@ +# tzdb timezone descriptions, for users who do not care about old timestamps +# +# This file is in the public domain. +# +# From Paul Eggert (2023-12-18): +# This file contains a table where each row stands for a timezone +# where civil timestamps are predicted to agree from now on. +# This file is like zone1970.tab (see zone1970.tab's comments), +# but with the following changes: +# +# 1. Each timezone corresponds to a set of clocks that are planned +# to agree from now on. This is a larger set of clocks than in +# zone1970.tab, where each timezone's clocks must agree from 1970 on. +# 2. The first column is irrelevant and ignored. +# 3. The table is sorted in a different way: +# first by standard time UTC offset; +# then, if DST is used, by daylight saving UTC offset; +# then by time zone abbreviation. +# 4. Every timezone has a nonempty comments column, with wording +# distinguishing the timezone only from other timezones with the +# same UTC offset at some point during the year. +# +# The format of this table is experimental, and may change in future versions. +# +# This table is intended as an aid for users, to help them select timezones +# appropriate for their practical needs. It is not intended to take or +# endorse any position on legal or territorial claims. +# +#XX coordinates TZ comments +# +# -11 - SST +XX -1416-17042 Pacific/Pago_Pago Midway; Samoa ("SST") +# +# -11 +XX -1901-16955 Pacific/Niue Niue +# +# -10 - HST +XX +211825-1575130 Pacific/Honolulu Hawaii ("HST") +# +# -10 +XX -1732-14934 Pacific/Tahiti Tahiti; Cook Islands +# +# -10/-09 - HST / HDT (North America DST) +XX +515248-1763929 America/Adak western Aleutians in Alaska ("HST/HDT") +# +# -09:30 +XX -0900-13930 Pacific/Marquesas Marquesas +# +# -09 +XX -2308-13457 Pacific/Gambier Gambier +# +# -09/-08 - AKST/AKDT (North America DST) +XX +611305-1495401 America/Anchorage most of Alaska ("AKST/AKDT") +# +# -08 +XX -2504-13005 Pacific/Pitcairn Pitcairn +# +# -08/-07 - PST/PDT (North America DST) +XX +340308-1181434 America/Los_Angeles Pacific ("PST/PDT") - US & Canada; Mexico near US border +# +# -07 - MST +XX +332654-1120424 America/Phoenix Mountain Standard ("MST") - Arizona; western Mexico; Yukon +# +# -07/-06 - MST/MDT (North America DST) +XX +394421-1045903 America/Denver Mountain ("MST/MDT") - US & Canada; Mexico near US border +# +# -06 +XX -0054-08936 Pacific/Galapagos Galápagos +# +# -06 - CST +XX +1924-09909 America/Mexico_City Central Standard ("CST") - Saskatchewan; central Mexico; Central America +# +# -06/-05 (Chile DST) +XX -2709-10926 Pacific/Easter Easter Island +# +# -06/-05 - CST/CDT (North America DST) +XX +415100-0873900 America/Chicago Central ("CST/CDT") - US & Canada; Mexico near US border +# +# -05 +XX -1203-07703 America/Lima eastern South America +# +# -05 - EST +XX +175805-0764736 America/Jamaica Eastern Standard ("EST") - Caymans; Jamaica; eastern Mexico; Panama +# +# -05/-04 - CST/CDT (Cuba DST) +XX +2308-08222 America/Havana Cuba +# +# -05/-04 - EST/EDT (North America DST) +XX +404251-0740023 America/New_York Eastern ("EST/EDT") - US & Canada +# +# -04 +XX +1030-06656 America/Caracas western South America +# +# -04 - AST +XX +1828-06954 America/Santo_Domingo Atlantic Standard ("AST") - eastern Caribbean +# +# -04/-03 (Chile DST) +XX -3327-07040 America/Santiago most of Chile +# +# -04/-03 - AST/ADT (North America DST) +XX +4439-06336 America/Halifax Atlantic ("AST/ADT") - Canada; Bermuda +# +# -03:30/-02:30 - NST/NDT (North America DST) +XX +4734-05243 America/St_Johns Newfoundland ("NST/NDT") +# +# -03 +XX -2332-04637 America/Sao_Paulo eastern South America +# +# -03/-02 (North America DST) +XX +4703-05620 America/Miquelon St Pierre & Miquelon +# +# -02 +XX -0351-03225 America/Noronha Fernando de Noronha; South Georgia +# +# -02/-01 (EU DST) +XX +6411-05144 America/Nuuk most of Greenland +# +# -01 +XX +1455-02331 Atlantic/Cape_Verde Cape Verde +# +# -01/+00 (EU DST) +XX +3744-02540 Atlantic/Azores Azores +# +# +00 - GMT +XX +0519-00402 Africa/Abidjan far western Africa; Iceland ("GMT") +# +# +00/+01 - GMT/BST (EU DST) +XX +513030-0000731 Europe/London United Kingdom ("GMT/BST") +# +# +00/+01 - WET/WEST (EU DST) +XX +3843-00908 Europe/Lisbon western Europe ("WET/WEST") +# +# +00/+02 - Troll DST +XX -720041+0023206 Antarctica/Troll Troll Station in Antarctica +# +# +01 - CET +XX +3647+00303 Africa/Algiers Algeria, Tunisia ("CET") +# +# +01 - WAT +XX +0627+00324 Africa/Lagos western Africa ("WAT") +# +# +01/+00 - IST/GMT (EU DST in reverse) +XX +5320-00615 Europe/Dublin Ireland ("IST/GMT") +# +# +01/+00 - (Morocco DST) +XX +3339-00735 Africa/Casablanca Morocco +# +# +01/+02 - CET/CEST (EU DST) +XX +4852+00220 Europe/Paris central Europe ("CET/CEST") +# +# +02 - CAT +XX -2558+03235 Africa/Maputo central Africa ("CAT") +# +# +02 - EET +XX +3254+01311 Africa/Tripoli Libya; Kaliningrad ("EET") +# +# +02 - SAST +XX -2615+02800 Africa/Johannesburg southern Africa ("SAST") +# +# +02/+03 - EET/EEST (EU DST) +XX +3758+02343 Europe/Athens eastern Europe ("EET/EEST") +# +# +02/+03 - EET/EEST (Egypt DST) +XX +3003+03115 Africa/Cairo Egypt +# +# +02/+03 - EET/EEST (Lebanon DST) +XX +3353+03530 Asia/Beirut Lebanon +# +# +02/+03 - EET/EEST (Moldova DST) +XX +4700+02850 Europe/Chisinau Moldova +# +# +02/+03 - EET/EEST (Palestine DST) +XX +3130+03428 Asia/Gaza Palestine +# +# +02/+03 - IST/IDT (Israel DST) +XX +314650+0351326 Asia/Jerusalem Israel +# +# +03 +XX +4101+02858 Europe/Istanbul Near East; Belarus +# +# +03 - EAT +XX -0117+03649 Africa/Nairobi eastern Africa ("EAT") +# +# +03 - MSK +XX +554521+0373704 Europe/Moscow Moscow ("MSK") +# +# +03:30 +XX +3540+05126 Asia/Tehran Iran +# +# +04 +XX +2518+05518 Asia/Dubai Russia; Caucasus; Persian Gulf; Seychelles; Réunion +# +# +04:30 +XX +3431+06912 Asia/Kabul Afghanistan +# +# +05 +XX +4120+06918 Asia/Tashkent Russia; Kazakhstan; Tajikistan; Turkmenistan; Uzbekistan; Maldives +# +# +05 - PKT +XX +2452+06703 Asia/Karachi Pakistan ("PKT") +# +# +05:30 +XX +0656+07951 Asia/Colombo Sri Lanka +# +# +05:30 - IST +XX +2232+08822 Asia/Kolkata India ("IST") +# +# +05:45 +XX +2743+08519 Asia/Kathmandu Nepal +# +# +06 +XX +2343+09025 Asia/Dhaka Russia; Kyrgyzstan; Bhutan; Bangladesh; Chagos +# +# +06:30 +XX +1647+09610 Asia/Yangon Myanmar; Cocos +# +# +07 +XX +1345+10031 Asia/Bangkok Russia; Indochina; Christmas Island +# +# +07 - WIB +XX -0610+10648 Asia/Jakarta Indonesia ("WIB") +# +# +08 +XX +0117+10351 Asia/Singapore Russia; Brunei; Malaysia; Singapore; Concordia +# +# +08 - AWST +XX -3157+11551 Australia/Perth Western Australia ("AWST") +# +# +08 - CST +XX +3114+12128 Asia/Shanghai China ("CST") +# +# +08 - HKT +XX +2217+11409 Asia/Hong_Kong Hong Kong ("HKT") +# +# +08 - PHT +XX +143512+1205804 Asia/Manila Philippines ("PHT") +# +# +08 - WITA +XX -0507+11924 Asia/Makassar Indonesia ("WITA") +# +# +08:45 +XX -3143+12852 Australia/Eucla Eucla +# +# +09 +XX +5203+11328 Asia/Chita Russia; Palau; East Timor +# +# +09 - JST +XX +353916+1394441 Asia/Tokyo Japan ("JST"); Eyre Bird Observatory +# +# +09 - KST +XX +3733+12658 Asia/Seoul Korea ("KST") +# +# +09 - WIT +XX -0232+14042 Asia/Jayapura Indonesia ("WIT") +# +# +09:30 - ACST +XX -1228+13050 Australia/Darwin Northern Territory ("ACST") +# +# +09:30/+10:30 - ACST/ACDT (Australia DST) +XX -3455+13835 Australia/Adelaide South Australia ("ACST/ACDT") +# +# +10 +XX +4310+13156 Asia/Vladivostok Russia; Yap; Chuuk; Papua New Guinea; Dumont d'Urville +# +# +10 - AEST +XX -2728+15302 Australia/Brisbane Queensland ("AEST") +# +# +10 - ChST +XX +1328+14445 Pacific/Guam Mariana Islands ("ChST") +# +# +10/+11 - AEST/AEDT (Australia DST) +XX -3352+15113 Australia/Sydney southeast Australia ("AEST/AEDT") +# +# +10:30/+11 +XX -3133+15905 Australia/Lord_Howe Lord Howe Island +# +# +11 +XX -0613+15534 Pacific/Bougainville Russia; Kosrae; Bougainville; Solomons +# +# +11/+12 (Australia DST) +XX -2903+16758 Pacific/Norfolk Norfolk Island +# +# +12 +XX +5301+15839 Asia/Kamchatka Russia; Tuvalu; Fiji; etc. +# +# +12/+13 (New Zealand DST) +XX -3652+17446 Pacific/Auckland New Zealand ("NZST/NZDT") +# +# +12:45/+13:45 (Chatham DST) +XX -4357-17633 Pacific/Chatham Chatham Islands +# +# +13 +XX -210800-1751200 Pacific/Tongatapu Kanton; Tokelau; Samoa (western); Tonga +# +# +14 +XX +0152-15720 Pacific/Kiritimati Kiritimati diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/share/man/man1/isympy.1 b/Prism/LLaDA/LLaDA_Prism/.venv/share/man/man1/isympy.1 new file mode 100644 index 0000000000000000000000000000000000000000..0ff966158a28c5ad1a6cd954e454842b25fdd999 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/share/man/man1/isympy.1 @@ -0,0 +1,188 @@ +'\" -*- coding: us-ascii -*- +.if \n(.g .ds T< \\FC +.if \n(.g .ds T> \\F[\n[.fam]] +.de URL +\\$2 \(la\\$1\(ra\\$3 +.. +.if \n(.g .mso www.tmac +.TH isympy 1 2007-10-8 "" "" +.SH NAME +isympy \- interactive shell for SymPy +.SH SYNOPSIS +'nh +.fi +.ad l +\fBisympy\fR \kx +.if (\nx>(\n(.l/2)) .nr x (\n(.l/5) +'in \n(.iu+\nxu +[\fB-c\fR | \fB--console\fR] [\fB-p\fR ENCODING | \fB--pretty\fR ENCODING] [\fB-t\fR TYPE | \fB--types\fR TYPE] [\fB-o\fR ORDER | \fB--order\fR ORDER] [\fB-q\fR | \fB--quiet\fR] [\fB-d\fR | \fB--doctest\fR] [\fB-C\fR | \fB--no-cache\fR] [\fB-a\fR | \fB--auto\fR] [\fB-D\fR | \fB--debug\fR] [ +-- | PYTHONOPTIONS] +'in \n(.iu-\nxu +.ad b +'hy +'nh +.fi +.ad l +\fBisympy\fR \kx +.if (\nx>(\n(.l/2)) .nr x (\n(.l/5) +'in \n(.iu+\nxu +[ +{\fB-h\fR | \fB--help\fR} +| +{\fB-v\fR | \fB--version\fR} +] +'in \n(.iu-\nxu +.ad b +'hy +.SH DESCRIPTION +isympy is a Python shell for SymPy. It is just a normal python shell +(ipython shell if you have the ipython package installed) that executes +the following commands so that you don't have to: +.PP +.nf +\*(T< +>>> from __future__ import division +>>> from sympy import * +>>> x, y, z = symbols("x,y,z") +>>> k, m, n = symbols("k,m,n", integer=True) + \*(T> +.fi +.PP +So starting isympy is equivalent to starting python (or ipython) and +executing the above commands by hand. It is intended for easy and quick +experimentation with SymPy. For more complicated programs, it is recommended +to write a script and import things explicitly (using the "from sympy +import sin, log, Symbol, ..." idiom). +.SH OPTIONS +.TP +\*(T<\fB\-c \fR\*(T>\fISHELL\fR, \*(T<\fB\-\-console=\fR\*(T>\fISHELL\fR +Use the specified shell (python or ipython) as +console backend instead of the default one (ipython +if present or python otherwise). + +Example: isympy -c python + +\fISHELL\fR could be either +\&'ipython' or 'python' +.TP +\*(T<\fB\-p \fR\*(T>\fIENCODING\fR, \*(T<\fB\-\-pretty=\fR\*(T>\fIENCODING\fR +Setup pretty printing in SymPy. By default, the most pretty, unicode +printing is enabled (if the terminal supports it). You can use less +pretty ASCII printing instead or no pretty printing at all. + +Example: isympy -p no + +\fIENCODING\fR must be one of 'unicode', +\&'ascii' or 'no'. +.TP +\*(T<\fB\-t \fR\*(T>\fITYPE\fR, \*(T<\fB\-\-types=\fR\*(T>\fITYPE\fR +Setup the ground types for the polys. By default, gmpy ground types +are used if gmpy2 or gmpy is installed, otherwise it falls back to python +ground types, which are a little bit slower. You can manually +choose python ground types even if gmpy is installed (e.g., for testing purposes). + +Note that sympy ground types are not supported, and should be used +only for experimental purposes. + +Note that the gmpy1 ground type is primarily intended for testing; it the +use of gmpy even if gmpy2 is available. + +This is the same as setting the environment variable +SYMPY_GROUND_TYPES to the given ground type (e.g., +SYMPY_GROUND_TYPES='gmpy') + +The ground types can be determined interactively from the variable +sympy.polys.domains.GROUND_TYPES inside the isympy shell itself. + +Example: isympy -t python + +\fITYPE\fR must be one of 'gmpy', +\&'gmpy1' or 'python'. +.TP +\*(T<\fB\-o \fR\*(T>\fIORDER\fR, \*(T<\fB\-\-order=\fR\*(T>\fIORDER\fR +Setup the ordering of terms for printing. The default is lex, which +orders terms lexicographically (e.g., x**2 + x + 1). You can choose +other orderings, such as rev-lex, which will use reverse +lexicographic ordering (e.g., 1 + x + x**2). + +Note that for very large expressions, ORDER='none' may speed up +printing considerably, with the tradeoff that the order of the terms +in the printed expression will have no canonical order + +Example: isympy -o rev-lax + +\fIORDER\fR must be one of 'lex', 'rev-lex', 'grlex', +\&'rev-grlex', 'grevlex', 'rev-grevlex', 'old', or 'none'. +.TP +\*(T<\fB\-q\fR\*(T>, \*(T<\fB\-\-quiet\fR\*(T> +Print only Python's and SymPy's versions to stdout at startup, and nothing else. +.TP +\*(T<\fB\-d\fR\*(T>, \*(T<\fB\-\-doctest\fR\*(T> +Use the same format that should be used for doctests. This is +equivalent to '\fIisympy -c python -p no\fR'. +.TP +\*(T<\fB\-C\fR\*(T>, \*(T<\fB\-\-no\-cache\fR\*(T> +Disable the caching mechanism. Disabling the cache may slow certain +operations down considerably. This is useful for testing the cache, +or for benchmarking, as the cache can result in deceptive benchmark timings. + +This is the same as setting the environment variable SYMPY_USE_CACHE +to 'no'. +.TP +\*(T<\fB\-a\fR\*(T>, \*(T<\fB\-\-auto\fR\*(T> +Automatically create missing symbols. Normally, typing a name of a +Symbol that has not been instantiated first would raise NameError, +but with this option enabled, any undefined name will be +automatically created as a Symbol. This only works in IPython 0.11. + +Note that this is intended only for interactive, calculator style +usage. In a script that uses SymPy, Symbols should be instantiated +at the top, so that it's clear what they are. + +This will not override any names that are already defined, which +includes the single character letters represented by the mnemonic +QCOSINE (see the "Gotchas and Pitfalls" document in the +documentation). You can delete existing names by executing "del +name" in the shell itself. You can see if a name is defined by typing +"'name' in globals()". + +The Symbols that are created using this have default assumptions. +If you want to place assumptions on symbols, you should create them +using symbols() or var(). + +Finally, this only works in the top level namespace. So, for +example, if you define a function in isympy with an undefined +Symbol, it will not work. +.TP +\*(T<\fB\-D\fR\*(T>, \*(T<\fB\-\-debug\fR\*(T> +Enable debugging output. This is the same as setting the +environment variable SYMPY_DEBUG to 'True'. The debug status is set +in the variable SYMPY_DEBUG within isympy. +.TP +-- \fIPYTHONOPTIONS\fR +These options will be passed on to \fIipython (1)\fR shell. +Only supported when ipython is being used (standard python shell not supported). + +Two dashes (--) are required to separate \fIPYTHONOPTIONS\fR +from the other isympy options. + +For example, to run iSymPy without startup banner and colors: + +isympy -q -c ipython -- --colors=NoColor +.TP +\*(T<\fB\-h\fR\*(T>, \*(T<\fB\-\-help\fR\*(T> +Print help output and exit. +.TP +\*(T<\fB\-v\fR\*(T>, \*(T<\fB\-\-version\fR\*(T> +Print isympy version information and exit. +.SH FILES +.TP +\*(T<\fI${HOME}/.sympy\-history\fR\*(T> +Saves the history of commands when using the python +shell as backend. +.SH BUGS +The upstreams BTS can be found at \(lahttps://github.com/sympy/sympy/issues\(ra +Please report all bugs that you find in there, this will help improve +the overall quality of SymPy. +.SH "SEE ALSO" +\fBipython\fR(1), \fBpython\fR(1) diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/dllm_eval/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ee67e7255dd0afeeb60dfa4f79e1a4596960feeb Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/dllm_eval/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/__pycache__/__main__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/dllm_eval/__pycache__/__main__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3bc19e500eb0c96c492f252d8e0cd04a0546c186 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/dllm_eval/__pycache__/__main__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/__pycache__/evaluator.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/dllm_eval/__pycache__/evaluator.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5328d197edc5817a8d09fca9a52300af304d9427 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/dllm_eval/__pycache__/evaluator.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/__pycache__/evaluator_utils.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/dllm_eval/__pycache__/evaluator_utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7750e2727ed7fef28bcef4f64dc8d4968f5b3fd2 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/dllm_eval/__pycache__/evaluator_utils.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/__pycache__/utils.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/dllm_eval/__pycache__/utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c8416e2a2995d9d514a98f6a422b83443a97e517 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/dllm_eval/__pycache__/utils.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/api/__init__.py b/Prism/LLaDA/LLaDA_Prism/dllm_eval/api/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/api/filter.py b/Prism/LLaDA/LLaDA_Prism/dllm_eval/api/filter.py new file mode 100644 index 0000000000000000000000000000000000000000..bddbf3ab8d1bcbba804f9790ef0290d437bcde69 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/dllm_eval/api/filter.py @@ -0,0 +1,56 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Callable, Iterable, List, Union + +from dllm_eval.api.instance import Instance + + +class Filter(ABC): + """ + Filter classes operate on a per-task level. + They take all model outputs (`instance.resps` for all `task.instances`) + across all instances of a task, and perform operations. + In a single run, one can configure any number of separate filters or lists of filters. + + """ + + def __init__(self, **kwargs) -> None: + """ + Can define custom behavior here, if an individual instantiation of a Filter class should have state. + """ + + @abstractmethod + def apply(self, resps: Union[List, Iterable], docs: List[dict]) -> Iterable: + """ + Defines the operation to perform on a list of the `inst.resps` properties of `Instance` objects. + Should return the list of (filtered) response lists *in the same order as they were input*, e.g. + if pass in [, ] should return + [, ] + """ + return resps + + +@dataclass +class FilterEnsemble: + """ + FilterEnsemble creates a pipeline applying multiple filters. + Its intended usage is to stack multiple post-processing steps in order. + `task.apply_filters` should use a list of FilterEnsemble classes that it stores, to apply each + pipeline separately. + """ + + name: str + filters: List[Callable[[], Filter]] + + def apply(self, instances: List[Instance]) -> None: + resps, docs = zip(*((inst.resps, inst.doc) for inst in instances)) + resps, docs = list(resps), list(docs) + + for f in self.filters: + # apply filters in sequence + resps = f().apply(resps, docs) + + # add the end results after filtering to filtered_requests of their respective source instances. + # has key `self.name`: each FilterEnsemble applied in a given run should use a different name. + for inst, resp in zip(instances, resps): + inst.filtered_resps[self.name] = resp diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/api/group.py b/Prism/LLaDA/LLaDA_Prism/dllm_eval/api/group.py new file mode 100644 index 0000000000000000000000000000000000000000..0c60739bbd26c79ecab91f54240798b2ae9e3313 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/dllm_eval/api/group.py @@ -0,0 +1,115 @@ +import abc +from dataclasses import asdict, dataclass +from inspect import getsource +from typing import Any, Callable, List, Optional, Union + + +@dataclass +class AggMetricConfig(dict): + metric: Optional[str] = None + aggregation: Optional[str] = "mean" + weight_by_size: Optional[str] = False + # list of filter names which should be incorporated into the aggregated metric. + filter_list: Optional[Union[str, list]] = "none" + + def __post_init__(self): + if self.aggregation != "mean" and not callable(self.aggregation): + raise ValueError( + f"Currently, 'mean' is the only pre-defined aggregation across groups' subtasks. Got '{self.aggregation}'." + ) + + if isinstance(self.filter_list, str): + self.filter_list = [self.filter_list] + + +@dataclass +class GroupConfig(dict): + group: Optional[str] = None + group_alias: Optional[str] = None + task: Optional[Union[str, list]] = None + aggregate_metric_list: Optional[ + Union[List[AggMetricConfig], AggMetricConfig, dict] + ] = None + metadata: Optional[dict] = ( + None # by default, not used in the code. allows for users to pass arbitrary info to tasks + ) + + def __getitem__(self, item): + return getattr(self, item) + + def __setitem__(self, item, value): + return setattr(self, item, value) + + def __post_init__(self): + if self.aggregate_metric_list is not None: + if isinstance(self.aggregate_metric_list, dict): + self.aggregate_metric_list = [self.aggregate_metric_list] + + self.aggregate_metric_list = [ + AggMetricConfig(**item) if isinstance(item, dict) else item + for item in self.aggregate_metric_list + ] + + def to_dict(self, keep_callable: bool = False) -> dict: + """dumps the current config as a dictionary object, as a printable format. + null fields will not be printed. + Used for dumping results alongside full task configuration + + :return: dict + A printable dictionary version of the TaskConfig object. + + # TODO: should any default value in the TaskConfig not be printed? + """ + cfg_dict = asdict(self) + # remove values that are `None` + for k, v in list(cfg_dict.items()): + if callable(v): + cfg_dict[k] = self.serialize_function(v, keep_callable=keep_callable) + return cfg_dict + + def serialize_function( + self, value: Union[Callable, str], keep_callable=False + ) -> Union[Callable, str]: + """Serializes a given function or string. + + If 'keep_callable' is True, the original callable is returned. + Otherwise, attempts to return the source code of the callable using 'getsource'. + """ + if keep_callable: + return value + else: + try: + return getsource(value) + except (TypeError, OSError): + return str(value) + + +class ConfigurableGroup(abc.ABC): + def __init__( + self, + config: Optional[dict] = None, + ) -> None: + self._config = GroupConfig(**config) + + @property + def group(self): + return self._config.group + + @property + def group_alias(self): + return self._config.group_alias + + @property + def version(self): + return self._config.version + + @property + def config(self): + return self._config.to_dict() + + @property + def group_name(self) -> Any: + return self._config.group + + def __repr__(self): + return f"ConfigurableGroup(group={self.group},group_alias={self.group_alias})" diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/api/instance.py b/Prism/LLaDA/LLaDA_Prism/dllm_eval/api/instance.py new file mode 100644 index 0000000000000000000000000000000000000000..d3c6afa0644e729ba441728c72a2469fdad07b8f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/dllm_eval/api/instance.py @@ -0,0 +1,38 @@ +from dataclasses import dataclass, field +from typing import Literal, Optional, Tuple + + +OutputType = Literal[ + "loglikelihood", "loglikelihood_rolling", "generate_until", "multiple_choice" +] + + +@dataclass +class Instance: + request_type: OutputType + doc: dict + arguments: tuple + idx: int + metadata: Tuple[Optional[str], Optional[int], Optional[int]] = field( + default_factory=lambda: (None, None, None) + ) + resps: list = field(default_factory=list) + filtered_resps: dict = field(default_factory=dict) + + # initialized after init + task_name: Optional[str] = None + doc_id: Optional[int] = None + repeats: Optional[int] = None + + def __post_init__(self) -> None: + # unpack metadata field + self.task_name, self.doc_id, self.repeats = self.metadata + + @property + def args(self): + """ + Returns (string,) where `string` is the string to calculate loglikelihood over + """ + return ( + self.arguments if isinstance(self.arguments, tuple) else (self.arguments,) + ) diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/api/metrics.py b/Prism/LLaDA/LLaDA_Prism/dllm_eval/api/metrics.py new file mode 100644 index 0000000000000000000000000000000000000000..2aff6ce92a154a05df3d0bb7d28e09071cd12fbc --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/dllm_eval/api/metrics.py @@ -0,0 +1,578 @@ +import logging +import math +import random +import re +import string +from collections.abc import Iterable +from typing import List + +import numpy as np +import sacrebleu + +from dllm_eval.api.registry import register_aggregation, register_metric + + +eval_logger = logging.getLogger(__name__) + + +# Register Aggregations First +@register_aggregation("bypass") +def bypass_agg(arr): + return 999 + + +@register_aggregation("nanmean") +def nanmean(arr): + if len(arr) == 0 or all(np.isnan(arr)): + return np.nan + return np.nanmean(arr) + + +@register_aggregation("mean") +def mean(arr): + return sum(arr) / len(arr) + + +@register_aggregation("median") +def median(arr): + return arr[len(arr) // 2] + + +# Certain metrics must be calculated across all documents in a benchmark. +# We use them as aggregation metrics, paired with no-op passthrough metric fns. +@register_aggregation("perplexity") +def perplexity(items): + return math.exp(-mean(items)) + + +@register_aggregation("weighted_perplexity") +def weighted_perplexity(items): + return math.exp(-weighted_mean(items)) + + +@register_aggregation("bits_per_byte") +def bits_per_byte(items): + return -weighted_mean(items) / math.log(2) + + +@register_aggregation("f1") +def f1_score(items): + from sklearn.metrics import f1_score + + unzipped_list = list(zip(*items)) + golds = unzipped_list[0] + preds = unzipped_list[1] + fscore = f1_score(golds, preds) + + return np.max(fscore) + + +@register_aggregation("matthews_corrcoef") +def matthews_corrcoef(items): + from sklearn.metrics import matthews_corrcoef + + unzipped_list = list(zip(*items)) + golds = unzipped_list[0] + preds = unzipped_list[1] + return matthews_corrcoef(golds, preds) + + +@register_aggregation("bleu") +def bleu(items): + """The Bilingual Evaluation Understudy Score, or BLEU for short, is a metric + for evaluating a generated sentence to a reference sentence. It counts matching + n-grams in the candidate translation to n-grams in the reference text, where + 1-gram or unigram would be each token and a bigram comparison would be each + word pair. The comparison is made regardless of word order + Source: https://machinelearningmastery.com/calculate-bleu-score-for-text-python/ + Paper: https://www.aclweb.org/anthology/P02-1040/ + + Higher is better + """ + refs = list(zip(*items))[0] + preds = list(zip(*items))[1] + refs, preds = _sacreformat(refs, preds) + return sacrebleu.corpus_bleu(preds, refs).score + + +@register_aggregation("chrf") +def chrf(items): + """chrF++ is a tool for automatic evaluation of machine translation output + based on character n-gram precision and recall enhanced with word n-grams. + Source: https://github.com/m-popovic/chrF + Paper: https://www.aclweb.org/anthology/W15-3049.pdf + + Higher is better # TODO I think + """ + refs = list(zip(*items))[0] + preds = list(zip(*items))[1] + refs, preds = _sacreformat(refs, preds) + return sacrebleu.corpus_chrf(preds, refs).score + + +@register_aggregation("ter") +def ter(items): + """Translation Error Rate is an error metric for machine translation that + measures the number of edits required to change a system output into one + of the references + Source: http://www.cs.umd.edu/~snover/tercom/ + Paper: http://mt-archive.info/AMTA-2006-Snover.pdf + + Lower is better + """ + refs = list(zip(*items))[0] + preds = list(zip(*items))[1] + refs, preds = _sacreformat(refs, preds) + return sacrebleu.corpus_ter(preds, refs).score + + +@register_aggregation("brier_score") +def brier_score(items): # This is a passthrough function + gold, predictions = list(zip(*items)) + bs, num_class = np.array(predictions).shape + + gold = list(gold) + gold_one_hot = np.eye(num_class)[gold] + return np.mean(np.sum((predictions - gold_one_hot) ** 2, axis=1)) + + +@register_metric( + metric="brier_score", + higher_is_better=False, + output_type=["multiple_choice"], + aggregation="brier_score", +) +def brier_score_fn(items): # This is a passthrough function + return items + + +@register_metric( + metric="acc", + higher_is_better=True, + output_type=["loglikelihood", "multiple_choice"], + aggregation="mean", +) +def acc_fn(items): # This is a passthrough function + return items + + +@register_metric( + metric="acc_norm", + higher_is_better=True, + output_type=["loglikelihood", "multiple_choice"], + aggregation="mean", +) +def acc_norm_fn(items): # This is a passthrough function + return items + + +@register_metric( + metric="acc_mutual_info", + higher_is_better=True, + output_type="multiple_choice", + aggregation="mean", +) +def acc_mutual_info_fn(items): # This is a passthrough function + return items + + +### the code used in the `exact_match_hf_evaluate` function is ported from +### https://github.com/huggingface/evaluate/blob/main/metrics/exact_match/exact_match.py +### which is under the apache license. + +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +def exact_match_hf_evaluate( + predictions, + references, + regexes_to_ignore=None, + ignore_case=False, + ignore_punctuation=False, + ignore_numbers=False, +): + if regexes_to_ignore is not None: + for s in regexes_to_ignore: + predictions = np.array([re.sub(s, "", x) for x in predictions]) + references = np.array([re.sub(s, "", x) for x in references]) + else: + predictions = np.asarray(predictions) + references = np.asarray(references) + + if ignore_case: + predictions = np.char.lower(predictions) + references = np.char.lower(references) + + if ignore_punctuation: + repl_table = string.punctuation.maketrans("", "", string.punctuation) + predictions = np.char.translate(predictions, table=repl_table) + references = np.char.translate(references, table=repl_table) + + if ignore_numbers: + repl_table = string.digits.maketrans("", "", string.digits) + predictions = np.char.translate(predictions, table=repl_table) + references = np.char.translate(references, table=repl_table) + + score_list = predictions == references + + return {"exact_match": np.mean(score_list)} + + +### + + +@register_metric( + metric="exact_match", + higher_is_better=True, + output_type="generate_until", + aggregation="mean", +) +def exact_match_fn(**kwargs): + return exact_match_hf_evaluate(**kwargs) + + +@register_metric( + metric="perplexity", + higher_is_better=False, + output_type="loglikelihood", + aggregation="perplexity", +) +def perplexity_fn(items): # This is a passthrough function + return items + + +@register_metric( + metric="word_perplexity", + higher_is_better=False, + output_type="loglikelihood_rolling", + aggregation="weighted_perplexity", +) +def word_perplexity_fn(items): # This is a passthrough function + return items + + +@register_metric( + metric="byte_perplexity", + higher_is_better=False, + output_type="loglikelihood_rolling", + aggregation="weighted_perplexity", +) +def byte_perplexity_fn(items): # This is a passthrough function + return items + + +@register_metric( + metric="bits_per_byte", + higher_is_better=False, + output_type="loglikelihood_rolling", + aggregation="bits_per_byte", +) +def bits_per_byte_fn(items): # This is a passthrough function + return items + + +def pop_stddev(arr): + mu = mean(arr) + return math.sqrt(sum([(x - mu) ** 2 for x in arr]) / len(arr)) + + +def sample_stddev(arr): + mu = mean(arr) + return math.sqrt(sum([(x - mu) ** 2 for x in arr]) / (len(arr) - 1)) + + +def mean_stderr(arr): + return sample_stddev(arr) / math.sqrt(len(arr)) + + +@register_metric( + metric="bypass", + higher_is_better=True, + output_type=["loglikelihood", "multiple_choice", "generate_until"], + aggregation="bypass", +) +def bypass(items): + return None + + +@register_metric( + metric="mcc", + higher_is_better=True, + output_type="multiple_choice", + aggregation="matthews_corrcoef", +) +def mcc_fn(items): # This is a passthrough function + return items + + +@register_metric( + metric="f1", + higher_is_better=True, + output_type="multiple_choice", + aggregation="f1", +) +def f1_fn(items): # This is a passthrough function + return items + + +@register_metric( + metric="bleu", + higher_is_better=True, + output_type="generate_until", + aggregation="bleu", +) +def bleu_fn(items): # This is a passthrough function + return items + + +@register_metric( + metric="chrf", + higher_is_better=True, + output_type="generate_until", + aggregation="chrf", +) +def chrf_fn(items): # This is a passthrough function + return items + + +@register_metric( + metric="ter", + higher_is_better=True, + output_type="generate_until", + aggregation="ter", +) +def ter_fn(items): # This is a passthrough function + return items + + +@register_metric( + metric="acc_all", + higher_is_better=True, + output_type="loglikelihood", + aggregation="mean", +) +def acc_all(items): + # Only count as correct if all answers are labeled correctly for each question + question_scoring_dict = {} + preds = list(zip(*items))[0] + docs = list(zip(*items))[1] + + for doc, pred in zip(docs, preds): + paragraph_id = doc["idx"]["paragraph"] + question_id = doc["idx"]["question"] + if (paragraph_id, question_id) not in question_scoring_dict: + question_scoring_dict[(paragraph_id, question_id)] = [] + + gold_label = doc["label"] == 1 + + question_scoring_dict[(paragraph_id, question_id)].append(gold_label == pred) + acc = np.mean([int(all(x)) for x in question_scoring_dict.values()]) + return acc + + +def acc_all_stderr(items): + # Only count as correct if all answers are labeled correctly for each question + question_scoring_dict = {} + preds = list(zip(*items))[0] + docs = list(zip(*items))[1] + + for doc, pred in zip(docs, preds): + question_id = doc["idx"]["question"] + if question_id not in question_scoring_dict: + question_scoring_dict[question_id] = [] + + gold_label = doc["label"] == 1 + question_scoring_dict[question_id].append(gold_label == pred) + + acc = mean_stderr([int(all(x)) for x in question_scoring_dict.values()]) + return acc + + +def metric_max_over_ground_truths(metric_fn, prediction, ground_truths): + """Compute max metric between prediction and each ground truth.""" + scores_for_ground_truths = [] + for ground_truth in ground_truths: + score = metric_fn(prediction, ground_truth) + scores_for_ground_truths.append(score) + return max(scores_for_ground_truths) + + +def weighted_mean(items): + a, b = zip(*items) + return sum(a) / sum(b) + + +def is_non_str_iterable(obj): + return isinstance(obj, Iterable) and not isinstance(obj, str) + + +def _sacreformat(refs, preds): + """Format refs and preds for sacrebleu corpus calculation. It is very particular""" + # Sacrebleu expects (List[str], List[List[str]) + # e.g. sacrebleu.corpus_bleu([pred_t], [[ref1_stream], [ref2_stream], ...]) + + # Note [ref1_stream] is the first reference for each pred. + # So lists are size N and (M, N) for N preds and M possible refs for each pred + # This is a different order of dimensions that I would expect + + # We expect refs to be List[str] or List[List[str]], the outer list corresponding to preds + # Must become List[List[str]] with the inner list corresponding to preds + if not is_non_str_iterable(refs): + refs = list(refs) + if not is_non_str_iterable(refs[0]): + refs = [[ref] for ref in refs] + refs = list(zip(*refs)) + # Note the number of refs in each ref list much match the number of preds + + # We expect preds to be List[str] or List[List[str]]. Must become List[str] + if not is_non_str_iterable(preds): + preds = list(preds) + if is_non_str_iterable(preds[0]): + assert len(preds[0]) == 1, f"Pred must be a str, was {preds[0]}" + preds = [pred[0] for pred in preds] + + return refs, preds + + +# stderr stuff + + +class _bootstrap_internal: + def __init__(self, f, n) -> None: + self.f = f + self.n = n + + def __call__(self, v): + i, xs = v + rnd = random.Random() + rnd.seed(i) + res = [] + for _ in range(self.n): + res.append(self.f(rnd.choices(xs, k=len(xs)))) + return res + + +def bootstrap_stderr(f, xs, iters): + import multiprocessing as mp + + pool = mp.Pool(mp.cpu_count()) + # this gives a biased estimate of the stderr (i.e w/ the mean, it gives something + # equivalent to stderr calculated without Bessel's correction in the stddev. + # Unfortunately, I haven't been able to figure out what the right correction is + # to make the bootstrap unbiased - i considered multiplying by sqrt(n/(n-1)) but + # that would be ad-hoc and I can't prove that that would actually be an unbiased estimator) + # Thankfully, shouldn't matter because our samples are pretty big usually anyways + res = [] + chunk_size = min(1000, iters) + from tqdm import tqdm + + print("bootstrapping for stddev:", f.__name__) + for bootstrap in tqdm( + pool.imap( + _bootstrap_internal(f, chunk_size), + [(i, xs) for i in range(iters // chunk_size)], + ), + total=iters // chunk_size, + ): + # sample w replacement + res.extend(bootstrap) + + pool.close() + return sample_stddev(res) + + +def stderr_for_metric(metric, bootstrap_iters: int): + if bootstrap_iters <= 0: + # return no function (don't compute stderr) if bootstrap iters = 0 + return None + + bootstrappable = [ + median, + matthews_corrcoef, + f1_score, + perplexity, + bleu, + chrf, + ter, + nanmean, + ] + + if metric in bootstrappable: + return lambda x: bootstrap_stderr(metric, x, iters=bootstrap_iters) + + stderr = {mean: mean_stderr, acc_all: acc_all_stderr} + + return stderr.get(metric, None) + + +def pooled_sample_stderr(stderrs: List[float], sizes: List[int]): + # Used to aggregate bootstrapped stderrs across subtasks in a group, + # when we are weighting by the size of each subtask. + # + + assert len(stderrs) == len(sizes) + + # formula source: https://en.wikipedia.org/wiki/Pooled_variance + # and: https://stats.stackexchange.com/a/4841331 + # this empirically seems to match running `stderr_for_metric` on all instances + # from the subtasks concatenated with each other. + pooled_sample_var = ( + sum([(size - 1) * stderr**2 * size for size, stderr in zip(sizes, stderrs)]) + ) / (sum(sizes) - len(sizes)) + + return np.sqrt(pooled_sample_var / sum(sizes)) + + +def combined_sample_stderr(stderrs: List[float], sizes: List[int], metrics=None): + assert metrics is not None, ( + "Need to pass a list of each subtask's metric for this stderr aggregation" + ) + assert len(stderrs) == len(sizes) and len(sizes) == len(metrics) + + # See https://github.com/EleutherAI/lm-evaluation-harness/pull/1390 for more documentation. + # This formula depends on sample means. + # removed because it seems to give erroneously huge stderrs for groupings of tasks + # and does not seem to match up with bootstrap-calculated stderrs for groups. + + ### don't use this unless a statistician has told you it's the right thing to do ### + + # accumulators: we'll aggregate pairwise N - 1 times + variance = stderrs[0] ** 2 + curr_size = sizes[0] + curr_score = metrics[0] + + for stderr, size, score in zip(stderrs[1:], sizes[1:], metrics[1:]): + curr_score = ((curr_score * curr_size) + (score * size)) / ( + curr_size + size + ) # NOTE: this assumes our aggregation fn is "mean" + + variance = ((curr_size - 1) * variance + (size - 1) * (stderr**2)) / ( + curr_size + size - 1 + ) + curr_size * size / ((curr_size + size) * (curr_size + size - 1)) * ( + curr_score - score + ) ** 2 + + return np.sqrt(variance) + + +def aggregate_subtask_metrics(metrics, sizes, weight_by_size=True): + # A helper function that is used to aggregate + # subtask scores cross-task. + # TODO: does not hold for non-mean aggregations + if not weight_by_size: + sizes = [1] * len(sizes) + + assert len(metrics) == len(sizes) + + return sum([metric * size for metric, size in zip(metrics, sizes)]) / sum(sizes) diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/api/model.py b/Prism/LLaDA/LLaDA_Prism/dllm_eval/api/model.py new file mode 100644 index 0000000000000000000000000000000000000000..9364a9312d78c1029e5edf38d61f192afca91334 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/dllm_eval/api/model.py @@ -0,0 +1,493 @@ +import abc +import hashlib +import json +import logging +import os +from typing import Dict, List, Optional, Tuple, Type, TypeVar, Union + +import transformers +from sqlitedict import SqliteDict +from tqdm import tqdm + +from dllm_eval import utils + + +eval_logger = logging.getLogger(__name__) + +T = TypeVar("T", bound="LM") + + +class LM(abc.ABC): + def __init__(self) -> None: + """Defines the interface that should be implemented by all LM subclasses. + LMs are assumed to take text (strings) as input and yield strings as output + (inputs/outputs should be tokenization-agnostic.) + + """ + # set rank and world size to a single process, by default. + self._rank = 0 + self._world_size = 1 + self.cache_hook = CacheHook(None) + + @abc.abstractmethod + def loglikelihood(self, requests) -> List[Tuple[float, bool]]: + """Compute log-likelihood of generating a continuation from a context. + Downstream tasks should attempt to use loglikelihood instead of other + LM calls whenever possible. + + :param requests: list[Instance] + A list of Instance objects, with property `args` which returns a tuple (context, continuation). + `context: str` + Context string. Implementations of LM must be able to handle an + empty context string. + `continuation: str` + The continuation over which log likelihood will be calculated. If + there is a word boundary, the space should be in the continuation. + For example, context="hello" continuation=" world" is correct. + + :return: list[tuple[float, bool]] + A list of pairs (logprob, isgreedy) + `logprob: float` + The log probability of `continuation`. + `isgreedy`: + Whether `continuation` would be generated by greedy sampling from `context`. + """ + pass + + @abc.abstractmethod + def loglikelihood_rolling(self, requests) -> List[float]: + """Compute full log-likelihood of a string, with no truncation, for perplexity computation + - We will use the full max context length of the model. + - For inputs that exceed the max context length, we divide the tokenized string into chunks of up to + the max context length. + - IMPORTANT: Each document's loglikelihood/perplexity is computed *separately*, unlike other implementations + which may simply concatenate multiple documents together. + - IMPORTANT: We maximize the amount of context for each prediction. Specifically, for inputs that we break into + multiple chunks, the last input will still a full-sized context. + Example: + Input tokens: [ 0 1 2 3 4 5 6 7 8 9 ] + Prefix: BOS/EOS + Max context length: 4 + Resulting input/prediction pairs: + + INPUT: BOS 0 1 2 + PRED: 0 1 2 3 + + INPUT: 3 4 5 6 + PRED: 4 5 6 7 + + INPUT: 5 6 7 8 + PRED: 8 9 + + Observe that: + 1. Each token is predicted exactly once + 2. For the last pair, we provide the full context, but only score the last two tokens + + :param requests: list[Instance] + A list of Instance objects with property `args` which returns a tuple (context,). + string: str + String for which we are computing overall loglikelihood + :return: list[tuple[float]] + A list of tuples (logprob,) + logprob: float + The log probability of `context` conditioned on the BOS/EOS token. + Can also be overridden for custom cases by `prefix_token_id`. + """ + pass + + # TODO: Add an optional max length + @abc.abstractmethod + def generate_until(self, requests) -> List[str]: + """Generate greedily until a stopping sequence + + :param requests: list[Instance] + A list of Instance objects with property `args` which returns a tuple (context, gen_kwargs). + context: str + Context string + gen_kwargs: dict + A dictionary of keyword arguments to pass to the generation function e.g. top_k, until, etc. + :return: list[str] + A list of model generated continuations. + continuation: str + The generated continuation. + """ + pass + + def apply_chat_template( + self, chat_history: List[Dict[str, str]], add_generation_prompt=True + ) -> str: + """ + Defines how to transform few-shot examples provided as chat history into a format that can be used as input to the LM. + + :param chat_history: list[dict[str, str]] + A list of dictionaries with keys 'role' and 'content'. + Values are strings representing the role name and the content of the message, respectively. + :param add_generation_prompt: bool + Whether to append an assistant gen prefix (for e.g. <|assistant|>) to the assistant messages in the chat history. False if prefilling an assistant message. + :return: str + A string representing the chat history in a format that can be used as input to the LM. + """ + raise NotImplementedError( + "To use this model with chat templates, please implement the 'apply_chat_template' method for your model type." + ) + + @classmethod + def create_from_arg_string( + cls: Type[T], arg_string: str, additional_config: Optional[dict] = None + ) -> T: + """ + Creates an instance of the LM class using the given argument string and additional config. + + Parameters: + - arg_string: A string containing arguments in the format key1=value1,key2=value2. + - additional_config: Optional dictionary containing additional configuration parameters. + + Returns: + - Instance of the LM class. + """ + additional_config = {} if additional_config is None else additional_config + args = utils.simple_parse_args_string(arg_string) + args2 = {k: v for k, v in additional_config.items() if v is not None} + return cls(**args, **args2) + + @classmethod + def create_from_arg_obj( + cls: Type[T], arg_dict: dict, additional_config: Optional[dict] = None + ) -> T: + """ + Creates an instance of the LM class using the given arg_obj + + Parameters: + - arg_obj: A dict containing arguments in the format key1=value1,key2=value2. + - additional_config: Optional dictionary containing additional configuration parameters. + + Returns: + - Instance of the LM class. + """ + + additional_config = {} if additional_config is None else additional_config + additional_config = { + k: v for k, v in additional_config.items() if v is not None + } + + return cls(**arg_dict, **additional_config) + + @property + def rank(self): + # used in the case of parallelism. Hardcoded to + # ensure no errors arise using API models which do + # not support multi-device parallelism nor expect it. + return self._rank + + @property + def world_size(self): + # used in the case of parallelism. Hardcoded to + # ensure no errors arise using API models which do + # not support multi-device parallelism nor expect it. + return self._world_size + + @property + def tokenizer_name(self) -> str: + """Must be defined for LM subclasses which implement Chat Templating. + Should return the name of the tokenizer or chat template used. + Used only to properly fingerprint caches when requests are being cached with `--cache_requests`, otherwise not used. + """ + raise NotImplementedError( + "To use this model with chat templates, please implement the 'tokenizer_name' property." + ) + + def chat_template(self, chat_template: Union[bool, str] = False) -> Optional[str]: + """Returns the chat template structure for user/assistant messages if a template is provided. + This method is intended to be overridden in a subclass to define a specific chat template format. + For models that do not support chat templates, this method returns None by default. + """ + + return "" + + def set_cache_hook(self, cache_hook) -> None: + self.cache_hook = cache_hook + + +### SQLite-based caching of LM responses +def hash_args(attr, args): + dat = json.dumps([attr] + list(args)) + return hashlib.sha256(dat.encode("utf-8")).hexdigest() + + +class CacheHook: + def __init__(self, cachinglm) -> None: + if cachinglm is None: + self.dbdict = None + return + + self.dbdict = cachinglm.dbdict + + def add_partial(self, attr, req, res) -> None: + if self.dbdict is None: + return + hsh = hash_args(attr, req) + self.dbdict[hsh] = res + + +class CachingLM: + def __init__(self, lm, cache_db) -> None: + """LM wrapper that returns cached results if they exist, and uses the underlying LM if not. + + :param lm: LM + Underlying LM + :param cache_db: str + Path to cache db + """ + self.lm = lm + self.cache_db = cache_db + if os.path.dirname(cache_db): + os.makedirs(os.path.dirname(cache_db), exist_ok=True) + self.dbdict = SqliteDict(cache_db, autocommit=True) + + # add hook to lm + lm.set_cache_hook(self.get_cache_hook()) + + def __getattr__(self, attr: str): + lm_attr = getattr(self.lm, attr) + if attr not in ["loglikelihood", "loglikelihood_rolling", "generate_until"]: + eval_logger.debug(f"Passing through attribute '{attr}' to underlying LM") + return lm_attr + + def fn(requests): + res = [] + remaining_reqs = [] + warned = False + # figure out which ones are cached and which ones are new + eval_logger.info( + f"Loading '{attr}' responses from cache '{self.cache_db}' where possible..." + ) + for req in tqdm(requests, desc="Checking cached requests"): + hsh = hash_args(attr, req.args) + if attr == "generate_until" and req.args[1].get("do_sample", False): + # when we are doing non-greedy generation, don't use the cache + # (else every "randomly sampled" generation would be identical for repeats > 1). + if not warned: + eval_logger.warning( + f"Arguments to lm.generate_until() '{req.args[1]}' include non-deterministic sampling. Caching will not be performed for such requests." + ) + warned = True + res.append(None) + remaining_reqs.append(req) + elif hsh in self.dbdict: + ob = self.dbdict[hsh] + + assert ob is not None + + res.append(ob) + else: + res.append(None) + remaining_reqs.append(req) + eval_logger.info( + f"Cached requests: {len(requests) - len(remaining_reqs)}, Requests remaining: {len(remaining_reqs)}" + ) + if remaining_reqs: + # actually run the LM on the requests that do not have cached results + rem_res = getattr(self.lm, attr)(remaining_reqs) + else: + rem_res = [] + + # stick the new ones back into the list and also cache any of the new ones + resptr = 0 + for req, r in zip(remaining_reqs, rem_res): + while res[resptr] is not None: + resptr += 1 + + res[resptr] = r + + # caching + hsh = hash_args(attr, req.args) + self.dbdict[hsh] = r + self.dbdict.commit() + + return res + + return fn + + def get_cache_hook(self): + return CacheHook(self) + + +class TemplateLM(LM): + """ + A class acting as intermediary between the LM base class + and boilerplate often included in other LM subclasses. + """ + + tokenizer = None + + @property + @abc.abstractmethod + def eot_token_id(self): + pass + + @property + def prefix_token_id(self): + # it is used as prefix for loglikelihood + return self.eot_token_id + + @abc.abstractmethod + def tok_encode(self, string: str, **kwargs) -> List[int]: + """ + Tokenize a string using the model's tokenizer and return a list of token IDs. + """ + pass + + @abc.abstractmethod + def _loglikelihood_tokens(self, requests, **kwargs) -> List[Tuple[float, bool]]: + pass + + def _encode_pair( + self, context: str, continuation: str + ) -> Tuple[List[int], List[int]]: + n_spaces = len(context) - len(context.rstrip()) + if n_spaces > 0: + continuation = context[-n_spaces:] + continuation + context = context[:-n_spaces] + + model_class = getattr(self, "AUTO_MODEL_CLASS", None) + + if model_class == transformers.AutoModelForSeq2SeqLM: + context_enc = self.tok_encode(context) + continuation_enc = self.tok_encode(continuation, add_special_tokens=False) + else: + whole_enc = self.tok_encode(context + continuation) + context_enc = self.tok_encode(context) + + context_enc_len = len(context_enc) + continuation_enc = whole_enc[context_enc_len:] + + return context_enc, continuation_enc + + def loglikelihood( + self, requests, disable_tqdm: bool = False + ) -> List[Tuple[float, bool]]: + new_reqs = [] + for context, continuation in [req.args for req in requests]: + if context == "": + # BOS or EOS as context + context_enc, continuation_enc = ( + [self.prefix_token_id], + self.tok_encode(continuation), + ) + else: + context_enc, continuation_enc = self._encode_pair(context, continuation) + + new_reqs.append(((context, continuation), context_enc, continuation_enc)) + + return self._loglikelihood_tokens(new_reqs, disable_tqdm=disable_tqdm) + + @abc.abstractmethod + def loglikelihood_rolling( + self, requests, disable_tqdm: bool = False + ) -> List[float]: + pass + + @abc.abstractmethod + def generate_until(self, requests, disable_tqdm: bool = False) -> List[str]: + pass + + def chat_template(self, chat_template: Union[bool, str] = False) -> Optional[str]: + """ + Set and get the appropriate chat template for the model. + This method sets the tokenizer's chat_template and returns the template string for reproducibility. + + The template selection logic is adapted from the Transformers library's `apply_chat_template` + method in the Tokenizer class. The original implementation can be found at: + https://github.com/huggingface/transformers/blob/fc35907f95459d7a6c5281dfadd680b6f7b620e3/src/transformers/tokenization_utils_base.py#L1687 + + This method ensures that the right template is chosen based on the following: + 0. If the model has no 'tokenizer' attribute: assumes that there is only a single possible chat template, handled on the model provider side internally. Returns the empty string. + 1. If the model's tokenizer has multiple templates: + a. Use the specified template if it exists in the dictionary. + b. Use the default template from the list if no specific template is provided. + c. Raise an error if no default template exists and no specific template is provided. + 2. If the model's tokenizer has a single template or no template: + a. Use the tokenizer's chat template if available. + b. Fall back to the default chat template if no tokenizer chat template exists. + + Args: + chat_template (Union[bool, str]): Specifies the chat template to use. + - If False or None, no template is applied. + - If True, the default or only available template is used. + - If a string, the template with the matching name is used. + + Returns: + Optional[str]: The selected chat template, or None if no template is applied. + """ + if self.tokenizer is None: + return "" + + if chat_template is False or chat_template is None: + eval_logger.warning( + "model.chat_template was called with the chat_template set to False or None. " + "Therefore no chat template will be applied. Make sure this is an intended behavior." + ) + return None + + # Convert boolean chat_template to None to ensure compatibility with the adapted logic + if isinstance(chat_template, bool): + chat_template = None + using_default_template = False + + # First, handle the cases when the model has a dict of multiple templates + try: + template = ( + self.tokenizer.chat_template or self.tokenizer.default_chat_template + ) + except AttributeError: + return None + + if isinstance(template, dict): + using_default_dict = self.tokenizer.chat_template is None + + if chat_template is not None: + if chat_template in template: + selected_template = template[chat_template] + if using_default_dict: + using_default_template = True + else: + raise ValueError( + f"The specified chat template '{chat_template}' is not available. " + f"Available template names are {sorted(template.keys())}." + ) + else: + # If user didn't pass a chat template, use the default template from the dict + if "default" in template: + selected_template = template["default"] + using_default_template = True + else: + raise ValueError( + "This model has multiple chat templates with no default specified! Please either pass a chat " + "template or the name of the template you wish to use to the `chat_template` argument. Available " + f"template names are {sorted(template.keys())}." + ) + + # Cases when the model has a single template or no template + else: + # priority: `chat_template` argument > `tokenizer.chat_template` > `tokenizer.default_chat_template + if isinstance(chat_template, str): + eval_logger.warning( + "Chat template name provided, but the tokenizer's chat template is not a dictionary. " + "Using the tokenizer's chat template or the default template instead." + ) + if self.tokenizer.chat_template is not None: + selected_template = self.tokenizer.chat_template + else: + selected_template = self.tokenizer.default_chat_template + using_default_template = True + + if using_default_template: + eval_logger.warning( + "No chat template is set for this tokenizer, falling back to a default class-level template. This is " + "very error-prone, because models are often trained with templates different from the class default! " + "Default chat templates are a legacy feature and will be removed in Transformers v4.43, at which " + "point any code depending on them will stop working. We recommend setting a valid chat template before " + "then to ensure that this model continues working without issues." + ) + + return selected_template diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/api/registry.py b/Prism/LLaDA/LLaDA_Prism/dllm_eval/api/registry.py new file mode 100644 index 0000000000000000000000000000000000000000..bf2b2e415a0a19862a41bde307bbad2e6ba326f5 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/dllm_eval/api/registry.py @@ -0,0 +1,196 @@ +import logging +from typing import Callable, Dict, Union + +import evaluate as hf_evaluate + +from dllm_eval.api.model import LM + + +eval_logger = logging.getLogger(__name__) + +MODEL_REGISTRY = {} + + +def register_model(*names): + # either pass a list or a single alias. + # function receives them as a tuple of strings + + def decorate(cls): + for name in names: + assert issubclass(cls, LM), ( + f"Model '{name}' ({cls.__name__}) must extend LM class" + ) + + assert name not in MODEL_REGISTRY, ( + f"Model named '{name}' conflicts with existing model! Please register with a non-conflicting alias instead." + ) + + MODEL_REGISTRY[name] = cls + return cls + + return decorate + + +def get_model(model_name): + try: + return MODEL_REGISTRY[model_name] + except KeyError: + raise ValueError( + f"Attempted to load model '{model_name}', but no model for this name found! Supported model names: {', '.join(MODEL_REGISTRY.keys())}" + ) + + +TASK_REGISTRY = {} +GROUP_REGISTRY = {} +ALL_TASKS = set() +func2task_index = {} + + +def register_task(name): + def decorate(fn): + assert name not in TASK_REGISTRY, ( + f"task named '{name}' conflicts with existing registered task!" + ) + + TASK_REGISTRY[name] = fn + ALL_TASKS.add(name) + func2task_index[fn.__name__] = name + return fn + + return decorate + + +def register_group(name): + def decorate(fn): + func_name = func2task_index[fn.__name__] + if name in GROUP_REGISTRY: + GROUP_REGISTRY[name].append(func_name) + else: + GROUP_REGISTRY[name] = [func_name] + ALL_TASKS.add(name) + return fn + + return decorate + + +OUTPUT_TYPE_REGISTRY = {} +METRIC_REGISTRY = {} +METRIC_AGGREGATION_REGISTRY = {} +AGGREGATION_REGISTRY: Dict[str, Callable[[], Dict[str, Callable]]] = {} +HIGHER_IS_BETTER_REGISTRY = {} +FILTER_REGISTRY = {} + +DEFAULT_METRIC_REGISTRY = { + "loglikelihood": [ + "perplexity", + "acc", + ], + "loglikelihood_rolling": ["word_perplexity", "byte_perplexity", "bits_per_byte"], + "multiple_choice": ["acc", "acc_norm"], + "generate_until": ["exact_match"], +} + + +def register_metric(**args): + # TODO: do we want to enforce a certain interface to registered metrics? + def decorate(fn): + assert "metric" in args + name = args["metric"] + + for key, registry in [ + ("metric", METRIC_REGISTRY), + ("higher_is_better", HIGHER_IS_BETTER_REGISTRY), + ("aggregation", METRIC_AGGREGATION_REGISTRY), + ]: + if key in args: + value = args[key] + assert value not in registry, ( + f"{key} named '{value}' conflicts with existing registered {key}!" + ) + + if key == "metric": + registry[name] = fn + elif key == "aggregation": + registry[name] = AGGREGATION_REGISTRY[value] + else: + registry[name] = value + + return fn + + return decorate + + +def get_metric(name: str, hf_evaluate_metric=False) -> Callable: + if not hf_evaluate_metric: + if name in METRIC_REGISTRY: + return METRIC_REGISTRY[name] + else: + eval_logger.warning( + f"Could not find registered metric '{name}' in lm-eval, searching in HF Evaluate library..." + ) + + try: + metric_object = hf_evaluate.load(name) + return metric_object.compute + except Exception: + eval_logger.error( + f"{name} not found in the evaluate library! Please check https://huggingface.co/evaluate-metric", + ) + + +def register_aggregation(name: str): + def decorate(fn): + assert name not in AGGREGATION_REGISTRY, ( + f"aggregation named '{name}' conflicts with existing registered aggregation!" + ) + + AGGREGATION_REGISTRY[name] = fn + return fn + + return decorate + + +def get_aggregation(name: str) -> Callable[[], Dict[str, Callable]]: + try: + return AGGREGATION_REGISTRY[name] + except KeyError: + eval_logger.warning(f"{name} not a registered aggregation metric!") + + +def get_metric_aggregation(name: str) -> Callable[[], Dict[str, Callable]]: + try: + return METRIC_AGGREGATION_REGISTRY[name] + except KeyError: + eval_logger.warning(f"{name} metric is not assigned a default aggregation!") + + +def is_higher_better(metric_name) -> bool: + try: + return HIGHER_IS_BETTER_REGISTRY[metric_name] + except KeyError: + eval_logger.warning( + f"higher_is_better not specified for metric '{metric_name}'!" + ) + + +def register_filter(name): + def decorate(cls): + if name in FILTER_REGISTRY: + eval_logger.info( + f"Registering filter `{name}` that is already in Registry {FILTER_REGISTRY}" + ) + FILTER_REGISTRY[name] = cls + return cls + + return decorate + + +def get_filter(filter_name: Union[str, Callable]) -> Callable: + try: + return FILTER_REGISTRY[filter_name] + except KeyError as e: + if callable(filter_name): + return filter_name + else: + eval_logger.warning(f"filter `{filter_name}` is not registered!") + raise e diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/api/samplers.py b/Prism/LLaDA/LLaDA_Prism/dllm_eval/api/samplers.py new file mode 100644 index 0000000000000000000000000000000000000000..969789ef2111dcb8ee3b7eed4c69d54572d6c302 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/dllm_eval/api/samplers.py @@ -0,0 +1,232 @@ +import logging +import warnings +from functools import partial +from typing import TYPE_CHECKING, Iterable, Optional, Union + +import datasets + + +if TYPE_CHECKING: + from random import Random + + from dllm_eval.api.task import ConfigurableTask, Task + +eval_logger = logging.getLogger("lm-eval") + + +class ContextSampler: + def __init__( + self, + docs: list[dict], + task: Union["Task", "ConfigurableTask"], + fewshot_indices: Optional[Iterable] = None, + rnd: Optional["Random"] = None, + ) -> None: + self.rnd = rnd + if not self.rnd: + raise ValueError( + "A `random.Random` generator argument must be provided to `rnd` of FewShotSampler!" + ) + + self.task = task + self.config = task._config + + self.target_delimiter = self.config.target_delimiter + self.fewshot_delimiter = self.config.fewshot_delimiter + + if ( + self.config.fewshot_config is not None + and self.config.fewshot_config.get("doc_to_text", None) is not None + ): + self.doc_to_text = partial( + self.task.doc_to_text, + doc_to_text=self.config.fewshot_config.get("doc_to_text", None), + ) + else: + self.doc_to_text = self.task.doc_to_text + + if ( + self.config.fewshot_config is not None + and self.config.fewshot_config.get("doc_to_target", None) is not None + ): + self.doc_to_target = partial( + self.task.doc_to_target, + doc_to_target=self.config.fewshot_config.get("doc_to_target", None), + ) + else: + self.doc_to_target = self.task.doc_to_target + + if ( + self.config.fewshot_config is not None + and self.config.fewshot_config.get("doc_to_choice", None) is not None + ): + self.doc_to_choice = partial( + self.task.doc_to_choice, + doc_to_choice=self.config.fewshot_config.get("doc_to_choice", None), + ) + else: + self.doc_to_choice = self.task.doc_to_choice + + self.docs = docs # HF dataset split, provided by task._fewshot_docs() + if fewshot_indices: # subset few-shot docs from + if not isinstance(self.docs, datasets.Dataset): + raise ValueError( + "Got `fewshot_indices` but fewshot_docs are not a HF dataset. Don't use both `fewshot_indices` and a user-defined few-shot sample list simultaneously" + ) + self.docs = self.docs.select(fewshot_indices) + + def get_context(self, doc: dict, num_fewshot: int, gen_prefix: str = None): + # draw an extra fewshot sample if using same split as evaluating on + prefix = gen_prefix + " " if gen_prefix else "" + n_samples = ( + num_fewshot + 1 + if self.config.fewshot_split == self.config.test_split + else num_fewshot + ) + + # draw `n_samples` docs from fewshot_docs + fewshotex = self.sample(n_samples) + + # get rid of the doc that's the one we're evaluating, if it's in the fewshot + # TODO: should we just stop people from using fewshot from same split as evaluating? + selected_docs = [x for x in fewshotex if x != doc][:num_fewshot] + + labeled_examples = "" + for doc in selected_docs: + doc_content = self.doc_to_text(doc) + doc_target = self.doc_to_target(doc) + if self.config.doc_to_choice is None or isinstance(doc_content, str): + labeled_examples += doc_content + else: + labeled_examples += self.doc_to_choice(doc)[doc_content] + + if doc_target != "": + if self.target_delimiter.isspace() and str(doc_target)[0].isspace(): + # TODO: add logger warn once here. + warnings.warn( + "Both target_delimiter and target start with a space. This may cause issues.", + Warning, + stacklevel=2, + ) + labeled_examples += self.target_delimiter + labeled_examples += prefix + labeled_examples += ( + str(doc_target[0]) + if isinstance(doc_target, list) + else doc_target + if self.config.doc_to_choice is None or isinstance(doc_target, str) + else str(self.doc_to_choice(doc)[doc_target]) + ) + labeled_examples += self.fewshot_delimiter + + return labeled_examples + + def get_chat_context( + self, + doc: dict, + num_fewshot: int, + fewshot_as_multiturn: bool = False, + gen_prefix: Optional[str] = None, + ): + # TODO: Do we need any other delimiter + prefix = gen_prefix + " " if gen_prefix else "" + chat_history = [] + # draw an extra fewshot sample if using same split as evaluating on + n_samples = ( + num_fewshot + 1 + if self.config.fewshot_split == self.config.test_split + else num_fewshot + ) + # draw `n_samples` docs from fewshot_docs + fewshotex = self.sample(n_samples) + + # get rid of the doc that's the one we're evaluating, if it's in the fewshot + # TODO: should we just stop people from using fewshot from same split as evaluating? + selected_docs = [x for x in fewshotex if x != doc][:num_fewshot] + + if fewshot_as_multiturn: + for doc in selected_docs: + doc_content = self.doc_to_text(doc) + doc_target = self.doc_to_target(doc) + chat_history.append( + { + "role": "user", + "content": doc_content + if self.config.doc_to_choice is None + or isinstance(doc_content, str) + else self.doc_to_choice(doc)[doc_content], + } + ) + chat_history.append( + { + "role": "assistant", + "content": prefix + str(doc_target[0]) + if isinstance(doc_target, list) + else prefix + doc_target + if self.config.doc_to_choice is None + or isinstance(doc_target, str) + else prefix + str(self.doc_to_choice(doc)[doc_target]), + } + ) + else: + # get fewshot context as one user turn + chat_history.append( + { + "role": "user", + "content": self.get_context( + doc, num_fewshot, gen_prefix=gen_prefix + ), + } + ) + + return chat_history + + def sample(self, n: int): + """ + Draw `n` samples from our fewshot docs. This method should be overridden by subclasses. + """ + + return self.rnd.sample(self.docs, n) + + +class FirstNSampler(ContextSampler): + def sample(self, n: int) -> None: + """ + Draw the first `n` samples in order from the specified split. + Used for tasks with "canonical" ordered fewshot examples, such as MMLU and CMMLU. + """ + assert n <= len(self.docs), ( + f"Error: number of fewshot samples requested exceeds the {len(self.docs)} that are available." + ) + return self.docs[:n] + + +class BalancedSampler(ContextSampler): + def sample(self, n: int) -> None: + """ + TODO: this should return approximately class-balanced samples from our fewshot examples. + TODO: what order should they be in? maybe random? + """ + + pass + + +class ManualSampler(ContextSampler): + def sample(self, n: int) -> None: + """ """ + pass + + +SAMPLER_REGISTRY = { + "default": ContextSampler, + "first_n": FirstNSampler, +} + + +def get_sampler(name: str): + try: + return SAMPLER_REGISTRY[name] + except KeyError: + raise ValueError( + f"Attempted to use contextsampler '{name}', but no sampling strategy for this name found! Supported model names: {', '.join(SAMPLER_REGISTRY.keys())}" + ) diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/api/task.py b/Prism/LLaDA/LLaDA_Prism/dllm_eval/api/task.py new file mode 100644 index 0000000000000000000000000000000000000000..4a6321af0b2b8777e0322745a9875656ec194190 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/dllm_eval/api/task.py @@ -0,0 +1,1881 @@ +import abc +import ast +import logging +import random +import re +from collections.abc import Callable +from copy import deepcopy +from dataclasses import asdict, dataclass +from inspect import getsource +from typing import ( + Any, + Dict, + Iterable, + Iterator, + List, + Literal, + Mapping, + Optional, + Tuple, + Union, +) + +import datasets +import numpy as np +from tqdm import tqdm + +from dllm_eval import utils +from dllm_eval.api import samplers +from dllm_eval.api.instance import Instance, OutputType +from dllm_eval.api.metrics import bits_per_byte, mean, weighted_perplexity +from dllm_eval.api.registry import ( + AGGREGATION_REGISTRY, + DEFAULT_METRIC_REGISTRY, + get_aggregation, + get_metric, + get_metric_aggregation, + is_higher_better, +) +from dllm_eval.caching.cache import load_from_cache, save_to_cache +from dllm_eval.filters import build_filter_ensemble +from dllm_eval.prompts import get_prompt + + +ALL_OUTPUT_TYPES = [ + "loglikelihood", + "multiple_choice", + "loglikelihood_rolling", + "generate_until", +] + +eval_logger = logging.getLogger(__name__) + + +@dataclass +class TaskConfig(dict): + # task naming/registry + task: Optional[str] = None + task_alias: Optional[str] = None + tag: Optional[Union[str, list]] = None + # HF dataset options. + # which dataset to use, + # and what splits for what purpose + custom_dataset: Optional[Callable] = None + dataset_path: Optional[str] = None + dataset_name: Optional[str] = None + dataset_kwargs: Optional[dict] = None + training_split: Optional[str] = None + validation_split: Optional[str] = None + test_split: Optional[str] = None + fewshot_split: Optional[str] = ( + None # TODO: assert that this not None if num_fewshot > 0. (?) assert if this is same split as one evaluating (?) + ) + # formatting / prompting options. + # see docs/advanced_task_guide.md for more info + process_docs: Optional[Callable] = None + doc_to_text: Optional[Union[Callable, str]] = None + doc_to_target: Optional[Union[Callable, str]] = None + doc_to_image: Union[Callable, str] = None + doc_to_audio: Union[Callable, str] = None + unsafe_code: bool = False + doc_to_choice: Optional[Union[Callable, str, dict, list]] = None + process_results: Optional[Union[Callable, str]] = None + use_prompt: Optional[str] = None + description: str = "" + target_delimiter: str = " " + fewshot_delimiter: str = "\n\n" + fewshot_config: Optional[dict] = None + # runtime configuration options + num_fewshot: Optional[int] = None + # scoring options + metric_list: Optional[list] = None + output_type: OutputType = "generate_until" + generation_kwargs: Optional[dict] = None + repeats: int = 1 + filter_list: Optional[Union[str, list]] = None + should_decontaminate: bool = False + doc_to_decontamination_query: Optional[str] = None + gen_prefix: Optional[str] = None + metadata: Optional[dict] = ( + None # by default, not used in the code. allows for users to pass arbitrary info to tasks + ) + + def __post_init__(self) -> None: + if self.generation_kwargs is not None: + if self.output_type != "generate_until": + eval_logger.warning( + f"[{self.task}] passed `generation_kwargs`, but not using `output_type: generate_until`!" + ) + + if "temperature" in self.generation_kwargs: + self.generation_kwargs["temperature"] = float( + self.generation_kwargs["temperature"] + ) + + if "until" not in self.generation_kwargs: + eval_logger.warning( + f"{self.task}: No `until` specified in `generation_kwargs`! Defaulting to the fewshot_delimiter={repr(self.fewshot_delimiter)}" + ) + self.generation_kwargs["until"] = [self.fewshot_delimiter] + else: + if self.output_type == "generate_until": + # ensure that we greedily generate in absence of explicit arguments otherwise + self.generation_kwargs = { + "until": ( + None + if self.fewshot_delimiter is None + else [self.fewshot_delimiter] + ), + "do_sample": False, + "temperature": 0, + } + eval_logger.warning( + f"{self.task}: No `generation_kwargs` specified in task config, defaulting to {self.generation_kwargs}" + ) + + def __getitem__(self, item): + return getattr(self, item) + + def __setitem__(self, item, value): + return setattr(self, item, value) + + def to_dict(self, keep_callable: bool = False) -> dict: + """dumps the current config as a dictionary object, as a printable format. + null fields will not be printed. + Used for dumping results alongside full task configuration + + :return: dict + A printable dictionary version of the TaskConfig object. + + # TODO: should any default value in the TaskConfig not be printed? + """ + cfg_dict = asdict(self) + # remove values that are `None` + for k, v in list(cfg_dict.items()): + if v is None: + cfg_dict.pop(k) + elif k == "metric_list": + for metric_dict in v: + for metric_key, metric_value in metric_dict.items(): + if callable(metric_value): + metric_dict[metric_key] = self.serialize_function( + metric_value, keep_callable=keep_callable + ) + cfg_dict[k] = v + elif callable(v): + cfg_dict[k] = self.serialize_function(v, keep_callable=keep_callable) + return cfg_dict + + def serialize_function( + self, value: Union[Callable, str], keep_callable=False + ) -> Union[Callable, str]: + """Serializes a given function or string. + + If 'keep_callable' is True, the original callable is returned. + Otherwise, attempts to return the source code of the callable using 'getsource'. + """ + if keep_callable: + return value + else: + try: + return getsource(value) + except (TypeError, OSError): + return str(value) + + +class Task(abc.ABC): + """A task represents an entire benchmark including its dataset, problems, + answers, and evaluation methods. See BoolQ for a simple example implementation + + A `doc` can be any python object which represents one instance of evaluation. + This is usually a dictionary e.g. + {"question": ..., "answer": ...} or + {"question": ..., question, answer) + """ + + VERSION: Optional[Union[int, str]] = None + + # The name of the `Task` benchmark as denoted in the HuggingFace datasets Hub + # or a path to a custom `datasets` loading script. + DATASET_PATH: Optional[str] = None + + # The name of a subset within `DATASET_PATH`. + DATASET_NAME: Optional[str] = None + + OUTPUT_TYPE: Optional[OutputType] = None + + def __init__( + self, + data_dir: Optional[str] = None, + cache_dir: Optional[str] = None, + download_mode: Optional[datasets.DownloadMode] = None, + config: Optional[Mapping] = None, # Union[dict, TaskConfig] + ) -> None: + """ + :param data_dir: str + Stores the path to a local folder containing the `Task`'s data files. + Use this to specify the path to manually downloaded data (usually when + the dataset is not publicly accessible). + :param cache_dir: str + The directory to read/write the `Task` dataset. This follows the + HuggingFace `datasets` API with the default cache directory located at: + `~/.cache/huggingface/datasets` + NOTE: You can change the cache location globally for a given process + to another directory: + `export HF_DATASETS_CACHE="/path/to/another/directory"` + :param download_mode: datasets.DownloadMode + How to treat pre-existing `Task` downloads and data. + - `datasets.DownloadMode.REUSE_DATASET_IF_EXISTS` + Reuse download and reuse dataset. + - `datasets.DownloadMode.REUSE_CACHE_IF_EXISTS` + Reuse download with fresh dataset. + - `datasets.DownloadMode.FORCE_REDOWNLOAD` + Fresh download and fresh dataset. + """ + self.download(data_dir, cache_dir, download_mode) + self._training_docs: Optional[list] = None + self._fewshot_docs: Optional[list] = None + self._instances: Optional[List[Instance]] = None + + self._config: TaskConfig = TaskConfig({**config}) if config else TaskConfig() + + self._filters = [build_filter_ensemble("none", [["take_first", None]])] + self.fewshot_rnd: Optional[random.Random] = ( + None # purposely induce errors in case of improper usage + ) + + def download( + self, + data_dir: Optional[str] = None, + cache_dir: Optional[str] = None, + download_mode=None, + ) -> None: + """Downloads and returns the task dataset. + Override this method to download the dataset from a custom API. + + :param data_dir: str + Stores the path to a local folder containing the `Task`'s data files. + Use this to specify the path to manually downloaded data (usually when + the dataset is not publicly accessible). + :param cache_dir: str + The directory to read/write the `Task` dataset. This follows the + HuggingFace `datasets` API with the default cache directory located at: + `~/.cache/huggingface/datasets` + NOTE: You can change the cache location globally for a given process + by setting the shell environment variable, `HF_DATASETS_CACHE`, + to another directory: + `export HF_DATASETS_CACHE="/path/to/another/directory"` + :param download_mode: datasets.DownloadMode + How to treat pre-existing `Task` downloads and data. + - `datasets.DownloadMode.REUSE_DATASET_IF_EXISTS` + Reuse download and reuse dataset. + - `datasets.DownloadMode.REUSE_CACHE_IF_EXISTS` + Reuse download with fresh dataset. + - `datasets.DownloadMode.FORCE_REDOWNLOAD` + Fresh download and fresh dataset. + """ + self.dataset = datasets.load_dataset( + path=self.DATASET_PATH, + name=self.DATASET_NAME, + data_dir=data_dir, + cache_dir=cache_dir, + download_mode=download_mode, + ) + + @property + def config(self) -> TaskConfig: + """Returns the TaskConfig associated with this class.""" + return self._config + + @abc.abstractmethod + def has_training_docs(self): + """Whether the task has a training set""" + pass + + @abc.abstractmethod + def has_validation_docs(self): + """Whether the task has a validation set""" + pass + + @abc.abstractmethod + def has_test_docs(self): + """Whether the task has a test set""" + pass + + def training_docs(self) -> Iterable: + """ + :return: Iterable[obj] + A iterable of any object, that doc_to_text can handle + """ + return [] + + def validation_docs(self) -> Iterable: + """ + :return: Iterable[obj] + A iterable of any object, that doc_to_text can handle + """ + return [] + + def test_docs(self) -> Iterable: + """ + :return: Iterable[obj] + A iterable of any object, that doc_to_text can handle + """ + return [] + + def fewshot_docs(self) -> Iterable: + """ + :return: Iterable[obj] + A iterable of any object, that doc_to_text can handle + """ + if self.has_training_docs(): + return self.training_docs() + elif self.has_validation_docs(): + return self.validation_docs() + else: + if self.config.get("num_fewshot", 0) > 0: + eval_logger.warning( + f"[Task: {self.config.task}] has_training_docs and has_validation_docs are False" + ", using test_docs as fewshot_docs but this is not recommended." + ) + return self.test_docs() + + def _process_doc(self, doc: dict) -> dict: + """ + Override this to process (detokenize, strip, replace, etc.) individual + documents. This can be used in a map over documents of a data split. + E.g. `map(self._process_doc, self.dataset["validation"])` + + :return: dict + The processed version of the specified `doc`. + """ + return doc + + @property + def instances(self) -> List[Instance]: + """After calling `task.build_all_requests()`, tasks + maintain a list of the dataset instances which will be evaluated. + """ + return self._instances + + def fewshot_examples(self, k, rnd): + if self._training_docs is None: + self._training_docs = list(self.training_docs()) + + return rnd.sample(self._training_docs, k) + + def doc_to_decontamination_query(self, doc): + raise NotImplementedError( + "Override doc_to_decontamination_query with document specific decontamination query." + ) + + @abc.abstractmethod + def doc_to_text(self, doc): + pass + + @abc.abstractmethod + def doc_to_target(self, doc): + pass + + # not an abstractmethod because not every language-only task has to implement this + def doc_to_image(self, doc): + raise NotImplementedError + + def doc_to_audio(self, doc): + raise NotImplementedError + + def doc_to_prefix(self, doc): + return "" + + def build_all_requests( + self, + *, + limit: Union[int, None] = None, + samples: Optional[List[int]] = None, + rank: int = 0, + world_size: int = 1, + cache_requests: bool = False, + rewrite_requests_cache: bool = False, + system_instruction: Optional[str] = None, + apply_chat_template: bool = False, + fewshot_as_multiturn: bool = False, + chat_template: Optional[Callable] = None, + tokenizer_name: str = "", + ) -> None: + """Build a set of Instances for a task, and store them in task.instances""" + + # used with caching + og_limit = limit + + cache_key = f"requests-{self._config.task}-{self.config.num_fewshot}shot-rank{rank}-world_size{world_size}" + cache_key += "-chat_template" if apply_chat_template else "" + cache_key += "-fewshot_as_multiturn" if fewshot_as_multiturn else "" + cache_key += ( + f"-system_prompt_hash{utils.hash_string(system_instruction)}" + if system_instruction is not None + else "" + ) + cache_key += f"-tokenizer{tokenizer_name}" + + cached_instances = load_from_cache(file_name=cache_key, cache=cache_requests) + + if cache_requests and cached_instances and not rewrite_requests_cache: + cached_instances = cached_instances[:limit] + + flattened_instances = [ + instance + for instance_group in cached_instances + for instance in instance_group + ] + + self._instances = flattened_instances + return + + eval_logger.info(f"Building contexts for {self.config.task} on rank {rank}...") + + instances = [] + + # process all documents when caching is specified for simplicity + if ( + cache_requests + and (not cached_instances or rewrite_requests_cache) + and limit is not None + ): + limit = None + + doc_id_docs = list( + self.doc_iterator( + rank=rank, limit=limit, samples=samples, world_size=world_size + ) + ) + + num_docs = len(doc_id_docs) + + for doc_id, doc in tqdm( + doc_id_docs, + total=num_docs, + ): + # sample fewshot context #TODO: need to offset doc_id by rank now! + fewshot_ctx = self.fewshot_context( + doc, + num_fewshot=0 + if self.config.num_fewshot is None + else self.config.num_fewshot, + system_instruction=system_instruction, + apply_chat_template=apply_chat_template, + fewshot_as_multiturn=fewshot_as_multiturn, + chat_template=chat_template, + gen_prefix=self.doc_to_prefix(doc), + ) + + # TODO: we should override self.config.repeats if doing greedy gen so users don't waste time+compute + inst = self.construct_requests( + doc=doc, + ctx=fewshot_ctx, + metadata=(self.config["task"], doc_id, self.config.repeats), + apply_chat_template=apply_chat_template, + chat_template=chat_template, + ) + + if not isinstance(inst, list): + inst = [inst] + + instances.append(inst) + + # now flatten, this is to allow slicing to work with pickles + + sliced_instances = instances[:og_limit] + + flattened_instances = [ + instance + for instance_group in sliced_instances + for instance in instance_group + ] + + self._instances = flattened_instances + + if len(self._instances) == 0: + raise ValueError("task.build_requests() did not find any docs!") + + if cache_requests and (not cached_instances or rewrite_requests_cache): + save_to_cache(file_name=cache_key, obj=instances) + + @abc.abstractmethod + def construct_requests(self, doc, ctx, **kwargs): + """Uses RequestFactory to construct Requests and returns an iterable of + Requests which will be sent to the LM. + + :param doc: + The document as returned from training_docs, validation_docs, or test_docs. + :param ctx: str + The context string, generated by fewshot_context. This includes the natural + language description, as well as the few shot examples, and the question + part of the document for `doc`. + :param doc_idx: int + The index of a document within `self.test_docs()` or `self.validation_docs()`, + whichever is the main split used. + :param repeats: int + TODO: update this docstring + The number of times each instance in a dataset is inferred on. Defaults to 1, + can be increased for techniques like majority voting. + """ + pass + + @abc.abstractmethod + def process_results(self, doc, results): + """Take a single document and the LM results and evaluates, returning a + dict where keys are the names of submetrics and values are the values of + the metric for that one document + + :param doc: + The document as returned from training_docs, validation_docs, or test_docs. + :param results: + The results of the requests created in construct_requests. + """ + pass + + @abc.abstractmethod + def aggregation(self): + """ + :returns: {str: [metric_score] -> float} + A dictionary where keys are the names of submetrics and values are + functions that aggregate a list of metric scores + """ + pass + + @abc.abstractmethod + def higher_is_better(self): + """ + :returns: {str: bool} + A dictionary where keys are the names of submetrics and values are + whether a higher value of the submetric is better + """ + pass + + def get_config(self, key: str) -> Any: + return getattr(self._config, key, None) + + @classmethod + def count_bytes(cls, doc): + """Used for byte-level perplexity metrics in rolling loglikelihood""" + return len(doc.encode("utf-8")) + + @classmethod + def count_words(cls, doc): + """Downstream loglikelihood_rolling perplexity tasks with custom word boundaries should override this!""" + return len(re.split(r"\s+", doc)) + + @utils.positional_deprecated + def fewshot_context(self, doc, num_fewshot, rnd=None, description=None, **kwargs): + """Returns a fewshot context string that is made up of a prepended description + (if provided), the `num_fewshot` number of examples, and an appended prompt example. + + :param doc: str + The document as returned from training_docs, validation_docs, or test_docs. + :param num_fewshot: int + The number of fewshot examples to provide in the returned context string. + :param rnd: random.Random + The pseudo-random number generator used to randomly sample examples. + WARNING: This is currently a required arg although it's optionalized with a default `None`. + :param description: str + The task's description that will be prepended to the fewshot examples. + :returns: str + The fewshot context. + """ + if rnd is None: + if self.fewshot_rnd is not None: + rnd = self.fewshot_rnd + else: + raise ValueError( + "A `random.Random` generator argument must be provided to `rnd`" + ) + + description = description if description else "" + + if num_fewshot == 0: + labeled_examples = "" + else: + # for sets with no training docs, draw from other set *but ensure no overlap with current doc* + if self.has_training_docs(): + fewshotex = self.fewshot_examples(k=num_fewshot, rnd=rnd) + else: + if self._fewshot_docs is None: + self._fewshot_docs = list( + self.validation_docs() + if self.has_validation_docs() + else self.test_docs() + ) + + fewshotex = rnd.sample(self._fewshot_docs, num_fewshot + 1) + + # get rid of the doc that's the one we're evaluating, if it's in the fewshot + fewshotex = [x for x in fewshotex if x != doc][:num_fewshot] + + labeled_examples = ( + "\n\n".join( + [ + self.doc_to_text(doc) + self.doc_to_target(doc) + for doc in fewshotex + ] + ) + + "\n\n" + ) + + example = self.doc_to_text(doc) + return description + labeled_examples + example + + def apply_filters(self) -> Optional[List[Instance]]: + """Iterates over FilterEnsembles and applies them to instances""" + if hasattr(self, "_filters"): + for f in self._filters: + f.apply(self._instances) + else: + eval_logger.warning("No filter defined, passing through instances") + return self._instances + + def dump_config(self) -> dict: + """Returns the config as a dictionary.""" + # TODO: this should only return the overrides applied to a non-YAML task's configuration. + # (num_fewshot) + return self.config.to_dict() + + def set_config(self, key: str, value: Any, update: bool = False) -> None: + """Set or update the configuration for a given key.""" + if key is None: + raise ValueError("Key must be provided.") + + if update: + current_value = getattr(self._config, key, {}) + if not isinstance(current_value, dict): + raise TypeError( + f"Expected a dict for key '{key}', got {type(current_value).__name__} instead." + ) + current_value.update(value) + else: + setattr(self._config, key, value) + + def override_metric(self, metric_name: str) -> None: + """ + Override the default metrics used for evaluation with custom metrics. + + Parameters: + - metric_name (str): The name of the custom metric to override. Should be registered in api.metrics. + """ + ( + self._metric_fn_list, + self._aggregation_list, + self._metric_fn_kwargs, + self._higher_is_better, + ) = ({}, {}, {}, {}) + self._metric_fn_list[metric_name] = get_metric(metric_name) + self._aggregation_list[metric_name] = get_metric_aggregation(metric_name) + self._higher_is_better[metric_name] = is_higher_better(metric_name) + self._metric_fn_kwargs[metric_name] = {} + if not isinstance(self, ConfigurableTask): + self.process_results = lambda x, y: {metric_name: get_metric(metric_name)} + self.aggregation = lambda: { + metric_name: get_metric_aggregation(metric_name) + } + setattr(self._config, "metric_list", [{"metric": metric_name}]) + setattr(self._config, "process_results", None) + + def set_fewshot_seed(self, seed: Optional[int] = None) -> None: + self.fewshot_rnd = random.Random(seed) + if hasattr(self, "sampler"): + self.sampler.rnd = self.fewshot_rnd + + @property + def eval_docs(self) -> Union[datasets.Dataset, List[dict]]: + if self.has_test_docs(): + return self.test_docs() + elif self.has_validation_docs(): + return self.validation_docs() + else: + raise ValueError( + f"Task dataset (path={self.DATASET_PATH}, name={self.DATASET_NAME}) must have valid or test docs!" + ) + + def doc_iterator( + self, + *, + rank: int = 0, + limit: Union[int, None] = None, + world_size: int = 1, + samples: Optional[List[int]] = None, + ) -> Iterator[Tuple[int, Any]]: + if samples: + n = len(self.eval_docs) + assert all([e < n for e in samples]), ( + f"Elements of --samples should be in the interval [0,k-1] where k is the number of total examples. In this case, k={n}." + ) + eval_logger.info( + f"{self.config.task}: Evaluating on {len(samples)} examples" + ) + doc_iterator = utils.create_iterator( + enumerate(x for i, x in enumerate(self.eval_docs) if i in samples), + rank=int(rank), + limit=None, # limit does not matter here since we are selecting samples directly + world_size=int(world_size), + ) + else: + limit = int(limit) if limit else None + doc_iterator = utils.create_iterator( + enumerate(self.eval_docs), + rank=int(rank), + limit=limit, + world_size=int(world_size), + ) + return doc_iterator + + +class ConfigurableTask(Task): + VERSION = "Yaml" + OUTPUT_TYPE = None + CONFIG = None + + def __init__( + self, + data_dir=None, + cache_dir=None, + download_mode=None, + config: Optional[dict] = None, + ) -> None: # TODO no super() call here + # Get pre-configured attributes + self._config = self.CONFIG + + # Use new configurations if there was no preconfiguration + if self.config is None: + self._config = TaskConfig(**config) + # Overwrite configs + else: + if config is not None: + self._config.__dict__.update(config) + + if self.config is None: + raise ValueError( + "Must pass a config to ConfigurableTask, either in cls.CONFIG or `config` kwarg" + ) + + if isinstance(self.config.metadata, dict): + if "version" in self.config.metadata: + self.VERSION = self.config.metadata["version"] + + if self.config.output_type is not None: + if self.config.output_type not in ALL_OUTPUT_TYPES: + raise ValueError( + f"Got invalid output_type '{self.config.output_type}', must be in '{','.join(ALL_OUTPUT_TYPES)}'" + ) + self.OUTPUT_TYPE = self.config.output_type + + if self.config.doc_to_image is not None: + # mark the task as requiring multimodality. + self.MULTIMODAL = True + + if self.config.doc_to_audio: + # mark the task as requiring multimodality. + self.MULTIMODAL = True + + if self.config.unsafe_code is not False: + self.UNSAFE_CODE = True + + if self.config.dataset_path is not None: + self.DATASET_PATH = self.config.dataset_path + + if self.config.dataset_name is not None: + self.DATASET_NAME = self.config.dataset_name + + self._metric_fn_list = {} + self._metric_fn_kwargs = {} + self._aggregation_list = {} + self._higher_is_better = {} + + if self.config.metric_list is None: + # TODO: handle this in TaskConfig.__post_init__ ? + _metric_list = DEFAULT_METRIC_REGISTRY[self.config.output_type] + + for metric_name in _metric_list: + self._metric_fn_list[metric_name] = get_metric(metric_name) + self._metric_fn_kwargs[metric_name] = {} + self._aggregation_list[metric_name] = get_metric_aggregation( + metric_name + ) + self._higher_is_better[metric_name] = is_higher_better(metric_name) + else: + for metric_config in self.config.metric_list: + if "metric" not in metric_config: + raise ValueError( + "'metric' key not provided for an entry in 'metric_list', must be specified!" + ) + metric_name = metric_config["metric"] + kwargs = { + key: metric_config[key] + for key in metric_config + if key + not in ["metric", "aggregation", "higher_is_better", "hf_evaluate"] + } + hf_evaluate_metric = ( + "hf_evaluate" in metric_config + and metric_config["hf_evaluate"] is True + ) + + if self.config.process_results is not None: + self._metric_fn_list[metric_name] = None + self._metric_fn_kwargs[metric_name] = {} + elif callable(metric_name): + metric_fn = metric_name.__call__ + metric_name = metric_name.__name__ + self._metric_fn_list[metric_name] = metric_fn + self._metric_fn_kwargs[metric_name] = kwargs + else: + self._metric_fn_list[metric_name] = get_metric( + metric_name, hf_evaluate_metric + ) + self._metric_fn_kwargs[metric_name] = kwargs + + if "aggregation" in metric_config: + agg_name = metric_config["aggregation"] + if isinstance(agg_name, str): + self._aggregation_list[metric_name] = get_aggregation(agg_name) + elif callable(agg_name): # noqa: E721 + self._aggregation_list[metric_name] = metric_config[ + "aggregation" + ] + else: + INV_AGG_REGISTRY = {v: k for k, v in AGGREGATION_REGISTRY.items()} + metric_agg = get_metric_aggregation(metric_name) + eval_logger.warning( + f"[Task: {self.config.task}] metric {metric_name} is defined, but aggregation is not. " + f"using default " + f"aggregation={INV_AGG_REGISTRY[metric_agg]}" + ) + self._aggregation_list[metric_name] = metric_agg + + if "higher_is_better" in metric_config: + self._higher_is_better[metric_name] = metric_config[ + "higher_is_better" + ] + else: + eval_logger.warning( + f"[Task: {self.config.task}] metric {metric_name} is defined, but higher_is_better is not. " + f"using default " + f"higher_is_better={is_higher_better(metric_name)}" + ) + self._higher_is_better[metric_name] = is_higher_better(metric_name) + + self.download(self.config.dataset_kwargs) + self._training_docs = None + self._fewshot_docs = None + + if self.config.filter_list is not None: + self._filters = [] + for filter_config in self.config.filter_list: + filter_name = filter_config["name"] + filter_functions = filter_config["filter"] + components = [] + for function in filter_functions: + kwargs = { + key: function[key] for key in function if key != "function" + } + components.append([function["function"], kwargs]) + filter_pipeline = build_filter_ensemble(filter_name, components) + self._filters.append(filter_pipeline) + else: + # TODO: handle repeats in a more general way rather than just discarding + eval_logger.debug( + "No custom filters defined. Using default 'take_first' filter for handling repeats." + ) + self._filters = [build_filter_ensemble("none", [["take_first", None]])] + + if self.config.use_prompt is not None: + eval_logger.info(f"loading prompt {self.config.use_prompt}") + self.prompt = get_prompt( + self.config.use_prompt, self.DATASET_PATH, self.DATASET_NAME + ) + else: + self.prompt = None + + if self.fewshot_docs() is not None: + self.fewshot_rnd = ( + random.Random() + ) # setting with no seed, to be overridden at a later time + config_sampler: Union[str, Callable] = ( + self.config.fewshot_config.get("sampler", "default") + if self.config.fewshot_config + else "default" + ) + if isinstance(config_sampler, str): + self.sampler = samplers.get_sampler(config_sampler)( + list(self.fewshot_docs()), self, rnd=self.fewshot_rnd + ) + elif callable(config_sampler) and issubclass( + config_sampler, samplers.ContextSampler + ): + self.sampler = config_sampler( + docs=list(self.fewshot_docs()), task=self, rnd=self.fewshot_rnd + ) + else: + raise TypeError( + f"fewshot_config.sampler should be a string or callable of ContextSampler type, " + f"not {type(config_sampler)}" + ) + + self.task_docs = self.eval_docs + + # Test One Doc + self.features = list(self.task_docs.features.keys()) + self.multiple_input = 0 + self.multiple_target = 0 + test_doc = self.task_docs[0] + test_text = self.doc_to_text(test_doc) + test_target = self.doc_to_target(test_doc) + + if self.config.doc_to_choice is not None: + test_choice = self.doc_to_choice(test_doc) + if not isinstance(test_choice, list): + eval_logger.error("doc_to_choice must return list") + else: + num_choice = len(test_choice) + + if isinstance(test_text, int): + eval_logger.debug( + "doc_to_text returned an int. Assuming multiple inputs." + ) + self.multiple_input = num_choice + else: + test_choice = None + + if isinstance(test_target, list): + eval_logger.debug( + "doc_to_target returned a list. Assuming multiple targets." + ) + self.multiple_target = len(test_target) + else: + if (isinstance(test_target, int)) and (test_choice is not None): + test_target = test_choice[test_target] + else: + test_target = str(test_target) + + if test_choice is not None: + check_choices = test_choice + else: + check_choices = [test_target] + if self.config.doc_to_choice is not None: + for choice in check_choices: + choice_has_whitespace = True if choice[0].isspace() else False + delimiter_has_whitespace = ( + True + if self.config.target_delimiter.rstrip() + != self.config.target_delimiter + else False + ) + + if delimiter_has_whitespace and choice_has_whitespace: + eval_logger.debug( + f'Both target_delimiter "{self.config.target_delimiter}" and target choice: "{choice}" have whitespace' + ) + elif (not delimiter_has_whitespace) and (not choice_has_whitespace): + eval_logger.debug( + f'Both target_delimiter "{self.config.target_delimiter}" and target choice: "{choice}" do not have whitespace, ignore if the language you are evaluating on does not require/use whitespace' + ) + + def download( + self, dataset_kwargs: Optional[Dict[str, Any]] = None, **kwargs + ) -> None: + if isinstance(self.config.custom_dataset, Callable): + eval_logger.warning( + f"{self.config.task}: Custom kwargs can be passed to `--metadata` in console (as json string) or to the TaskManager." + + "\nFor example --metadata='{\"max_seq_lengths\":[4096, 8192]}'. For details see task Readme." + ) + self.dataset = self.config.custom_dataset( + **(self.config.metadata or {}), **(self.config.dataset_kwargs or {}) + ) + else: + self.dataset = datasets.load_dataset( + path=self.DATASET_PATH, + name=self.DATASET_NAME, + **dataset_kwargs if dataset_kwargs is not None else {}, + ) + + def has_training_docs(self) -> bool: + if self.config.training_split is not None: + return True + else: + return False + + def has_validation_docs(self) -> bool: + if self.config.validation_split is not None: + return True + else: + return False + + def has_test_docs(self) -> bool: + if self.config.test_split is not None: + return True + else: + return False + + def training_docs(self) -> datasets.Dataset: + if self.has_training_docs(): + if self.config.process_docs is not None: + return self.config.process_docs( + self.dataset[self.config.training_split] + ) + return self.dataset[self.config.training_split] + + def validation_docs(self) -> datasets.Dataset: + if self.has_validation_docs(): + if self.config.process_docs is not None: + return self.config.process_docs( + self.dataset[self.config.validation_split] + ) + return self.dataset[self.config.validation_split] + + def test_docs(self) -> datasets.Dataset: + if self.has_test_docs(): + if self.config.process_docs is not None: + return self.config.process_docs(self.dataset[self.config.test_split]) + return self.dataset[self.config.test_split] + + def fewshot_docs(self): + if self.config.fewshot_split is not None: + if self.config.process_docs is not None: + return self.config.process_docs(self.dataset[self.config.fewshot_split]) + return self.dataset[self.config.fewshot_split] + elif ( + self.config.fewshot_config is not None + and self.config.fewshot_config.get("samples", None) is not None + ): + if isinstance(self.config.fewshot_config["samples"], list): + return self.config.fewshot_config["samples"] + elif callable(self.config.fewshot_config["samples"]): + return self.config.fewshot_config["samples"]() + else: + raise Exception( + "`fewshot_config['samples']` was incorrectly defined in the configuration. It should be either a list of samples as a dict, or function returning this list." + ) + else: + if (self.config.num_fewshot is not None) and (self.config.num_fewshot > 0): + eval_logger.warning( + f"[Task: {self.config.task}] " + "num_fewshot > 0 but fewshot_split is None. " + "using preconfigured rule." + ) + return super().fewshot_docs() + + @staticmethod + def append_target_question( + labeled_examples: List[Dict[str, str]], + question: str, + fewshot_as_multiturn: bool = False, + gen_prefix: Optional[str] = None, + ) -> None: + """Adds a target question to the labeled examples list. + If fewshot_as_multiturn is True, or labeled_examples is empty, or the last entry is a system turn, appends the question as a new user entry. + Otherwise, it is appended to the last user entry, ensuring that the conversation alternates between the user and the assistant. + """ + if not fewshot_as_multiturn: + # if no messages or last message is system, append as new user entry + if len(labeled_examples) == 0 or labeled_examples[-1]["role"] == "system": + labeled_examples.append({"role": "user", "content": question}) + # if last message is user, append to it to avoid two user messages in a row + else: + labeled_examples[-1]["content"] += question + else: + # if fewshot_as_multiturn is True, append as next user entry (last is always assistant) + labeled_examples.append({"role": "user", "content": question}) + if gen_prefix: + labeled_examples.append({"role": "assistant", "content": gen_prefix}) + + @utils.positional_deprecated + def fewshot_context( + self, + doc: dict, + num_fewshot: int, + system_instruction: Optional[str] = None, + apply_chat_template: bool = False, + fewshot_as_multiturn: bool = False, + chat_template: Optional[Callable] = None, + gen_prefix: Optional[str] = None, + ) -> Union[str, List[str]]: + """Returns a fewshot context string that is made up of a prepended description + (if provided), the `num_fewshot` number of examples, and an appended prompt example. + + :param doc: str + The document as returned from training_docs, validation_docs, or test_docs. + :param num_fewshot: int + The number of fewshot examples to provide in the returned context string. + :param system_instruction: str + System instruction to be applied to the prompt. + :param apply_chat_template: bool + Whether to apply the chat template to the fewshot context. + :param fewshot_as_multiturn: bool + Whether to provide the fewshot examples as a multiturn conversation or a single user turn. + :param chat_template: + callable (from lm.apply_chat_template) that takes in a list[Dict] chat transcript and renders it into a string. + :param gen_prefix: + String to append after the <|assistant|> token. + :returns: str + The fewshot context. + """ + if apply_chat_template: + labeled_examples = [] + else: + labeled_examples = "" + + # get task description + if description := self.config.description: + description = utils.apply_template(self.config.description, doc) + + # create system prompt based on the provided system instruction and description + if system_instruction is not None and description: + system_prompt = ( + f"{system_instruction}{self.sampler.fewshot_delimiter}{description}" + ) + elif system_instruction is not None: + system_prompt = system_instruction + elif description: + system_prompt = description + else: + system_prompt = "" + + # add system prompt if specified + if system_prompt: + if apply_chat_template: + labeled_examples.append({"role": "system", "content": system_prompt}) + else: + labeled_examples = system_prompt + # if few-shot - append examples after the system prompt + if num_fewshot > 0: + if apply_chat_template: + labeled_examples.extend( + self.sampler.get_chat_context( + doc, + num_fewshot, + fewshot_as_multiturn, + gen_prefix=gen_prefix, + ) + ) + else: + labeled_examples += self.sampler.get_context( + doc, num_fewshot, gen_prefix=gen_prefix + ) + + example = self.doc_to_text(doc) + if apply_chat_template: + if self.multiple_input: + # TODO: append prefill? + if not labeled_examples: + return "" + return chat_template(labeled_examples) + if isinstance(example, str): + self.append_target_question( + labeled_examples, + example, + fewshot_as_multiturn, + gen_prefix=gen_prefix, + ) + # for loglikelihood create a list of questions with appended choices + elif isinstance(example, list): + labeled_examples_list = [] + # copy chat history for each example and append the answer + for ex in example: + chat = deepcopy(labeled_examples) + self.append_target_question( + chat, + ex, + fewshot_as_multiturn, + gen_prefix=gen_prefix, + ) + # TODO: append prefill? + labeled_examples_list.append( + chat_template( + chat, + add_generation_prompt=False if gen_prefix else True, + ) + ) + return labeled_examples_list + # if example is an integer, append the choice or convert to string + elif isinstance(example, int): + if self.config.doc_to_choice is not None: + choices = self.doc_to_choice(doc) + self.append_target_question( + labeled_examples, + choices[example], + fewshot_as_multiturn, + gen_prefix=gen_prefix, + ) + else: + self.append_target_question( + labeled_examples, + str(example), + fewshot_as_multiturn, + gen_prefix=gen_prefix, + ) + # return lm.apply_chat_template(labeled_examples) + return chat_template( + labeled_examples, + add_generation_prompt=False if gen_prefix else True, + ) + else: + prefix = ( + self.config.target_delimiter + gen_prefix + if gen_prefix is not None + else "" + ) + if self.multiple_input: + return labeled_examples + if isinstance(example, str): + return labeled_examples + example + prefix + elif isinstance(example, list): + return [labeled_examples + ex + prefix for ex in example] + elif isinstance(example, int): + if self.config.doc_to_choice is not None: + choices = self.doc_to_choice(doc) + return labeled_examples + choices[example] + prefix + else: + return labeled_examples + str(example) + prefix + + def apply_filters(self) -> Optional[List[Instance]]: + """Iterates over FilterEnsembles and applies them to instances""" + if hasattr(self, "_filters"): + for f in self._filters: + f.apply(self._instances) + else: + eval_logger.warning("No filter defined, passing through instances") + return self._instances + + def should_decontaminate(self): + return self.config.should_decontaminate + + def doc_to_decontamination_query(self, doc: dict): + if self.config.should_decontaminate: + if self.config.doc_to_decontamination_query is None: + return self.doc_to_text(doc) + else: + doc_to_decontamination_query = self.config.doc_to_decontamination_query + if doc_to_decontamination_query in self.features: + return doc[doc_to_decontamination_query] + elif callable(doc_to_decontamination_query): + return doc_to_decontamination_query(doc) + else: + return ast.literal_eval( + utils.apply_template( + self.config.doc_to_decontamination_query, doc + ) + ) + + def _process_doc(self, doc: dict) -> dict: + """ + Override this to process (detokenize, strip, replace, etc.) individual + documents. This can be used in a map over documents of a data split. + E.g. `map(self._process_doc, self.dataset["validation"])` + + :return: dict + The processed version of the specified `doc`. + """ + return doc + + def doc_to_text(self, doc, doc_to_text=None): + if self.prompt is not None: + doc_to_text = self.prompt + elif doc_to_text is not None: + doc_to_text = doc_to_text + else: + doc_to_text = self.config.doc_to_text + + if isinstance(doc_to_text, int): + return doc_to_text + elif isinstance(doc_to_text, str): + if doc_to_text in self.features: + # if self.config.doc_to_choice is not None: + # return self.doc_to_choice(doc)[doc[doc_to_text]] + # else: + return doc[doc_to_text] + else: + text_string = utils.apply_template(doc_to_text, doc) + if text_string.isdigit() and self._config.doc_to_choice is not None: + return ast.literal_eval(text_string) + else: + return text_string + elif callable(doc_to_text): + return doc_to_text(doc) + # Used when applying a Promptsource template + elif hasattr(doc_to_text, "apply"): + applied_prompt = doc_to_text.apply(doc) + if len(applied_prompt) == 2: + return applied_prompt[0] + else: + eval_logger.warning("Applied prompt returns empty string") + return self.config.fewshot_delimiter + else: + print(type(doc_to_text)) + raise TypeError + + def doc_to_target(self, doc: Mapping, doc_to_target=None) -> Union[int, str, list]: + if self.prompt is not None: + doc_to_target = self.prompt + elif doc_to_target is not None: + doc_to_target = doc_to_target + else: + doc_to_target = self.config.doc_to_target + + if isinstance(doc_to_target, int): + return doc_to_target + elif isinstance(doc_to_target, str): + if doc_to_target in self.features: + # if self.config.doc_to_choice is not None: + # return self.doc_to_choice(doc)[doc[doc_to_target]] + # else: + return doc[doc_to_target] + else: + target_string = utils.apply_template(doc_to_target, doc) + if target_string.isdigit() and self._config.doc_to_choice is not None: + return ast.literal_eval(target_string) + elif ( + len(target_string) >= 2 + and (target_string[0] == "[") + and (target_string[-1] == "]") + ): + try: + return ast.literal_eval(target_string) + except (SyntaxError, ValueError): + return target_string + else: + return target_string + elif isinstance(doc_to_target, list): + return doc_to_target + elif callable(doc_to_target): + return doc_to_target(doc) + # Used when applying a Promptsource template + elif hasattr(doc_to_target, "apply"): + applied_prompt = doc_to_target.apply(doc) + if len(applied_prompt) == 2: + return applied_prompt[1] + else: + eval_logger.warning("Applied prompt returns empty string") + return self.config.fewshot_delimiter + else: + raise TypeError + + def doc_to_choice(self, doc: Any, doc_to_choice=None) -> List[str]: + if self.prompt is not None: + doc_to_choice = self.prompt + elif doc_to_choice is not None: + doc_to_choice = doc_to_choice + elif self.config.doc_to_choice is None: + eval_logger.error("doc_to_choice was called but not set in config") + else: + doc_to_choice = self.config.doc_to_choice + + if isinstance(doc_to_choice, str): + if doc_to_choice in self.features: + return doc[doc_to_choice] + else: + return ast.literal_eval(utils.apply_template(doc_to_choice, doc)) + elif isinstance(doc_to_choice, list): + return doc_to_choice + elif isinstance(doc_to_choice, dict): + return list(doc_to_choice.values()) + elif callable(doc_to_choice): + return doc_to_choice(doc) + elif hasattr(doc_to_choice, "get_answer_choices_list"): + return doc_to_choice.get_answer_choices_list(doc) + else: + raise TypeError + + def doc_to_image(self, doc: Any, doc_to_image=None) -> Union[int, str, list]: + if doc_to_image is not None: + doc_to_image = doc_to_image + elif self.config.doc_to_image is not None: + doc_to_image = self.config.doc_to_image + else: + return None + + if isinstance(doc_to_image, list): + image_feature = [ + self.doc_to_image(doc, feature) for feature in doc_to_image + ] + return [feature for feature in image_feature if feature is not None] + elif isinstance(doc_to_image, str): + if doc_to_image in self.features: + return doc[doc_to_image] + else: + return ast.literal_eval(utils.apply_template(doc_to_image, doc)) + elif callable(doc_to_image): + return doc_to_image(doc) + else: + return None + + def doc_to_audio(self, doc: Any, doc_to_audio=None) -> Union[int, str, list]: + if doc_to_audio is not None: + doc_to_audio = doc_to_audio + elif self.config.doc_to_audio is not None: + doc_to_audio = self.config.doc_to_audio + else: + return None + + if isinstance(doc_to_audio, list): + audio_feature = [ + self.doc_to_audio(doc, feature) for feature in doc_to_audio + ] + return [feature for feature in audio_feature if feature is not None] + elif isinstance(doc_to_audio, str): + if doc_to_audio in self.features: + return doc[doc_to_audio] + else: + return ast.literal_eval(utils.apply_template(doc_to_audio, doc)) + elif callable(doc_to_audio): + return doc_to_audio(doc) + else: + return None + + def doc_to_prefix(self, doc): + if (gen_prefix := self.config.gen_prefix) is not None: + if gen_prefix in self.features: + return doc[gen_prefix] + else: + return utils.apply_template(gen_prefix, doc) + return None + + def construct_requests( + self, doc: dict, ctx: str, **kwargs + ) -> Union[List[Instance], Instance]: + apply_chat_template = kwargs.pop("apply_chat_template", False) + chat_template: Callable | None = kwargs.pop("chat_template", None) + + aux_arguments = None + + if self.OUTPUT_TYPE == "loglikelihood": + arguments = (ctx, self.doc_to_target(doc)) + elif self.OUTPUT_TYPE == "loglikelihood_rolling": + arguments = (self.doc_to_target(doc),) + elif self.OUTPUT_TYPE == "multiple_choice": + choices = self.doc_to_choice(doc) + target_delimiter = self.config.target_delimiter + if apply_chat_template: + target_delimiter = "" + if self.multiple_input: + # If there are multiple inputs, choices are placed in the ctx + # apply chat_template to choices if apply_chat_template + cont = self.doc_to_target(doc) + + arguments = [ + ( + ctx + + ( + chat_template([{"role": "user", "content": choice}]) + if apply_chat_template + else choice + ), + f"{target_delimiter}{cont}", + ) + for choice in choices + ] + else: + # Otherwise they are placed in the continuation + arguments = [(ctx, f"{target_delimiter}{cont}") for cont in choices] + + # TODO: we should raise a warning telling users this will at most ~2x runtime. + if "acc_mutual_info" in self._metric_fn_list.keys(): + # if we are calculating multiple choice accuracy + # using mutual information instead of raw loglikelihood as metric, need unconditional lls. + + # here mutual info refers to calculating + # log(P(choice|ctx) / P(choice)) = log(P(choice|ctx)) - log(P(choice)) + # in other words normalizing by subtracting the unconditional logprob of each choice. + # TODO: should these be strided? will have to modify the processing in process_results if so + aux_arguments = [ + ("", f"{target_delimiter}{choice}") for choice in choices + ] + + arguments.extend(aux_arguments) + + elif self.OUTPUT_TYPE == "generate_until": + arguments = (ctx, deepcopy(self.config.generation_kwargs)) + + multimodal_arg = {} + if ( + self.config.doc_to_image + ): # TODO: ensure that non-multimodal tasks aren't getting visual args + multimodal_arg = { + **multimodal_arg, + **{"visual": self.doc_to_image(doc)}, + } + + if ( + self.config.doc_to_audio + ): # TODO: ensure that non-multimodal tasks aren't getting audio args + multimodal_arg = { + **multimodal_arg, + **{"audio": self.doc_to_audio(doc)}, + } + + if bool(multimodal_arg): + if isinstance(arguments, list): + arguments = [arg + (multimodal_arg,) for arg in arguments] + else: + arguments = arguments + (multimodal_arg,) + + if self.OUTPUT_TYPE == "multiple_choice": + request_list = [ + Instance( + request_type="loglikelihood", + doc=doc, + arguments=arg, + idx=i, + **kwargs, + ) + for i, arg in enumerate(arguments) + ] + + return request_list + + return Instance( + request_type=self.OUTPUT_TYPE, + doc=doc, + arguments=arguments, + idx=0, + **kwargs, + ) + + def process_results(self, doc, results): + if callable(self.config.process_results): + return self.config.process_results(doc, results) + + result_dict = {} + use_metric = list(self._metric_fn_list.keys()) + if self.OUTPUT_TYPE == "loglikelihood": + results = results[0] + ll, is_greedy = results + return { + **({"perplexity": ll} if "perplexity" in use_metric else {}), + **({"acc": int(is_greedy)} if "acc" in use_metric else {}), + } + elif self.OUTPUT_TYPE == "loglikelihood_rolling": + (loglikelihood,) = results + _words = self.count_words(self.doc_to_target(doc)) + _bytes = self.count_bytes(self.doc_to_target(doc)) + return { + **( + {"word_perplexity": (loglikelihood, _words)} + if "word_perplexity" in use_metric + else {} + ), + **( + {"byte_perplexity": (loglikelihood, _bytes)} + if "byte_perplexity" in use_metric + else {} + ), + **( + {"bits_per_byte": (loglikelihood, _bytes)} + if "bits_per_byte" in use_metric + else {} + ), + } + elif self.OUTPUT_TYPE == "multiple_choice": + lls, is_greedy = zip(*results) + + # retrieve choices in List[str] form, to compute choice lengths, etc. + choices = self.doc_to_choice(doc) + completion_len = np.array([float(len(i)) for i in choices]) + + if ( + 2 * len(choices) == len(lls) + and "acc_mutual_info" in self._metric_fn_list.keys() + ): + # then we are doing mutual info. + # this stores the "dryrun" / unconditional answer loglikelihoods + # as we extend the args list with unconditional ("", continuation) pairs + lls_unconditional = lls[len(choices) :] + if len(lls_unconditional) != len(choices): + raise ValueError + # and this stores our "regular" conditional loglikelihoods + lls = lls[: len(choices)] + + pred = np.argmax(lls) + pred_norm = np.argmax(lls / completion_len) + + if self.multiple_input: + gold = self.doc_to_text(doc) + else: + gold = self.doc_to_target(doc) + + gold_index_error = False + if isinstance(gold, list): + gold = [i if i < len(choices) else -100 for i in gold] + if -100 in gold: + gold_index_error = True + else: + if isinstance(gold, int): + gold = gold if gold < len(choices) else -100 + elif isinstance(gold, str): + gold = choices.index(gold) if gold in choices else -100 + + if gold == -100: + gold_index_error = True + + if gold_index_error: + eval_logger.warning( + f"Label index was not in within range of available choices," + f"Sample:\n\n{doc}\n\n" + ) + + if self.multiple_target: + acc = 1.0 if pred in gold else 0.0 + acc_norm = 1.0 if pred_norm in gold else 0.0 + exact_match = int(any([is_greedy[i] if i != -100 else 0 for i in gold])) + else: + acc = 1.0 if pred == gold else 0.0 + acc_norm = 1.0 if pred_norm == gold else 0.0 + # TODO: this gets score of 0 on arc_challenge for pythia-70m. need to test that this works properly + exact_match = int(is_greedy[gold]) if gold != -100 else 0 + + prob_norm = utils.softmax(lls) + + # TODO use keyword arguments to the metric? + # gold, pred, norm stuff, the original lls, + result_dict = { + **({"acc": acc} if "acc" in use_metric else {}), + **({"f1": (gold, pred)} if "f1" in use_metric else {}), + **({"mcc": (gold, pred)} if "mcc" in use_metric else {}), + **({"acc_norm": acc_norm} if "acc_norm" in use_metric else {}), + **({"exact_match": exact_match} if "exact_match" in use_metric else {}), + **( + {"brier_score": (gold, prob_norm)} + if "brier_score" in use_metric + else {} + ), + } + + if "acc_mutual_info" in use_metric: + lls_mutual_info = [ + ll_c - ll_u for ll_c, ll_u in zip(lls, lls_unconditional) + ] + acc_mutual_info = 1.0 if np.argmax(lls_mutual_info) == gold else 0.0 + result_dict["acc_mutual_info"] = acc_mutual_info + + elif self.OUTPUT_TYPE == "generate_until": + gold = self.doc_to_target(doc) + result = results[0] + if self.config.doc_to_choice is not None: + # If you set doc_to_choice, + # it assumes that doc_to_target returns a number. + choices = self.doc_to_choice(doc) + gold = choices[gold] + # we expect multiple_targets to be a list. + elif self.multiple_target: + gold = list(gold) + # TODO: handle this better + elif type(gold) is not type(result) and not ( + "bypass" in self._metric_fn_list.keys() or isinstance(result, list) + ): + # cast gold to the same type as result + gold = type(result)(gold) + + for metric in self._metric_fn_list.keys(): + if self.multiple_target: + # in the case where we have multiple targets, + # return true if any are true + # TODO: this may break for multipLe_target, non zero-or-1 metrics + scores = [] + if not isinstance(gold, list): + # sometimes, a multiple_target dataset has exceptions where one doc has only one string answer + # print(gold) + gold = [gold] + if metric == "exact_match": + result = [result for _ in range(len(gold))] + scores = self._metric_fn_list[metric]( + references=gold, + predictions=result, + **self._metric_fn_kwargs[metric], + )[metric] + result_score = 1.0 if scores > 0.0 else 0.0 + else: + for gold_option in gold: + try: + result_score = self._metric_fn_list[metric]( + references=[gold_option], + predictions=[result], + **self._metric_fn_kwargs[metric], + ) + except ( + TypeError + ): # TODO: this is hacky and I don't want to do it + result_score = self._metric_fn_list[metric]( + [gold_option, result] + ) + if isinstance(result_score, dict): + # TODO: this handles the case where HF evaluate returns a dict. + result_score = result_score[metric] + scores.append(result_score) + if any(scores): + result_score = 1.0 + else: + result_score = 0.0 + else: + try: + result_score = self._metric_fn_list[metric]( + references=[gold], + predictions=[result], + **self._metric_fn_kwargs[metric], + ) + except TypeError: # needed for now in order to use a different interface between our own metrics and HF Evaluate metrics + result_score = self._metric_fn_list[metric]([gold, result]) + if isinstance(result_score, dict): + # TODO: this handles the case where HF evaluate returns a dict. + # This allows for multiple metrics to be returned from the same function + for k, v in result_score.items(): + result_dict[k] = v + else: + result_dict[metric] = result_score + else: + raise ValueError( + f"Passed invalid output_type '{self.OUTPUT_TYPE}' ! Please use one of ", + "'loglikelihood', 'loglikelihood_rolling', 'generate_until' or 'multiple_choice'", + ) + + return result_dict + + def aggregation(self) -> dict: + return self._aggregation_list + + def higher_is_better(self) -> dict: + return self._higher_is_better + + def get_config(self, key: str) -> Any: + return getattr(self._config, key, None) + + @property + def task_name(self) -> Any: + return getattr(self.config, "task", None) + + def __repr__(self): + return ( + f"ConfigurableTask(task_name={getattr(self.config, 'task', None)}," + f"output_type={self.OUTPUT_TYPE}," + f"num_fewshot={getattr(self.config, 'num_fewshot', None)}," + f"num_samples={len(self.eval_docs)})" + ) + + +class MultipleChoiceTask(Task): + OUTPUT_TYPE = "loglikelihood" + + def doc_to_target(self, doc: dict) -> str: + return " " + doc["choices"][doc["gold"]] + + def construct_requests(self, doc: dict, ctx: str, **kwargs) -> List[Instance]: + # TODO: add mutual info here? + return [ + Instance( + request_type="loglikelihood", + doc=doc, + arguments=(ctx, " {}".format(choice)), + idx=i, + **kwargs, + ) + for i, choice in enumerate(doc["choices"]) + ] + + def process_results(self, doc: dict, results: Iterable[Tuple[float, bool]]) -> dict: + results = [ + res[0] for res in results + ] # only retain loglikelihoods, discard is_greedy TODO: do we need is_greedy anywhere? + gold = doc["gold"] + + acc = 1.0 if np.argmax(results) == gold else 0.0 + completion_len = np.array([float(len(i)) for i in doc["choices"]]) + acc_norm = 1.0 if np.argmax(results / completion_len) == gold else 0.0 + + return { + "acc": acc, + "acc_norm": acc_norm, + } + + def higher_is_better(self) -> dict: + return { + "acc": True, + "acc_norm": True, + } + + def aggregation(self) -> dict: + return { + "acc": mean, + "acc_norm": mean, + } + + +class PerplexityTask(Task): + OUTPUT_TYPE = "loglikelihood_rolling" + + def has_training_docs(self) -> bool: + return False + + def fewshot_examples(self, k: int, rnd) -> List: + if k != 0: + raise ValueError( + "The number of fewshot examples must be 0 for perplexity tasks." + ) + return [] + + def fewshot_context(self, doc: dict, num_fewshot: int) -> Literal[""]: + if num_fewshot != 0: + raise ValueError( + "The number of fewshot examples must be 0 for perplexity tasks." + ) + + return "" + + def higher_is_better(self) -> dict: + return { + "word_perplexity": False, + "byte_perplexity": False, + "bits_per_byte": False, + } + + def doc_to_decontamination_query(self, doc): + return doc + + def doc_to_text(self, doc) -> str: + return "" + + def doc_to_target(self, doc): + return doc + + def construct_requests(self, doc: dict, ctx: Optional[str], **kwargs): + if bool(ctx): + raise ValueError + + return Instance( + request_type=self.OUTPUT_TYPE, + doc=doc, + arguments=(self.doc_to_target(doc),), + idx=0, + **kwargs, + ) + + def process_results(self, doc: dict, results: Tuple[float]) -> dict: + (loglikelihood,) = results + words = self.count_words(self.doc_to_target(doc)) + bytes_ = self.count_bytes(self.doc_to_target(doc)) + return { + "word_perplexity": (loglikelihood, words), + "byte_perplexity": (loglikelihood, bytes_), + "bits_per_byte": (loglikelihood, bytes_), + } + + def aggregation(self) -> dict: + return { + "word_perplexity": weighted_perplexity, + "byte_perplexity": weighted_perplexity, + "bits_per_byte": bits_per_byte, + } + + @classmethod + def count_bytes(cls, doc) -> int: + return len(doc.encode("utf-8")) + + @classmethod + def count_words(cls, doc) -> int: + """Downstream tasks with custom word boundaries should override this!""" + return len(re.split(r"\s+", doc)) diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/caching/__init__.py b/Prism/LLaDA/LLaDA_Prism/dllm_eval/caching/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/caching/cache.py b/Prism/LLaDA/LLaDA_Prism/dllm_eval/caching/cache.py new file mode 100644 index 0000000000000000000000000000000000000000..f8d293b0ff8b1ebac186f5ac078cdb49227562db --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/dllm_eval/caching/cache.py @@ -0,0 +1,59 @@ +import hashlib +import logging +import os + +import dill + + +eval_logger = logging.getLogger(__name__) + + +MODULE_DIR = os.path.dirname(os.path.realpath(__file__)) + +OVERRIDE_PATH = os.getenv("LM_HARNESS_CACHE_PATH") + + +PATH = OVERRIDE_PATH if OVERRIDE_PATH else f"{MODULE_DIR}/.cache" + +# This should be sufficient for uniqueness +HASH_INPUT = "EleutherAI-lm-evaluation-harness" + +HASH_PREFIX = hashlib.sha256(HASH_INPUT.encode("utf-8")).hexdigest() + +FILE_SUFFIX = f".{HASH_PREFIX}.pickle" + + +def load_from_cache(file_name: str, cache: bool = False): + if not cache: + return + try: + path = f"{PATH}/{file_name}{FILE_SUFFIX}" + + with open(path, "rb") as file: + cached_task_dict = dill.loads(file.read()) + return cached_task_dict + + except Exception: + eval_logger.debug(f"{file_name} is not cached, generating...") + pass + + +def save_to_cache(file_name, obj): + if not os.path.exists(PATH): + os.mkdir(PATH) + + file_path = f"{PATH}/{file_name}{FILE_SUFFIX}" + + eval_logger.debug(f"Saving {file_path} to cache...") + with open(file_path, "wb") as file: + file.write(dill.dumps(obj)) + + +# NOTE the "key" param is to allow for flexibility +def delete_cache(key: str = ""): + files = os.listdir(PATH) + + for file in files: + if file.startswith(key) and file.endswith(FILE_SUFFIX): + file_path = f"{PATH}/{file}" + os.unlink(file_path) diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/decontamination/__init__.py b/Prism/LLaDA/LLaDA_Prism/dllm_eval/decontamination/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/decontamination/archiver.py b/Prism/LLaDA/LLaDA_Prism/dllm_eval/decontamination/archiver.py new file mode 100644 index 0000000000000000000000000000000000000000..c132232116c2ae5f5ab1dc3a2a0afc0dbd4ef1bd --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/dllm_eval/decontamination/archiver.py @@ -0,0 +1,174 @@ +import datetime +import io +import json +import mmap +import os +from pathlib import Path +from typing import Any + +import jsonlines +import tqdm +import zstandard + + +def json_serial(obj: Any) -> str: + """JSON serializer for objects not serializable by default json code""" + + if isinstance(obj, (datetime.datetime,)): + return obj.isoformat() + raise TypeError("Type %s not serializable" % type(obj)) + + +# Modified version of lm_dataformat Archive for single file. +class Archive: + def __init__(self, file_path: str, compression_level: int = 3) -> None: + self.file_path = file_path + dir_name = os.path.dirname(file_path) + if dir_name: + os.makedirs(dir_name, exist_ok=True) + self.fh = open(self.file_path, "wb") + self.cctx = zstandard.ZstdCompressor(level=compression_level) + self.compressor = self.cctx.stream_writer(self.fh) + + def add_data(self, data, meta=None) -> None: + if meta is None: + meta = {} + self.compressor.write( + json.dumps({"text": data, "meta": meta}, default=json_serial).encode( + "UTF-8" + ) + + b"\n" + ) + + def commit(self) -> None: + self.compressor.flush(zstandard.FLUSH_FRAME) + self.fh.flush() + self.fh.close() + + +# Modified version of lm_dataformat Reader with self.fh set, allowing peeking for tqdm. +class Reader: + def __init__(self) -> None: + pass + + def read( + self, + file, + get_meta: bool = False, + autojoin_paragraphs: bool = True, + para_joiner: str = "\n\n", + ): + with open(file, "rb") as fh: + self.fh = fh + cctx = zstandard.ZstdDecompressor() + reader = io.BufferedReader(cctx.stream_reader(fh)) + rdr = jsonlines.Reader(reader) + for ob in rdr: + # naive jsonl where each object is just the string itself, with no meta. For legacy compatibility. + if isinstance(ob, str): + assert not get_meta + yield ob + continue + + text = ob["text"] + + if autojoin_paragraphs and isinstance(text, list): + text = para_joiner.join(text) + + if get_meta: + yield text, (ob["meta"] if "meta" in ob else {}) + else: + yield text + + +class TextArchive: + def __init__(self, file_path, mode: str = "rb+") -> None: + self.file_path = file_path + dir_name = os.path.dirname(file_path) + if dir_name: + os.makedirs(dir_name, exist_ok=True) + + if not os.path.exists(file_path): + Path(file_path).touch() + + self.fh = open(self.file_path, mode) + + def add_data(self, data) -> None: + self.fh.write(data.encode("UTF-8") + b"\n") + + def commit(self) -> None: + self.fh.flush() + self.fh.close() + + +class TextReader: + def __init__(self, file_path) -> None: + self.file_path = file_path + + # Optimized mmap read with infrequent tqdm updates to maintain speed + # Tested up to 250MB/s. + def read_tqdm(self, update_frequency: int = 10000): + current_file_position = 0 + line_counter = 0 + with ( + open(self.file_path, "r", encoding="utf-8") as fh, + tqdm.tqdm( + total=os.path.getsize(self.file_path), + dynamic_ncols=True, + unit="byte", + unit_scale=1, + ) as progress, + ): + with mmap.mmap(fh.fileno(), length=0, access=mmap.ACCESS_READ) as mmap_obj: + for line in iter(mmap_obj.readline, b""): + line = line.decode("utf-8") + line_counter += 1 + if line_counter == update_frequency: + new_file_pos = mmap_obj.tell() + bytes_read = new_file_pos - current_file_position + current_file_position = new_file_pos + progress.update(bytes_read) + line_counter = 0 + yield line[:-1] + + def read_and_tell(self): + current_file_position = 0 + with open(self.file_path, "r", encoding="utf8") as fh: + with mmap.mmap(fh.fileno(), length=0, access=mmap.ACCESS_READ) as mmap_obj: + for line in iter(mmap_obj.readline, b""): + line = line.decode("utf-8") + new_file_pos = mmap_obj.tell() + raw_bytes_read = new_file_pos - current_file_position + current_file_position = new_file_pos + yield line[:-1], raw_bytes_read + + def read(self): + with open(self.file_path, "r", encoding="utf8") as fh: + with mmap.mmap(fh.fileno(), length=0, access=mmap.ACCESS_READ) as mmap_obj: + for line in iter(mmap_obj.readline, b""): + line = line.decode("utf-8") + yield line[:-1] + + def read_slow(self): + with open(self.file_path, "r", encoding="utf8") as fh: + while True: + line = fh.readline() + if line == -1 or line == "": + break + else: + yield line[:-1] + + +# Optimized for speed. Decompresses the archive in shell before +# using the mmap'd TextReader. +class ZStdTextReader: + def __init__(self, file) -> None: + self.file = file + + def read_tqdm(self): + decompressed_file = self.file[:-4] + print("Decompressing file, please wait...") + os.system(f"zstd -d {self.file}") # linux decompress is faster + reader = TextReader(decompressed_file) + yield from reader.read_tqdm() + os.remove(decompressed_file) diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/decontamination/decontaminate.py b/Prism/LLaDA/LLaDA_Prism/dllm_eval/decontamination/decontaminate.py new file mode 100644 index 0000000000000000000000000000000000000000..2d1250d39bf7cd0272e412452d970ec7c52992c5 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/dllm_eval/decontamination/decontaminate.py @@ -0,0 +1,166 @@ +import collections +import glob +import json +import os +import pickle +import random +import time + +from .archiver import ZStdTextReader +from .janitor import Janitor, word_ngrams + + +# Was used for testing the evaluator decoupled from the full logic below +def get_train_overlap_stub(docs: dict, ngrams_path: str, ngrams_n_size: str): + simulated_overlap = 0.1 + contaminated = int(len(docs) * simulated_overlap) + return random.sample(range(len(docs)), contaminated) + + +# Returns a dictionary containing all overlapping documents in each +# task. In the standard use case, an overlap occurs when any of the 13-grams +# found in the task document exist in the training set documents. +# +# To generate 13-grams for the pile see scripts/clean_training_data. The final output of these +# scripts are an info.json file containing the n_gram_size (13) and a bunch of "ngrams_{x}.bkt.txt.sorted.zst" +# files. These should exist in the "ngrams_path" provided to this function. + + +# Algorithm: +# 1. Build lookups for each dataset {ngram: list(document_ids)} +# 2. Merge into an overall lookup {ngram: [(task_name, task_set, doc_ids),]} +# 3. Full scan the 13-grams from the training set against the merged lookup, +# saving matches in the "duplicates" dictionary {(task_name, task_set): set(doc_ids)} +# 4. Strip the task_set from the dictionary keys and return +# +# We cache the task+set lookups as well as the overlaps. +def get_train_overlap(docs_by_task_set: dict, ngrams_path: str, limit: int) -> dict: + # return get_train_overlap_stub(docs, ngrams_path, ngrams_n_size) + + info_dict_path = os.path.join(ngrams_path, "info.json") + info_dict = json.load(open(info_dict_path, "r", encoding="utf-8")) + ngrams_n_size = info_dict["ngram_size"] + + janitor = Janitor() + + # Build lookup for each dataset first in case we use different task combinations later + print("Building Lookups...") + start = time.perf_counter() + + def get_overlaps_dump_path(task_name, task_set, ngrams_n_size, limit) -> str: + return f"data/{task_name}/{task_set}_{ngrams_n_size}grams_limit{limit}.overlaps" + + lookups = {} + duplicates = {} # (task_name, task_set): set(doc_ids)} + sets_to_decontaminate = len(docs_by_task_set.keys()) + + for (task_name, task_set), docs in docs_by_task_set.items(): + if not os.path.exists(f"data/{task_name}"): + os.mkdir(f"data/{task_name}") + + # Check if we've decontaminated this combination before + overlaps_dump_path = get_overlaps_dump_path( + task_name, task_set, ngrams_n_size, limit + ) + if os.path.exists(overlaps_dump_path): + duplicates[(task_name, task_set)] = pickle.load( + open(overlaps_dump_path, "rb") + ) + sets_to_decontaminate -= 1 + continue + else: + duplicates[(task_name, task_set)] = set() + + # Build/load the task lookup {ngram: set(documents)}. + task_set_lookup_path = ( + f"data/{task_name}/{task_set}_{ngrams_n_size}grams_limit{limit}.lookup" + ) + if os.path.exists(task_set_lookup_path): + print(f"{task_set_lookup_path} available, loading...") + lookups[(task_name, task_set)] = pickle.load( + open(task_set_lookup_path, "rb") + ) + else: + print(f"{task_set_lookup_path} not available, building...") + lookup = collections.defaultdict(set) + + for doc_id, document in enumerate(docs): + ngrams = word_ngrams(janitor.normalize_string(document), ngrams_n_size) + for ngram in ngrams: + lookup[ngram].add(doc_id) + + pickle.dump(lookup, open(task_set_lookup_path, "wb")) + lookups[(task_name, task_set)] = lookup + + elapsed = time.perf_counter() - start + print(f"Building lookups took {elapsed:0.5f} seconds.") + + matched_ngrams = [] + + if sets_to_decontaminate > 0: + print("Merging lookups...") + start = time.perf_counter() + merged_lookup = collections.defaultdict(list) + for (task_name, task_set), lookup in lookups.items(): + for ngram, doc_ids in lookup.items(): + merged_lookup[ngram].append((task_name, task_set, doc_ids)) + + elapsed = time.perf_counter() - start + print(f"Merging lookups took {elapsed:0.5f} seconds.") + + print(f"{ngrams_n_size} grams files found in {ngrams_path}:") + files = glob.glob(os.path.join(ngrams_path, "*.sorted.zst")) + print(files) + + for file in files: + start = time.perf_counter() + print(f"Scanning {file}") + reader = ZStdTextReader(file) + total_ngrams = 0 + unique_ngrams = 0 + matching_unique = 0 + non_matching_unique = 0 + + current_ngram = "" + for line in reader.read_tqdm(): # Scan training set ngrams file + total_ngrams += 1 + [ngram, document_id] = line.rsplit(" ", 1) + if ( + ngram != current_ngram + ): # Only need to match the ngram once in training set + unique_ngrams += 1 + current_ngram = ngram + if ngram in merged_lookup: + matched_ngrams.append(ngram) # For logging + matching_unique += 1 + for task_name, task_set, doc_ids in merged_lookup[ngram]: + task_doc_set = duplicates[(task_name, task_set)] + for doc_id in doc_ids: # Record contamination across all relevant task/set combos + task_doc_set.add(doc_id) + del merged_lookup[ngram] # No point matching again + else: + non_matching_unique += 1 + + print(f"Total Ngrams: {total_ngrams}") + print(f"Unique Ngrams: {unique_ngrams}") + print(f"Unique Matching: {matching_unique}") + print(f"Unique Non Matching: {non_matching_unique}") + print("Matched ngrams:") + for ngram in matched_ngrams: + print(ngram) + + elapsed = time.perf_counter() - start + print(f"Read took {elapsed:0.5f} seconds.") + print(f"Speed: {(os.path.getsize(file) / 1000000.0) / elapsed}MB/second") + + print(duplicates) + + # Dump overlaps separately + for (task_name, task_set), doc_ids in duplicates.items(): + overlaps_dump_path = get_overlaps_dump_path( + task_name, task_set, ngrams_n_size, limit + ) + pickle.dump(doc_ids, open(overlaps_dump_path, "wb")) + + # Strip task set and return + return {task_name: doc_ids for (task_name, task_set), doc_ids in duplicates.items()} diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/decontamination/janitor.py b/Prism/LLaDA/LLaDA_Prism/dllm_eval/decontamination/janitor.py new file mode 100644 index 0000000000000000000000000000000000000000..cedf8a5717aa8156674836ba236fdcabf36e0487 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/dllm_eval/decontamination/janitor.py @@ -0,0 +1,328 @@ +import pickle +import re +import string +import traceback +from typing import Iterator, List, Sequence, Tuple, TypeVar + + +# This is a cpp module. Compile janitor_util.cpp with: +# c++ -O3 -Wall -shared -std=c++11 -fPIC $(python3 -m pybind11 --includes) janitor_util.cpp -o janitor_util$(python3-config --extension-suffix) -undefined dynamic_lookup +try: + import janitor_util + + JANITOR_CPP = True +except Exception: + print("WARNING: C++ module could not be loaded. Janitor running in python mode") + traceback.print_exc() + JANITOR_CPP = False + +T = TypeVar("T") + + +# Implementation from nltk source +# https://www.nltk.org/_modules/nltk/util.html +def form_ngrams(sequence: Iterator[T], n: int) -> Iterator[Tuple[T, ...]]: + history = [] + while n > 1: + # PEP 479, prevent RuntimeError from being raised when StopIteration bubbles out of generator + try: + next_item = next(sequence) + except StopIteration: + # no more data, terminate the generator + return + history.append(next_item) + n -= 1 + for item in sequence: + history.append(item) + yield tuple(history) + del history[0] + + +def word_ngrams(s: str, n: int) -> Iterator[str]: + """Splits a string into ngram words""" + tokens = s.split() # not a generator :( + ngram_seqs = form_ngrams(iter(tokens), n) + return (" ".join(ngram) for ngram in ngram_seqs) + + +# Does character sequences only - combined faster function to play around with later +# def word_ngrams_indices_combined(sequence, n): +# current_word = "" +# history = [] +# gap = False; +# start = 0 +# end = 0 +# for character in sequence: +# if character == " ": +# if not gap: +# gap = True +# history.append(current_word) +# end += len(current_word) - 1 +# current_word = "" +# if len(history) == n: +# yield (tuple(history), start, end) +# del history[0] +# start = end + 1 +# end = start +# else: +# gap = False +# current_word += character + + +# https://stackoverflow.com/questions/13734451/string-split-with-indices-in-python +def split_indices(s: str) -> Iterator[Tuple[str, Tuple[int, int]]]: + """Splits a string on whitespaces and records the indices of each in the original string. + @:return generator((word, (start_idx, end_idx)), ...) + """ + return ((m.group(0), (m.start(), m.end() - 1)) for m in re.finditer(r"\S+", s)) + + +def word_ngrams_indices(s: str, n: int) -> Iterator[Tuple[str, Tuple[int, int]]]: + """Splits a string into pairs of (ngram words, their start/end indices)""" + tokens_with_indices = split_indices(s) + + # Generator of ngrams of (word, idx_pairs) + # ( + # [(word, (start,end)), (word, (start, end))...], + # [(word, (start, end)), ...], + # ... + # ) + ngram_seqs_with_indices = form_ngrams(tokens_with_indices, n) + + # Generator of pairs of word and index ngrams + # ( + # ([word, word, ...], [(start,end), (start,end), ...]), + # ... + # ) + ngram_indices_pairs = ( + zip(*ngram_with_indices) for ngram_with_indices in ngram_seqs_with_indices + ) + + # Generator of ( (word_ngram, (start, end)), (word_ngram, start, end)), ...) + return ( + (" ".join(ngram_seq), (indices[0][0], indices[-1][1])) + for ngram_seq, indices in ngram_indices_pairs + ) + + +class Janitor: + # FIXME delete_chars: Should anything else go here? Special chars? + def __init__( + self, + ngram_n: int = 13, + window_to_remove: int = 200, + too_dirty_cutoff: int = 10, + minimum_slice_length: int = 200, + delete_chars: str = string.punctuation, + ) -> None: + self.ngram_n = ngram_n + self.window_to_remove = window_to_remove + self.too_dirty_cutoff = too_dirty_cutoff + self.minimum_slice_length = minimum_slice_length + self.delete_chars = delete_chars + + self.dirt_ngrams = set() + + # If in python, we'll translate uppercase to lowercase and delete naughty characters. + # This is fast by python standards + # https://stackoverflow.com/questions/638893/what-is-the-most-efficient-way-in-python-to-convert-a-string-to-all-lowercase-st + self.translation_table = str.maketrans( + string.ascii_lowercase + string.ascii_uppercase, # These characters + string.ascii_lowercase * 2, # Become these characters + self.delete_chars, # These are deleted + ) + + ############## + # I/O for saving contamination ngrams + ############## + + def save_contamination_ngrams(self, filename: str) -> None: + with open(filename, "wb") as fp: + pickle.dump(filename, fp) + + def load_contamination_ngrams(self, filename: str) -> None: + with open(filename, "rb") as fp: + self.dirt_ngrams = pickle.load(fp) + + ############## + # Call these :) + ############## + + def register_contaminant(self, dirt_string: str) -> None: + """Register a string as contamination to be removed, e.g. a test set + This breaks the dirt_string into ngrams to store for future cleaning""" + if JANITOR_CPP: + return self.register_contaminant_cpp(dirt_string) + else: + print("WARNING: Janitor running in python mode") + return self.register_contaminant_python(dirt_string) + + def clean(self, dirty_string: str) -> List[str]: + """Clean a string (e.g. a training set) by removing all ngrams previously + registered as contaminants. Returns a list of clean chunks, or empty if + the string was too dirty""" + if JANITOR_CPP: + return self.clean_cpp(dirty_string) + else: + print("WARNING: Janitor running in python mode") + return self.clean_python(dirty_string) + + def _split_chunks( + self, dirty_string: str, dirty_parts: Sequence[Tuple] + ) -> List[str]: + clean_chunks = [] + splice_idx = 0 + end = -1 + for i, (ngram, start, end) in enumerate(dirty_parts): + if i >= self.too_dirty_cutoff: + return [] + start = max(0, start - self.window_to_remove) + end = min(len(dirty_string), end + self.window_to_remove) + + if start - splice_idx > self.minimum_slice_length: + clean_chunks.append(dirty_string[splice_idx:start]) + splice_idx = end + + if end < len(dirty_string) - self.minimum_slice_length: + clean_chunks.append(dirty_string[end + 1 :]) + + return clean_chunks + + ############## + # Fast C++ + ############## + + def register_contaminant_cpp(self, dirt_string) -> None: + self.dirt_ngrams.update( + janitor_util.clean_ngram(dirt_string, self.delete_chars, self.ngram_n) + ) + + def clean_cpp(self, dirty_string: str) -> List[str]: + contamination_indices = janitor_util.clean_ngram_with_indices( + dirty_string, self.delete_chars, self.ngram_n + ) + return self._split_chunks(dirty_string, contamination_indices) + + ############## + # Slow python + ############## + + def normalize_string(self, s: str) -> str: + return s.translate(self.translation_table) + + def register_contaminant_python(self, dirt_string: str) -> None: + self.dirt_ngrams.update( + word_ngrams(self.normalize_string(dirt_string), self.ngram_n) + ) + + def clean_python(self, dirty_string: str) -> List[str]: + contamination_indices = ( + (None, *idx_pair) + for dirty_ngram, idx_pair in word_ngrams_indices(dirty_string, self.ngram_n) + if self.normalize_string(dirty_ngram) in self.dirt_ngrams + ) + return self._split_chunks(dirty_string, contamination_indices) + + +################################################################## +# Tests +################################################################# + +# def print_cpp(): +# source = """ ,, I'm a very !dirty,, ,, dirty boy. Clean me daddy. \n\nhe he he hehe heh. lastword """ * 2 + +# for i in range(1, 10, 2): +# pprint(janitor_util.clean_ngram(source, string.punctuation, i)) +# for ngram, start, end in \ +# janitor_util.clean_ngram_with_indices(source, string.punctuation, i): +# print(ngram, "\t", start, end, source[start:end].replace("\n", "\\n")) + + +# def test_cpp(): +# source = """ ,, I'm a very !dirty,, ,, dirty boy. Clean me daddy. \n\nhe he he hehe heh. lastword """ * 2 +# contaminant = "dirty boy. Clean he he" + +# jan_python = Janitor() +# jan_cpp = Janitor() + +# jan_python.register_contaminant_python(contaminant) +# jan_cpp.register_contaminant(contaminant) + +# assert jan_python.dirt_ngrams == jan_cpp.dirt_ngrams, (jan_python.dirt_ngrams, jan_cpp.dirt_ngrams) + +# assert jan_python.clean_python(source) == jan_cpp.clean(source), \ +# (jan_python.clean_python(source), jan_cpp.clean(source)) + +# print("Passed test, python==cpp") + + +# def benchmark(): +# # Download and put in data folder: enwik8 (100 MB) from https://cs.fit.edu/~mmahoney/compression/textdata.html +# setup = \ +# """ +# with open("data/enwik8", "r") as f: +# data = f.read() +# jan = Janitor(too_dirty_cutoff=1000) +# jan.register_contaminant(''' +# theories is that there is a connection between "geekdom" and autism. +# This is hinted, for instance, by a ''Wired Magazine'' article in 2001 entitled " +# The [[Geek]] Syndrome", which is a point argued by many in the autism rights +# movement{{ref|Wired}}. This article, many professionals assert, is just one example of +# the media's application of mental disease labels to what is actually variant normal behavior +# &mdash;they argue that shyness, lack of athletic ability or social skills, and intellectual +# interests, even when they seem unusual to others, are not in themselves signs of autism or +# Asperger's syndrome. Others assert that it is actually the medical profession which is applying +# mental disease labels to children who in the past would have simply been accepted as a little +# different or even labeled 'gifted'. See [[clinomorphism]] for further discussion of this issue. +# Due to the recent publicity surrounding autism and autis +# ultan Al Nahyan]] granted [[Petroleum]] concessions, and oil was first found in 1958. At first, +# oil money had a marginal impact. A few lowrise concete buildings were erected, and the first +# paved road was completed in 1961, but Sheikh Shakbut, uncertain whether the new oil royalties +# would last, took a cautious approach, preferring to save the revenue rather than investing it in +# development. His brother, [[Zayed bin Sultan Al Nahayan]], saw that oil wealth had the potential +# to transform Abu Dhabi. The ruling Al Nahayan family decided that Sheikh Zayed should replace his +# brother as Ruler and carry out his vision of developing the country. On [[August 6]], [[1966]], +# with the assistance of the British, Sheikh Zayed became the new ruler. See generally, Al-Fahim, M, +# ''From Rags to Riches: A Story of Abu Dhabi'', Chapter Six (London Centre of Arab Studies, 1995), +# ISBN 1 900404 00 1. With the announcement by Britain in 1968 that it would withdraw from the +# Gulf area by 1971, Sheikh Zayed became the main driving force behind the formation of the +# [[United Arab Emirates]]. After the Emirates gained independence in 1971, +# ''') +# """ + +# n = 1 +# print(f"Timing {n} run on 100 MB") +# print("Register contaminant") +# # print("\tPython", timeit.timeit("jan.register_contaminant_python(data)", setup=setup, globals=globals(), number=n)) +# print("\tCpp", timeit.timeit("jan.register_contaminant(data)", setup=setup, globals=globals(), number=n)) + +# print("Clean") +# # print("\tPython", timeit.timeit("jan.clean_python(data)", setup=setup, globals=globals(), number=n)) +# print("\tCpp", timeit.timeit("jan.clean(data)", setup=setup, globals=globals(), number=n)) + + +# def test_janitor_general(): +# source = """ ,, I'm a very !dirty,, ,, dirty boy. Clean me daddy. \n\nhe he he hehe heh. lastword """ * 2 +# contaminant = "dirty boy. Clean he he" + +# jan = Janitor(ngram_n=3) +# jan.register_contaminant(contaminant) +# cleaned = " ".join(jan.clean(source)) +# for contam in jan.dirt_ngrams: +# assert contam not in cleaned, contam + +# filename = "data/saved_contam" +# jan.save_contamination_ngrams(filename) + +# jan = Janitor(ngram_n=3) +# jan.load_contamination_ngrams(filename) +# cleaned = " ".join(jan.clean(source)) +# for contam in jan.dirt_ngrams: +# assert contam not in cleaned, contam + + +# if __name__ == "__main__": +# test() +# # print_cpp() +# # test_cpp() +# # benchmark() diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/filters/__init__.py b/Prism/LLaDA/LLaDA_Prism/dllm_eval/filters/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8911d26c34cc07d1c92d20b904f48ef6fcce8ea4 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/dllm_eval/filters/__init__.py @@ -0,0 +1,25 @@ +from functools import partial +from typing import List + +from dllm_eval.api.filter import FilterEnsemble +from dllm_eval.api.registry import get_filter + +from . import custom, extraction, selection, transformation + + +def build_filter_ensemble( + filter_name: str, components: List[List[str]] +) -> FilterEnsemble: + """ + Create a filtering pipeline. + """ + filters = [] + for function, kwargs in components: + if kwargs is None: + kwargs = {} + # create a filter given its name in the registry + f = partial(get_filter(function), **kwargs) + # add the filter as a pipeline step + filters.append(f) + + return FilterEnsemble(name=filter_name, filters=filters) diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/filters/__pycache__/selection.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/dllm_eval/filters/__pycache__/selection.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4e57161d05b9b8bb5d8b41652452f73ee7d3c278 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/dllm_eval/filters/__pycache__/selection.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/filters/custom.py b/Prism/LLaDA/LLaDA_Prism/dllm_eval/filters/custom.py new file mode 100644 index 0000000000000000000000000000000000000000..07576f8a503f816de42ca1a80729edb517d75a5c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/dllm_eval/filters/custom.py @@ -0,0 +1,17 @@ +from dllm_eval.api.filter import Filter +from dllm_eval.api.registry import register_filter + + +@register_filter("custom") +class CustomFilter(Filter): + """ + Custom filter that applies a custom, user-defined function to the model responses. + """ + + def __init__(self, **kwargs) -> None: + self.filter_fn = kwargs.pop("filter_fn") + + super().__init__(**kwargs) + + def apply(self, resps, docs): + return self.filter_fn(resps, docs) diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/filters/decontamination.py b/Prism/LLaDA/LLaDA_Prism/dllm_eval/filters/decontamination.py new file mode 100644 index 0000000000000000000000000000000000000000..ff4ff15a2d856a0a191aaeb5288c3706275dddd8 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/dllm_eval/filters/decontamination.py @@ -0,0 +1,25 @@ +from dllm_eval.api.filter import Filter +from dllm_eval.api.registry import register_filter + + +@register_filter("decontaminate") +class DecontaminationFilter(Filter): + """ + A filter which evaluates + """ + + name = "track_decontamination" + + def __init__(self, path) -> None: + """ + + TODO: make sure only ever run one time on the train set (should this be cached as a class var? keyed by value for "path"). + should further cache result on a given (task_name, doc_id) + """ + self._decontam_results = None + + def apply(self, resps, docs) -> None: + """ + Return {"no_contamination", "only_contamination"} keys for the 2 different subsets + """ + pass diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/filters/extraction.py b/Prism/LLaDA/LLaDA_Prism/dllm_eval/filters/extraction.py new file mode 100644 index 0000000000000000000000000000000000000000..3998e7c463e5f75cff6ed19c135441cc40ba3c8b --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/dllm_eval/filters/extraction.py @@ -0,0 +1,233 @@ +import re +import sys +import unicodedata + +from dllm_eval.api.filter import Filter +from dllm_eval.api.registry import register_filter + + +@register_filter("regex") +class RegexFilter(Filter): + """A filter that extracts values from text using regex pattern matching. + + This filter applies a regex pattern to each model response and extracts matched values. + If no match is found, returns a fallback value. Useful for extracting structured data + (like numbers) from unstructured model outputs. + """ + + def __init__( + self, + regex_pattern: str = r"#### (\-?[0-9\.\,]+)", + group_select: int = 0, + fallback: str = "[invalid]", + ) -> None: + """ + pass a string `regex` to run `re.compile(r"regex")` on. + `fallback` defines the output returned if no matches for the regex are located. + """ + self.regex_pattern = regex_pattern + self.regex = re.compile(regex_pattern) + self.group_select = group_select + self.fallback = fallback + + def apply(self, resps: list[list[str]], docs: list[dict]) -> list[list[str]]: + # here, we assume we have a list, in which each element is + # a list of model responses for some particular input/target pair. + # so we process each of these (same input/target response sets) + # independently (and keep them a list.) + def filter_set(inst): + filtered = [] + for resp in inst: + match = self.regex.findall(resp) + if match: + match = match[self.group_select] + if isinstance(match, tuple): + match = [m for m in match if m] + if match: + match = match[0] + else: + match = self.fallback + match = match.strip() + else: + match = self.fallback + filtered.append(match) + return filtered + + filtered_resps = list(map(lambda x: filter_set(x), resps)) + return filtered_resps + + +@register_filter("regex_pos") +class POSFilter(Filter): + """ """ + + def __init__( + self, + regex_pattern: str = r"\['(.*?)'\]", + group_select=0, + fallback=None, + ) -> None: + """ + pass a string `regex` to run `re.compile(r"regex")` on. + `fallback` defines the output returned if no matches for the regex are located. + """ + if fallback is None: + fallback = ["invalid"] + self.regex_pattern = regex_pattern + self.regex = re.compile(regex_pattern) + self.group_select = group_select + self.fallback = fallback + + def apply(self, resps, docs): + def extract_tagged_tokens(text): + # Extract tagged tokens list from text input using regex + tokens = re.findall(r"\('([^']*)', '([^']*)'\)", text) + return [(token, pos) for token, pos in tokens] + + def extract_pos_tags(result): + pos_tags = [] + if isinstance(result, str): + result = extract_tagged_tokens(result) + pos_tags.extend(pos for _, pos in result) + return pos_tags if pos_tags else self.fallback + + def filter_set(inst): + filtered = [] + for resp in inst: + match = extract_pos_tags(resp) + filtered.append(match) + return filtered + + filtered_resps = map(lambda x: filter_set(x), resps) + + return filtered_resps + + +@register_filter("remove_whitespace") +class WhitespaceFilter(Filter): + """Filters out leading whitespace from responses.""" + + def apply(self, resps: list[list[str]], docs: list[dict]) -> list[list[str]]: + def filter_set(inst): + filtered_resp = [] + for resp in inst: + resp = resp.lstrip() + filtered_resp.append(resp) + return filtered_resp + + filtered_resps = [filter_set(resp) for resp in resps] + + return filtered_resps + + +@register_filter("multi_choice_regex") +class MultiChoiceRegexFilter(RegexFilter): + """ + A filter used to extract a model's answer on multiple choice questions with + letter answers. assumes each document has a "choices" field + containing the list of answer choices and that the answer label symbols + are of the form (A), (B), (C), ... or A, B, C. + """ + + def __init__( + self, + regex_pattern: str = r"#### (\-?[0-9\.\,]+)", + group_select=0, + fallback: str = "[invalid]", + ignore_case=False, + ignore_punctuation=False, + regexes_to_ignore=None, + ) -> None: + """ + regex_pattern: The basic regex pattern to use. If fails to match, we will use the customized match procedure + - step 1 : We parse the choices between ([A-Z])s then try to find these choices in the response. + - step 2 : We parse the choice with regex :[\s]*([A-?]), where ? varies by number of choices. + group_select: Selects the (group_select)th match from the findall result. + ignore_case: Ignores the case during step 1 matching + ignore_punctuation: Remove the punctuation during step 1 matching + regexes_to_ignore: Remove these regexes during step 1 matching + """ + super().__init__(regex_pattern, group_select, fallback) + self.ignore_case = ignore_case + self.ignore_punctuation = ignore_punctuation + self.regexes_to_ignore = regexes_to_ignore + + def apply(self, resps: list[list[str]], docs: list[dict]) -> list[list[str]]: + # here, we assume we have a list, in which each element is + # a list of model responses for some particular input/target pair. + # so we process each of these (same input/target response sets) + # independently (and keep them a list.) + + def find_match(regex, resp, convert_dict={}): + match = regex.findall(resp) + if match: + match = match[self.group_select] + if isinstance(match, tuple): + match = [m for m in match if m][0] + match = match.strip() + if match and match in convert_dict: + match = convert_dict[match] + return match + + punct_tbl = dict.fromkeys( + i + for i in range(sys.maxunicode) + if unicodedata.category(chr(i)).startswith("P") + ) + + def filter_ignores(st): + if self.regexes_to_ignore is not None: + for s in self.regexes_to_ignore: + st = re.sub(s, "", st) + + if self.ignore_case: + st = st.lower() + + if self.ignore_punctuation: + # https://stackoverflow.com/a/266162 + st = st.translate(punct_tbl) + return st + + filtered_resps = [] + + for r, doc in zip(resps, docs): + fallback_regexes = [] + choice_to_alpha = {} + next_alpha = "A" + + without_paren_fallback_regexes = [] + without_paren_to_target = {} + + choices = doc["choices"] + for c in choices: + m = filter_ignores(c.strip()) + fallback_regexes.append(f"{re.escape(m)}") + choice_to_alpha[m] = f"({next_alpha})" + + without_paren_fallback_regexes.append(next_alpha) + without_paren_to_target[next_alpha] = f"({next_alpha})" + + next_alpha = chr(ord(next_alpha) + 1) + fallback_regex = re.compile("|".join(fallback_regexes)) + without_paren_fallback_regex = "|".join(without_paren_fallback_regexes) + without_paren_fallback_regex = re.compile( + rf":[\s]*({without_paren_fallback_regex})" + ) + + filtered = [] + for resp in r: + match = find_match(self.regex, resp) + if not match: + match = find_match( + fallback_regex, filter_ignores(resp), choice_to_alpha + ) + if not match: + match = find_match( + without_paren_fallback_regex, resp, without_paren_to_target + ) + if not match: + match = self.fallback + filtered.append(match) + filtered_resps.append(filtered) + + return filtered_resps diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/filters/selection.py b/Prism/LLaDA/LLaDA_Prism/dllm_eval/filters/selection.py new file mode 100644 index 0000000000000000000000000000000000000000..47b9c9bc71f254c91ba92aa8578b8c9f8cb3341f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/dllm_eval/filters/selection.py @@ -0,0 +1,61 @@ +from collections import Counter + +from dllm_eval.api.filter import Filter +from dllm_eval.api.registry import register_filter + + +# TODO: implement "arg_max" filter. either it should take in an arbitrary "scoring"/reward function +# that takes an input and returns a scalar and then should select the max reward, +# or should implement different filters for different ways of handling a reward model's inference. + + +@register_filter("take_first") +class TakeFirstFilter(Filter): + def __init__(self) -> None: + """ + Can define custom behavior here, if an individual instantiation of a Filter class should have state. + """ + + def apply(self, resps, docs): + """ + Assuming each entry of `resps` is a list of model responses, we discard all but the first response. + """ + return map(lambda r: r[0], resps) + + +@register_filter("take_first_k") +class TakeKFilter(Filter): + def __init__(self, **kwargs) -> None: + self.k = kwargs.pop("k") + + super().__init__(**kwargs) + + def apply(self, resps, docs): + # need resp to be subscriptable to check below + resps = list(resps) + # check we have at least k responses per doc, else we can't take the first k + assert len(resps[0]) >= self.k, ( + f"Need at least {self.k} responses per doc to take first {self.k}, but got {len(resps[0])} only! Please increase TaskConfig.repeats ." + ) + return map(lambda r: r[: self.k], resps) + + +@register_filter("majority_vote") +class MajorityVoteFilter(Filter): + def __init__(self) -> None: + """ + Can define custom behavior here, if an individual instantiation of a Filter class should have state. + """ + + def apply(self, resps, docs): + """ + Each entry of `resps` is a list of model responses. + We select the response that occurs most frequently in each entry of `resps`. + """ + + def select_majority(resp): + counts = Counter(resp) + vote = counts.most_common(1)[0][0] + return vote + + return map(lambda r: [select_majority(r)], resps) diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/filters/transformation.py b/Prism/LLaDA/LLaDA_Prism/dllm_eval/filters/transformation.py new file mode 100644 index 0000000000000000000000000000000000000000..48d2a21d7d510991977ebcf6601c2e7437ecb4bb --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/dllm_eval/filters/transformation.py @@ -0,0 +1,122 @@ +import re + +from dllm_eval.api.filter import Filter +from dllm_eval.api.registry import register_filter + + +@register_filter("lowercase") +class LowercaseFilter(Filter): + def __init__(self) -> None: + pass + + def apply(self, resps, docs): + def filter_set(inst): + return [resp.lower() for resp in inst] + + return [filter_set(resp) for resp in resps] + + +@register_filter("uppercase") +class UppercaseFilter(Filter): + def __init__(self) -> None: + pass + + def apply(self, resps, docs): + def filter_set(inst): + return [resp.upper() for resp in inst] + + return [filter_set(resp) for resp in resps] + + +@register_filter("map") +class MapFilter(Filter): + def __init__(self, mapping_dict: dict = None, default_value=None) -> None: + """ + Initializes the MapFilter with a given mapping dictionary and default value. + + Args: + - mapping_dict (dict): A dictionary containing the key-value mappings. + Default is an empty dictionary. + - default_value (Any): The value to be returned when a key is not found in the mapping_dict. + Default is None. + + Example: + mapper = MapFilter({'A': 1, 'B': 2}, default_value=0) + """ + if mapping_dict is None: + mapping_dict = {} + assert isinstance(mapping_dict, dict), ( + "Provided mapping_dict is not a dictionary" + ) + self.mapping_dict = mapping_dict + self.default_value = default_value + + def apply(self, resps, docs): + def filter_set(inst): + return [self.mapping_dict.get(resp, self.default_value) for resp in inst] + + return [filter_set(resp) for resp in resps] + + +@register_filter("format_span") +class SPANFilter(Filter): + def __init__(self) -> None: + pass + + def apply(self, resps, docs): + def format_ner_text(text): + label_dict = { + "person": "PER", + "location": "LOC", + "organization": "ORG", + "counties": "LOC", + "places": "LOC", + "people": "PER", + "persons": "PER", + "company": "ORG", + "country": "LOC", + "continent": "LOC", + "time": "DATE", + "date": "DATE", + "per": "PER", + "loc": "LOC", + "org": "ORG", + } + text = text.lower() + for key, value in label_dict.items(): + text = text.replace(key, value) + + text = "$".join(i for i in text.split("$$")) + return text.rstrip("$$") + + def format_named_entities(text): + """ + Extract named entities from text and format them as 'label: value $$ label: value'. + Handles grouped entities (e.g., LOC: kenya, uganda) and excludes 'none' values. + """ + # Regular expression to match label: entities pattern + pattern = r"\b(PER|LOC|ORG|DATE):\s*([^$]+)" + # Normalize newline characters + text = text.replace("\n", "$").strip() + matches = re.findall(pattern, text) + + formatted_entities = [] + + for label, values in matches: + # Split multiple entities separated by commas and strip whitespace + entities = [value.strip() for value in values.split(",")] + + # Exclude 'none' entities + for entity in entities: + if entity.lower() != "none": + formatted_entities.append(f"{label.lower()}: {entity}") + + # Join entities with the desired separator + return " $ ".join(formatted_entities) + + def filter_set(inst): + return [ + format_named_entities(format_ner_text(resp.lower())) for resp in inst + ] + + return [filter_set(resp) for resp in resps] diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/loggers/__init__.py b/Prism/LLaDA/LLaDA_Prism/dllm_eval/loggers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..02b7a6834c6486fde35ef02d715e90be3fba223a --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/dllm_eval/loggers/__init__.py @@ -0,0 +1,2 @@ +from .evaluation_tracker import EvaluationTracker +from .wandb_logger import WandbLogger diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/loggers/evaluation_tracker.py b/Prism/LLaDA/LLaDA_Prism/dllm_eval/loggers/evaluation_tracker.py new file mode 100644 index 0000000000000000000000000000000000000000..7f88978e73a8fad88d83a9563e85090b8c7e5594 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/dllm_eval/loggers/evaluation_tracker.py @@ -0,0 +1,530 @@ +import json +import logging +import os +import re +import time +from collections import defaultdict +from dataclasses import asdict, dataclass +from datetime import datetime +from pathlib import Path + +from datasets import load_dataset +from datasets.utils.metadata import MetadataConfigs +from huggingface_hub import ( + DatasetCard, + DatasetCardData, + HfApi, + hf_hub_url, +) +from huggingface_hub.utils import build_hf_headers, get_session, hf_raise_for_status + +from dllm_eval.utils import ( + get_file_datetime, + get_file_task_name, + get_results_filenames, + get_sample_results_filenames, + handle_non_serializable, + hash_string, + sanitize_list, + sanitize_model_name, + sanitize_task_name, +) + + +eval_logger = logging.getLogger(__name__) + + +@dataclass(init=False) +class GeneralConfigTracker: + """ + Tracker for the evaluation parameters. + + Attributes: + model_source (str): Source of the model (e.g. Hugging Face, GGUF, etc.) + model_name (str): Name of the model. + model_name_sanitized (str): Sanitized model name for directory creation. + start_time (float): Start time of the experiment. Logged at class init. + end_time (float): Start time of the experiment. Logged when calling [`GeneralConfigTracker.log_end_time`] + total_evaluation_time_seconds (str): Inferred total evaluation time in seconds (from the start and end times). + """ + + model_source: str = None + model_name: str = None + model_name_sanitized: str = None + system_instruction: str = None + system_instruction_sha: str = None + fewshot_as_multiturn: bool = None + chat_template: str = None + chat_template_sha: str = None + start_time: float = None + end_time: float = None + total_evaluation_time_seconds: str = None + + def __init__(self) -> None: + """Starts the evaluation timer.""" + self.start_time = time.perf_counter() + + @staticmethod + def _get_model_name(model_args: str) -> str: + """Extracts the model name from the model arguments.""" + + def extract_model_name(model_args: str, key: str) -> str: + """Extracts the model name from the model arguments using a key.""" + args_after_key = model_args.split(key)[1] + return args_after_key.split(",")[0] + + # order does matter, e.g. peft and delta are provided together with pretrained + prefixes = ["peft=", "delta=", "pretrained=", "model=", "path=", "engine="] + for prefix in prefixes: + if prefix in model_args: + return extract_model_name(model_args, prefix) + return "" + + def log_experiment_args( + self, + model_source: str, + model_args: str, + system_instruction: str, + chat_template: str, + fewshot_as_multiturn: bool, + ) -> None: + """Logs model parameters and job ID.""" + self.model_source = model_source + self.model_name = GeneralConfigTracker._get_model_name(model_args) + self.model_name_sanitized = sanitize_model_name(self.model_name) + self.system_instruction = system_instruction + self.system_instruction_sha = ( + hash_string(system_instruction) if system_instruction else None + ) + self.chat_template = chat_template + self.chat_template_sha = hash_string(chat_template) if chat_template else None + self.fewshot_as_multiturn = fewshot_as_multiturn + + def log_end_time(self) -> None: + """Logs the end time of the evaluation and calculates the total evaluation time.""" + self.end_time = time.perf_counter() + self.total_evaluation_time_seconds = str(self.end_time - self.start_time) + + +class EvaluationTracker: + """ + Keeps track and saves relevant information of the evaluation process. + Compiles the data from trackers and writes it to files, which can be published to the Hugging Face hub if requested. + """ + + def __init__( + self, + output_path: str = None, + hub_results_org: str = "", + hub_repo_name: str = "", + details_repo_name: str = "", + results_repo_name: str = "", + push_results_to_hub: bool = False, + push_samples_to_hub: bool = False, + public_repo: bool = False, + token: str = "", + leaderboard_url: str = "", + point_of_contact: str = "", + gated: bool = False, + ) -> None: + """ + Creates all the necessary loggers for evaluation tracking. + + Args: + output_path (str): Path to save the results. If not provided, the results won't be saved. + hub_results_org (str): The Hugging Face organization to push the results to. If not provided, the results will be pushed to the owner of the Hugging Face token. + hub_repo_name (str): The name of the Hugging Face repository to push the results to. If not provided, the results will be pushed to `lm-eval-results`. + details_repo_name (str): The name of the Hugging Face repository to push the details to. If not provided, the results will be pushed to `lm-eval-results`. + result_repo_name (str): The name of the Hugging Face repository to push the results to. If not provided, the results will not be pushed and will be found in the details_hub_repo. + push_results_to_hub (bool): Whether to push the results to the Hugging Face hub. + push_samples_to_hub (bool): Whether to push the samples to the Hugging Face hub. + public_repo (bool): Whether to push the results to a public or private repository. + token (str): Token to use when pushing to the Hugging Face hub. This token should have write access to `hub_results_org`. + leaderboard_url (str): URL to the leaderboard on the Hugging Face hub on the dataset card. + point_of_contact (str): Contact information on the Hugging Face hub dataset card. + gated (bool): Whether to gate the repository. + """ + self.general_config_tracker = GeneralConfigTracker() + + self.output_path = output_path + self.push_results_to_hub = push_results_to_hub + self.push_samples_to_hub = push_samples_to_hub + self.public_repo = public_repo + self.leaderboard_url = leaderboard_url + self.point_of_contact = point_of_contact + self.api = HfApi(token=token) if token else None + self.gated_repo = gated + + if not self.api and (push_results_to_hub or push_samples_to_hub): + raise ValueError( + "Hugging Face token is not defined, but 'push_results_to_hub' or 'push_samples_to_hub' is set to True. " + "Please provide a valid Hugging Face token by setting the HF_TOKEN environment variable." + ) + + if ( + self.api + and hub_results_org == "" + and (push_results_to_hub or push_samples_to_hub) + ): + hub_results_org = self.api.whoami()["name"] + eval_logger.warning( + f"hub_results_org was not specified. Results will be pushed to '{hub_results_org}'." + ) + + if hub_repo_name == "": + details_repo_name = ( + details_repo_name if details_repo_name != "" else "lm-eval-results" + ) + results_repo_name = ( + results_repo_name if results_repo_name != "" else details_repo_name + ) + else: + details_repo_name = hub_repo_name + results_repo_name = hub_repo_name + eval_logger.warning( + "hub_repo_name was specified. Both details and results will be pushed to the same repository. Using hub_repo_name is no longer recommended, details_repo_name and results_repo_name should be used instead." + ) + + self.details_repo = f"{hub_results_org}/{details_repo_name}" + self.details_repo_private = f"{hub_results_org}/{details_repo_name}-private" + self.results_repo = f"{hub_results_org}/{results_repo_name}" + self.results_repo_private = f"{hub_results_org}/{results_repo_name}-private" + + def save_results_aggregated( + self, + results: dict, + samples: dict, + ) -> None: + """ + Saves the aggregated results and samples to the output path and pushes them to the Hugging Face hub if requested. + + Args: + results (dict): The aggregated results to save. + samples (dict): The samples results to save. + """ + self.general_config_tracker.log_end_time() + + if self.output_path: + try: + eval_logger.info("Saving results aggregated") + + # calculate cumulative hash for each task - only if samples are provided + task_hashes = {} + if samples: + for task_name, task_samples in samples.items(): + sample_hashes = [ + s["doc_hash"] + s["prompt_hash"] + s["target_hash"] + for s in task_samples + ] + task_hashes[task_name] = hash_string("".join(sample_hashes)) + + # update initial results dict + results.update({"task_hashes": task_hashes}) + results.update(asdict(self.general_config_tracker)) + dumped = json.dumps( + results, + indent=2, + default=handle_non_serializable, + ensure_ascii=False, + ) + + path = Path(self.output_path if self.output_path else Path.cwd()) + self.date_id = datetime.now().isoformat().replace(":", "-") + if path.suffix == ".json": + path.parent.mkdir(parents=True, exist_ok=True) + file_results_aggregated = path.with_name( + f"{path.stem}_{self.date_id}.json" + ) + else: + path.mkdir(parents=True, exist_ok=True) + file_results_aggregated = path.joinpath( + f"results_{self.date_id}.json" + ) + + file_results_aggregated.open("w", encoding="utf-8").write(dumped) + + if self.api and self.push_results_to_hub: + repo_id = ( + self.results_repo + if self.public_repo + else self.results_repo_private + ) + self.api.create_repo( + repo_id=repo_id, + repo_type="dataset", + private=not self.public_repo, + exist_ok=True, + ) + self.api.upload_file( + repo_id=repo_id, + path_or_fileobj=str(file_results_aggregated), + path_in_repo=os.path.join( + self.general_config_tracker.model_name, + file_results_aggregated.name, + ), + repo_type="dataset", + commit_message=f"Adding aggregated results for {self.general_config_tracker.model_name}", + ) + eval_logger.info( + "Successfully pushed aggregated results to the Hugging Face Hub. " + f"You can find them at: {repo_id}" + ) + + except Exception as e: + eval_logger.warning("Could not save results aggregated") + eval_logger.info(repr(e)) + else: + eval_logger.info( + "Output path not provided, skipping saving results aggregated" + ) + + def save_results_samples( + self, + task_name: str, + samples: dict, + ) -> None: + """ + Saves the samples results to the output path and pushes them to the Hugging Face hub if requested. + + Args: + task_name (str): The task name to save the samples for. + samples (dict): The samples results to save. + """ + if self.output_path: + try: + eval_logger.info(f"Saving per-sample results for: {task_name}") + + path = Path(self.output_path if self.output_path else Path.cwd()) + if path.suffix == ".json": + path = path.parent + path.mkdir(parents=True, exist_ok=True) + + file_results_samples = path.joinpath( + f"samples_{task_name}_{self.date_id}.jsonl" + ) + + for sample in samples: + # we first need to sanitize arguments and resps + # otherwise we won't be able to load the dataset + # using the datasets library + arguments = {} + for i, arg in enumerate(sample["arguments"]): + arguments[f"gen_args_{i}"] = {} + for j, tmp in enumerate(arg): + arguments[f"gen_args_{i}"][f"arg_{j}"] = tmp + + sample["resps"] = sanitize_list(sample["resps"]) + sample["filtered_resps"] = sanitize_list(sample["filtered_resps"]) + sample["arguments"] = arguments + sample["target"] = str(sample["target"]) + + sample_dump = ( + json.dumps( + sample, + default=handle_non_serializable, + ensure_ascii=False, + ) + + "\n" + ) + + with open(file_results_samples, "a", encoding="utf-8") as f: + f.write(sample_dump) + + if self.api and self.push_samples_to_hub: + repo_id = ( + self.details_repo + if self.public_repo + else self.details_repo_private + ) + self.api.create_repo( + repo_id=repo_id, + repo_type="dataset", + private=not self.public_repo, + exist_ok=True, + ) + try: + if self.gated_repo: + headers = build_hf_headers() + r = get_session().put( + url=f"https://huggingface.co/api/datasets/{repo_id}/settings", + headers=headers, + json={"gated": "auto"}, + ) + hf_raise_for_status(r) + except Exception as e: + eval_logger.warning("Could not gate the repository") + eval_logger.info(repr(e)) + self.api.upload_folder( + repo_id=repo_id, + folder_path=str(path), + path_in_repo=self.general_config_tracker.model_name_sanitized, + repo_type="dataset", + commit_message=f"Adding samples results for {task_name} to {self.general_config_tracker.model_name}", + ) + eval_logger.info( + f"Successfully pushed sample results for task: {task_name} to the Hugging Face Hub. " + f"You can find them at: {repo_id}" + ) + + except Exception as e: + eval_logger.warning("Could not save sample results") + eval_logger.info(repr(e)) + else: + eval_logger.info("Output path not provided, skipping saving sample results") + + def recreate_metadata_card(self) -> None: + """ + Creates a metadata card for the evaluation results dataset and pushes it to the Hugging Face hub. + """ + + eval_logger.info("Recreating metadata card") + repo_id = self.details_repo if self.public_repo else self.details_repo_private + + files_in_repo = self.api.list_repo_files(repo_id=repo_id, repo_type="dataset") + results_files = get_results_filenames(files_in_repo) + sample_files = get_sample_results_filenames(files_in_repo) + + # Build a dictionary to store the latest evaluation datetime for: + # - Each tested model and its aggregated results + # - Each task and sample results, if existing + # i.e. { + # "org__model_name__gsm8k": "2021-09-01T12:00:00", + # "org__model_name__ifeval": "2021-09-01T12:00:00", + # "org__model_name__results": "2021-09-01T12:00:00" + # } + latest_task_results_datetime = defaultdict(lambda: datetime.min.isoformat()) + + for file_path in sample_files: + file_path = Path(file_path) + filename = file_path.name + model_name = file_path.parent + task_name = get_file_task_name(filename) + results_datetime = get_file_datetime(filename) + task_name_sanitized = sanitize_task_name(task_name) + # Results and sample results for the same model and task will have the same datetime + samples_key = f"{model_name}__{task_name_sanitized}" + results_key = f"{model_name}__results" + latest_datetime = max( + latest_task_results_datetime[samples_key], + results_datetime, + ) + latest_task_results_datetime[samples_key] = latest_datetime + latest_task_results_datetime[results_key] = max( + latest_task_results_datetime[results_key], + latest_datetime, + ) + + # Create metadata card + card_metadata = MetadataConfigs() + + # Add the latest aggregated results to the metadata card for easy access + for file_path in results_files: + file_path = Path(file_path) + results_filename = file_path.name + model_name = file_path.parent + eval_date = get_file_datetime(results_filename) + eval_date_sanitized = re.sub(r"[^\w\.]", "_", eval_date) + results_filename = Path("**") / Path(results_filename).name + config_name = f"{model_name}__results" + sanitized_last_eval_date_results = re.sub( + r"[^\w\.]", "_", latest_task_results_datetime[config_name] + ) + + if eval_date_sanitized == sanitized_last_eval_date_results: + # Ensure that all results files are listed in the metadata card + current_results = card_metadata.get(config_name, {"data_files": []}) + current_results["data_files"].append( + {"split": eval_date_sanitized, "path": [str(results_filename)]} + ) + card_metadata[config_name] = current_results + # If the results file is the newest, update the "latest" field in the metadata card + card_metadata[config_name]["data_files"].append( + {"split": "latest", "path": [str(results_filename)]} + ) + + # Add the tasks details configs + for file_path in sample_files: + file_path = Path(file_path) + filename = file_path.name + model_name = file_path.parent + task_name = get_file_task_name(filename) + eval_date = get_file_datetime(filename) + task_name_sanitized = sanitize_task_name(task_name) + eval_date_sanitized = re.sub(r"[^\w\.]", "_", eval_date) + results_filename = Path("**") / Path(filename).name + config_name = f"{model_name}__{task_name_sanitized}" + sanitized_last_eval_date_results = re.sub( + r"[^\w\.]", "_", latest_task_results_datetime[config_name] + ) + if eval_date_sanitized == sanitized_last_eval_date_results: + # Ensure that all sample results files are listed in the metadata card + current_details_for_task = card_metadata.get( + config_name, {"data_files": []} + ) + current_details_for_task["data_files"].append( + {"split": eval_date_sanitized, "path": [str(results_filename)]} + ) + card_metadata[config_name] = current_details_for_task + # If the samples results file is the newest, update the "latest" field in the metadata card + card_metadata[config_name]["data_files"].append( + {"split": "latest", "path": [str(results_filename)]} + ) + + # Get latest results and extract info to update metadata card examples + latest_datetime = max(latest_task_results_datetime.values()) + latest_model_name = max( + latest_task_results_datetime, key=lambda k: latest_task_results_datetime[k] + ) + last_results_file = [ + f for f in results_files if latest_datetime.replace(":", "-") in f + ][0] + last_results_file_path = hf_hub_url( + repo_id=repo_id, filename=last_results_file, repo_type="dataset" + ) + latest_results_file = load_dataset( + "json", data_files=last_results_file_path, split="train" + ) + results_dict = latest_results_file["results"][0] + new_dictionary = {"all": results_dict} + new_dictionary.update(results_dict) + results_string = json.dumps(new_dictionary, indent=4) + + dataset_summary = ( + "Dataset automatically created during the evaluation run of model " + ) + if self.general_config_tracker.model_source == "hf": + dataset_summary += f"[{self.general_config_tracker.model_name}](https://huggingface.co/{self.general_config_tracker.model_name})\n" + else: + dataset_summary += f"{self.general_config_tracker.model_name}\n" + dataset_summary += ( + f"The dataset is composed of {len(card_metadata) - 1} configuration(s), each one corresponding to one of the evaluated task.\n\n" + f"The dataset has been created from {len(results_files)} run(s). Each run can be found as a specific split in each " + 'configuration, the split being named using the timestamp of the run.The "train" split is always pointing to the latest results.\n\n' + 'An additional configuration "results" store all the aggregated results of the run.\n\n' + "To load the details from a run, you can for instance do the following:\n" + ) + if self.general_config_tracker.model_source == "hf": + dataset_summary += ( + "```python\nfrom datasets import load_dataset\n" + f'data = load_dataset(\n\t"{repo_id}",\n\tname="{latest_model_name}",\n\tsplit="latest"\n)\n```\n\n' + ) + dataset_summary += ( + "## Latest results\n\n" + f"These are the [latest results from run {latest_datetime}]({last_results_file_path.replace('/resolve/', '/blob/')}) " + "(note that there might be results for other tasks in the repos if successive evals didn't cover the same tasks. " + 'You find each in the results and the "latest" split for each eval):\n\n' + f"```python\n{results_string}\n```" + ) + card_data = DatasetCardData( + dataset_summary=dataset_summary, + repo_url=f"https://huggingface.co/{self.general_config_tracker.model_name}", + pretty_name=f"Evaluation run of {self.general_config_tracker.model_name}", + leaderboard_url=self.leaderboard_url, + point_of_contact=self.point_of_contact, + ) + card_metadata.to_dataset_card_data(card_data) + card = DatasetCard.from_template( + card_data, + pretty_name=card_data.pretty_name, + ) + card.push_to_hub(repo_id, repo_type="dataset") diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/loggers/utils.py b/Prism/LLaDA/LLaDA_Prism/dllm_eval/loggers/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..ba795edb72d7b665a2c0fe6d4f3e3a5ed91b6940 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/dllm_eval/loggers/utils.py @@ -0,0 +1,149 @@ +import logging +import os +import re +import subprocess +from importlib.metadata import version +from pathlib import Path +from typing import Any, Dict, Optional, Tuple, Union + +import numpy as np +from torch.utils.collect_env import get_pretty_env_info +from transformers import __version__ as trans_version + + +logger = logging.getLogger(__name__) + + +def remove_none_pattern(input_string: str) -> Tuple[str, bool]: + """Remove the ',none' substring from the input_string if it exists at the end. + + Args: + input_string (str): The input string from which to remove the ',none' substring. + + Returns: + Tuple[str, bool]: A tuple containing the modified input_string with the ',none' substring removed + and a boolean indicating whether the modification was made (True) or not (False). + """ + # Define the pattern to match ',none' at the end of the string + pattern = re.compile(r",none$") + + # Use sub() to replace ',none' with an empty string + result = re.sub(pattern, "", input_string) + + # check if the input_string changed + removed = result != input_string + + return result, removed + + +def _handle_non_serializable(o: Any) -> Union[int, str, list]: + """Handle non-serializable objects by converting them to serializable types. + + Args: + o (Any): The object to be handled. + + Returns: + Union[int, str, list]: The converted object. If the object is of type np.int64 or np.int32, + it will be converted to int. If the object is of type set, it will be converted + to a list. Otherwise, it will be converted to str. + """ + if isinstance(o, np.int64) or isinstance(o, np.int32): + return int(o) + elif isinstance(o, set): + return list(o) + else: + return str(o) + + +def get_commit_from_path(repo_path: Union[Path, str]) -> Optional[str]: + try: + git_folder = Path(repo_path, ".git") + if git_folder.is_file(): + git_folder = Path( + git_folder.parent, + git_folder.read_text(encoding="utf-8").split("\n")[0].split(" ")[-1], + ) + if Path(git_folder, "HEAD").exists(): + head_name = ( + Path(git_folder, "HEAD") + .read_text(encoding="utf-8") + .split("\n")[0] + .split(" ")[-1] + ) + head_ref = Path(git_folder, head_name) + git_hash = head_ref.read_text(encoding="utf-8").replace("\n", "") + else: + git_hash = None + except Exception as err: + logger.debug( + f"Failed to retrieve a Git commit hash from path: {str(repo_path)}. Error: {err}" + ) + return None + return git_hash + + +def get_git_commit_hash(): + """ + Gets the git commit hash of your current repo (if it exists). + Source: https://github.com/EleutherAI/gpt-neox/blob/b608043be541602170bfcfb8ec9bf85e8a0799e0/megatron/neox_arguments/neox_args.py#L42 + """ + try: + git_hash = subprocess.check_output(["git", "describe", "--always"]).strip() + git_hash = git_hash.decode() + except (subprocess.CalledProcessError, FileNotFoundError): + # FileNotFoundError occurs when git not installed on system + git_hash = get_commit_from_path(os.getcwd()) # git hash of repo if exists + return git_hash + + +def add_env_info(storage: Dict[str, Any]): + try: + pretty_env_info = get_pretty_env_info() + except Exception as err: + pretty_env_info = str(err) + try: + dllm_eval_version = version("dllm_eval") + except Exception as err: + dllm_eval_version = str(err) + transformers_version = trans_version + upper_dir_commit = get_commit_from_path( + Path(os.getcwd(), "..") + ) # git hash of upper repo if exists + added_info = { + "pretty_env_info": pretty_env_info, + "transformers_version": transformers_version, + "dllm_eval_version": dllm_eval_version, + "upper_git_hash": upper_dir_commit, # in case this repo is submodule + } + storage.update(added_info) + + +def add_tokenizer_info(storage: Dict[str, Any], lm): + if getattr(lm, "tokenizer", False): + try: + tokenizer_info = { + "tokenizer_pad_token": [ + lm.tokenizer.pad_token, + str(lm.tokenizer.pad_token_id), + ], + "tokenizer_eos_token": [ + lm.tokenizer.eos_token, + str(lm.tokenizer.eos_token_id), + ], + "tokenizer_bos_token": [ + lm.tokenizer.bos_token, + str(lm.tokenizer.bos_token_id), + ], + "eot_token_id": getattr(lm, "eot_token_id", None), + "max_length": getattr(lm, "max_length", None), + } + storage.update(tokenizer_info) + except Exception as err: + logger.debug( + f"Logging detailed tokenizer info failed with {err}, skipping..." + ) + # seems gguf and textsynth do not have tokenizer + else: + logger.debug( + "LM does not have a 'tokenizer' attribute, not logging tokenizer metadata to results." + ) diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/loggers/wandb_logger.py b/Prism/LLaDA/LLaDA_Prism/dllm_eval/loggers/wandb_logger.py new file mode 100644 index 0000000000000000000000000000000000000000..9c0859b3c8e90437f21b6f06143b14941a7a96d2 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/dllm_eval/loggers/wandb_logger.py @@ -0,0 +1,358 @@ +import copy +import json +import logging +from typing import Any, Dict, List, Literal, Tuple + +import numpy as np +import pandas as pd +from packaging.version import Version + +from dllm_eval.loggers.utils import _handle_non_serializable, remove_none_pattern + + +logger = logging.getLogger(__name__) + + +def get_wandb_printer() -> Literal["Printer"]: + """Returns a wandb printer instance for pretty stdout.""" + from wandb.sdk.lib.printer import new_printer + + printer = new_printer() + return printer + + +class WandbLogger: + def __init__(self, init_args=None, config_args=None) -> None: + """Attaches to wandb logger if already initialized. Otherwise, passes init_args to wandb.init() and config_args to wandb.config.update() + + Args: + init_args Optional[Dict]: Arguments for init configuration. + config_args Optional[Dict]: Arguments for config + + Parse and log the results returned from evaluator.simple_evaluate() with: + wandb_logger.post_init(results) + wandb_logger.log_eval_result() + wandb_logger.log_eval_samples(results["samples"]) + """ + try: + import wandb + + assert Version(wandb.__version__) >= Version("0.13.6") + if Version(wandb.__version__) < Version("0.13.6"): + wandb.require("report-editing:v0") + except Exception as e: + logger.warning( + "To use the wandb reporting functionality please install wandb>=0.13.6.\n" + "To install the latest version of wandb run `pip install wandb --upgrade`\n" + f"{e}" + ) + + self.wandb_args: Dict[str, Any] = init_args or {} + self.wandb_config_args: Dict[str, Any] = config_args or {} + + # pop the step key from the args to save for all logging calls + self.step = self.wandb_args.pop("step", None) + + # initialize a W&B run + if wandb.run is None: + self.run = wandb.init(**self.wandb_args) + if self.wandb_config_args: + self.run.config.update(self.wandb_config_args) + else: + self.run = wandb.run + + self.printer = get_wandb_printer() + + def post_init(self, results: Dict[str, Any]) -> None: + self.results: Dict[str, Any] = copy.deepcopy(results) + self.task_names: List[str] = list(results.get("results", {}).keys()) + self.group_names: List[str] = list(results.get("groups", {}).keys()) + + def _get_config(self) -> Dict[str, Any]: + """Get configuration parameters.""" + self.task_configs = self.results.get("configs", {}) + cli_configs = self.results.get("config", {}) + configs = { + "task_configs": self.task_configs, + "cli_configs": cli_configs, + } + + return configs + + def _sanitize_results_dict(self) -> Tuple[Dict[str, str], Dict[str, Any]]: + """Sanitize the results dictionary.""" + _results = copy.deepcopy(self.results.get("results", dict())) + + # Remove None from the metric string name + tmp_results = copy.deepcopy(_results) + for task_name in self.task_names: + task_result = tmp_results.get(task_name, dict()) + for metric_name, metric_value in task_result.items(): + _metric_name, removed = remove_none_pattern(metric_name) + if removed: + _results[task_name][_metric_name] = metric_value + _results[task_name].pop(metric_name) + + # remove string valued keys from the results dict + wandb_summary = {} + for task in self.task_names: + task_result = _results.get(task, dict()) + for metric_name, metric_value in task_result.items(): + if isinstance(metric_value, str): + wandb_summary[f"{task}/{metric_name}"] = metric_value + + for summary_metric, summary_value in wandb_summary.items(): + _task, _summary_metric = summary_metric.split("/") + _results[_task].pop(_summary_metric) + + tmp_results = copy.deepcopy(_results) + for task_name, task_results in tmp_results.items(): + for metric_name, metric_value in task_results.items(): + _results[f"{task_name}/{metric_name}"] = metric_value + _results[task_name].pop(metric_name) + for task in self.task_names: + _results.pop(task) + + return wandb_summary, _results + + def _log_results_as_table(self) -> None: + """Generate and log evaluation results as a table to W&B.""" + columns = [ + "Version", + "Filter", + "num_fewshot", + "Metric", + "Value", + "Stderr", + ] + + def make_table(columns: List[str], key: str = "results"): + import wandb + + table = wandb.Table(columns=columns) + results = copy.deepcopy(self.results) + + for k, dic in results.get(key).items(): + if k in self.group_names and not key == "groups": + continue + version = results.get("versions").get(k) + if version == "N/A": + version = None + n = results.get("n-shot").get(k) + + for (mf), v in dic.items(): + m, _, f = mf.partition(",") + if m.endswith("_stderr"): + continue + if m == "alias": + continue + + if m + "_stderr" + "," + f in dic: + se = dic[m + "_stderr" + "," + f] + if se != "N/A": + se = "%.4f" % se + table.add_data(*[k, version, f, n, m, str(v), str(se)]) + else: + table.add_data(*[k, version, f, n, m, str(v), ""]) + + return table + + # log the complete eval result to W&B Table + table = make_table(["Tasks"] + columns, "results") + self.run.log({"evaluation/eval_results": table}, step=self.step) + + if "groups" in self.results.keys(): + table = make_table(["Groups"] + columns, "groups") + self.run.log({"evaluation/group_eval_results": table}, step=self.step) + + def _log_results_as_artifact(self) -> None: + """Log results as JSON artifact to W&B.""" + import wandb + + dumped = json.dumps( + self.results, indent=2, default=_handle_non_serializable, ensure_ascii=False + ) + artifact = wandb.Artifact("results", type="eval_results") + with artifact.new_file("results.json", mode="w", encoding="utf-8") as f: + f.write(dumped) + self.run.log_artifact(artifact) + + def log_eval_result(self) -> None: + """Log evaluation results to W&B.""" + # Log configs to wandb + configs = self._get_config() + self.run.config.update(configs, allow_val_change=self.step is not None) + + wandb_summary, self.wandb_results = self._sanitize_results_dict() + # update wandb.run.summary with items that were removed + self.run.summary.update(wandb_summary) + # Log the evaluation metrics to wandb + self.run.log(self.wandb_results, step=self.step) + # Log the evaluation metrics as W&B Table + self._log_results_as_table() + # Log the results dict as json to W&B Artifacts + self._log_results_as_artifact() + + def _generate_dataset( + self, data: List[Dict[str, Any]], config: Dict[str, Any] + ) -> pd.DataFrame: + """Generate a dataset from evaluation data. + + Args: + data (List[Dict[str, Any]]): The data to generate a dataset for. + config (Dict[str, Any]): The configuration of the task. + + Returns: + pd.DataFrame: A dataframe that is ready to be uploaded to W&B. + """ + ids = [x["doc_id"] for x in data] + labels = [x["target"] for x in data] + instance = [""] * len(ids) + resps = [""] * len(ids) + filtered_resps = [""] * len(ids) + model_outputs = {} + + metrics_list = config["metric_list"] + metrics = {} + for metric in metrics_list: + metric = metric.get("metric") + if metric in ["word_perplexity", "byte_perplexity", "bits_per_byte"]: + metrics[f"{metric}_loglikelihood"] = [x[metric][0] for x in data] + if metric in ["byte_perplexity", "bits_per_byte"]: + metrics[f"{metric}_bytes"] = [x[metric][1] for x in data] + else: + metrics[f"{metric}_words"] = [x[metric][1] for x in data] + else: + metrics[metric] = [x[metric] for x in data] + + if config["output_type"] == "loglikelihood": + instance = [x["arguments"][0][0] for x in data] + labels = [x["arguments"][0][1] for x in data] + resps = [ + f"log probability of continuation is {x['resps'][0][0][0]} " + + "\n\n" + + "continuation will {} generated with greedy sampling".format( + "not be" if not x["resps"][0][0][1] else "be" + ) + for x in data + ] + filtered_resps = [ + f"log probability of continuation is {x['filtered_resps'][0][0]} " + + "\n\n" + + "continuation will {} generated with greedy sampling".format( + "not be" if not x["filtered_resps"][0][1] else "be" + ) + for x in data + ] + elif config["output_type"] == "multiple_choice": + instance = [x["arguments"][0][0] for x in data] + choices = [ + "\n".join([f"{idx}. {y[1]}" for idx, y in enumerate(x["arguments"])]) + for x in data + ] + resps = [np.argmax([n[0][0] for n in x["resps"]]) for x in data] + filtered_resps = [ + np.argmax([n[0] for n in x["filtered_resps"]]) for x in data + ] + elif config["output_type"] == "loglikelihood_rolling": + instance = [x["arguments"][0][0] for x in data] + resps = [x["resps"][0][0] for x in data] + filtered_resps = [x["filtered_resps"][0] for x in data] + elif config["output_type"] == "generate_until": + instance = [x["arguments"][0][0] for x in data] + resps = [x["resps"][0][0] for x in data] + filtered_resps = [x["filtered_resps"][0] for x in data] + + model_outputs["raw_predictions"] = resps + model_outputs["filtered_predictions"] = filtered_resps + + df_data = { + "id": ids, + "data": instance, + } + if config["output_type"] == "multiple_choice": + df_data["choices"] = choices + + tmp_data = { + "input_len": [len(x) for x in instance], + "labels": labels, + "output_type": config["output_type"], + } + df_data.update(tmp_data) + df_data.update(model_outputs) + df_data.update(metrics) + + return pd.DataFrame(df_data) + + def _log_samples_as_artifact( + self, data: List[Dict[str, Any]], task_name: str + ) -> None: + import wandb + + # log the samples as an artifact + dumped = json.dumps( + data, + indent=2, + default=_handle_non_serializable, + ensure_ascii=False, + ) + artifact = wandb.Artifact(f"{task_name}", type="samples_by_task") + with artifact.new_file( + f"{task_name}_eval_samples.json", mode="w", encoding="utf-8" + ) as f: + f.write(dumped) + self.run.log_artifact(artifact) + # artifact.wait() + + def log_eval_samples(self, samples: Dict[str, List[Dict[str, Any]]]) -> None: + """Log evaluation samples to W&B. + + Args: + samples (Dict[str, List[Dict[str, Any]]]): Evaluation samples for each task. + """ + task_names: List[str] = [ + x for x in self.task_names if x not in self.group_names + ] + + ungrouped_tasks = [] + tasks_by_groups = {} + + for task_name in task_names: + group_names = self.task_configs[task_name].get("group", None) + if group_names: + if isinstance(group_names, str): + group_names = [group_names] + + for group_name in group_names: + if not tasks_by_groups.get(group_name): + tasks_by_groups[group_name] = [task_name] + else: + tasks_by_groups[group_name].append(task_name) + else: + ungrouped_tasks.append(task_name) + + for task_name in ungrouped_tasks: + eval_preds = samples[task_name] + + # log the samples as a W&B Table + df = self._generate_dataset(eval_preds, self.task_configs.get(task_name)) + self.run.log({f"{task_name}_eval_results": df}, step=self.step) + + # log the samples as a json file as W&B Artifact + self._log_samples_as_artifact(eval_preds, task_name) + + for group, grouped_tasks in tasks_by_groups.items(): + grouped_df = pd.DataFrame() + for task_name in grouped_tasks: + eval_preds = samples[task_name] + df = self._generate_dataset( + eval_preds, self.task_configs.get(task_name) + ) + df["group"] = group + df["task"] = task_name + grouped_df = pd.concat([grouped_df, df], ignore_index=True) + + # log the samples as a json file as W&B Artifact + self._log_samples_as_artifact(eval_preds, task_name) + + self.run.log({f"{group}_eval_results": grouped_df}, step=self.step) diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/models/LLaDA.py b/Prism/LLaDA/LLaDA_Prism/dllm_eval/models/LLaDA.py new file mode 100644 index 0000000000000000000000000000000000000000..bfe0b51b21f2ab3a584af362635e38036d415a36 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/dllm_eval/models/LLaDA.py @@ -0,0 +1,786 @@ +import logging +import os +from datetime import timedelta +from typing import Dict, List, Literal, Optional, Tuple, Union, TypeVar +import torch +import torch.nn.functional as F +import numpy as np +import transformers +import json +from accelerate import ( + Accelerator, + InitProcessGroupKwargs, +) +from datasets import Dataset +from accelerate.utils import get_max_memory +from packaging import version +from tqdm import tqdm +import torch.distributed as dist +from transformers.models.auto.modeling_auto import ( + MODEL_FOR_CAUSAL_LM_MAPPING_NAMES, + MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES, +) +from dllm_eval.api.instance import Instance +from dllm_eval.api.model import LM, TemplateLM +from dllm_eval.api.registry import register_model +from dllm_eval.models.utils import get_dtype, configure_pad_token + +try: + from .hts_sampler import HTSSampler +except ImportError: + HTSSampler = None + +eval_logger = logging.getLogger(__name__) +T = TypeVar("T", bound="LM") + + +def add_gumbel_noise(logits, temperature): + """Add Gumbel noise for sampling""" + if temperature == 0.0: + return logits + logits = logits.to(torch.float32) + noise = torch.rand_like(logits, dtype=torch.float32) + gumbel_noise = (-torch.log(noise)) ** temperature + return logits.exp() / gumbel_noise + + +def get_num_transfer_tokens(mask_index, steps): + """Calculate number of tokens to transfer at each step""" + mask_num = mask_index.sum(dim=1, keepdim=True) + base = mask_num // steps + remainder = mask_num % steps + num_transfer_tokens = base.expand(-1, steps).clone() + if remainder.sum() > 0: + indices = torch.arange(steps, device=mask_index.device) + mask = indices.unsqueeze(0) < remainder + num_transfer_tokens[mask] += 1 + return num_transfer_tokens.to(torch.int64) + + +@torch.no_grad() +def generate_llada_v1(model, prompt, attention_mask=None, steps=128, gen_length=128, + block_length=128, temperature=0., cfg_scale=0., + remasking='low_confidence', mask_id=126336, + logits_eos_inf=False, confidence_eos_eot_inf=False): + """ + LLaDA v1 generation function + This is the original generate function from LLaDA v1 + """ + x = torch.full((prompt.shape[0], prompt.shape[1] + gen_length), mask_id, + dtype=torch.long).to(model.device) + x[:, :prompt.shape[1]] = prompt.clone() + + if attention_mask is not None: + attention_mask = torch.cat([ + attention_mask, + torch.ones((prompt.shape[0], gen_length), dtype=attention_mask.dtype, + device=model.device) + ], dim=-1) + + prompt_index = (x != mask_id) + + assert gen_length % block_length == 0 + num_blocks = gen_length // block_length + + assert steps % num_blocks == 0 + steps_per_block = steps // num_blocks + + for num_block in range(num_blocks): + block_mask_index = (x[:, prompt.shape[1] + num_block * block_length: + prompt.shape[1] + (num_block + 1) * block_length] == mask_id) + num_transfer_tokens = get_num_transfer_tokens(block_mask_index, steps_per_block) + + for i in range(steps_per_block): + mask_index = (x == mask_id) + + if cfg_scale > 0.: + un_x = x.clone() + un_x[prompt_index] = mask_id + x_ = torch.cat([x, un_x], dim=0) + if attention_mask is not None: + attention_mask_ = torch.cat([attention_mask, attention_mask], dim=0) + logits = model(x_, attention_mask=attention_mask_).logits + logits, un_logits = torch.chunk(logits, 2, dim=0) + logits = un_logits + (cfg_scale + 1) * (logits - un_logits) + else: + logits = model(x, attention_mask=attention_mask).logits + + if logits_eos_inf: + logits[:, :, 126081] = -torch.inf + + logits_with_noise = add_gumbel_noise(logits, temperature=temperature) + x0 = torch.argmax(logits_with_noise, dim=-1) + + if confidence_eos_eot_inf: + logits_with_noise[:, :, 126081] = logits[:, :, 126348] = -torch.inf + + if remasking == 'low_confidence': + p = F.softmax(logits, dim=-1) + x0_p = torch.squeeze( + torch.gather(p, dim=-1, index=torch.unsqueeze(x0, -1)), -1) + elif remasking == 'random': + x0_p = torch.rand((x0.shape[0], x0.shape[1]), device=x0.device) + else: + raise NotImplementedError(remasking) + + x0_p[:, prompt.shape[1] + (num_block + 1) * block_length:] = -np.inf + + x0 = torch.where(mask_index, x0, x) + confidence = torch.where(mask_index, x0_p, -np.inf) + + transfer_index = torch.zeros_like(x0, dtype=torch.bool, device=x0.device) + for j in range(confidence.shape[0]): + _, select_index = torch.topk(confidence[j], k=num_transfer_tokens[j, i]) + transfer_index[j, select_index] = True + + x[transfer_index] = x0[transfer_index] + + return x + + +@register_model("LLaDA") +class LLaDA(TemplateLM): + AUTO_MODEL_CLASS = transformers.AutoModel + _DEFAULT_MAX_LENGTH = 20480 + + def __init__( + self, + pretrained: Union[str, transformers.PreTrainedModel], + backend: Literal["default", "causal", "seq2seq"] = "causal", + revision: Optional[str] = "main", + subfolder: Optional[str] = None, + tokenizer: Optional[ + Union[ + str, + transformers.PreTrainedTokenizer, + transformers.PreTrainedTokenizerFast, + ] + ] = None, + truncation: Optional[bool] = False, + logits_cache: bool = True, + max_length: Optional[int] = None, + device: Optional[str] = "cuda", + dtype: Optional[Union[str, torch.dtype]] = "auto", + batch_size: Optional[Union[int]] = 1, + max_batch_size: Optional[int] = 64, + trust_remote_code: Optional[bool] = True, + use_fast_tokenizer: Optional[bool] = True, + add_bos_token: Optional[bool] = False, + escape_until: Optional[bool] = False, + prefix_token_id: Optional[int] = None, + parallelize: Optional[bool] = False, + max_memory_per_gpu: Optional[Union[int, str]] = None, + max_cpu_memory: Optional[Union[int, str]] = None, + offload_folder: Optional[Union[str, os.PathLike]] = "./offload", + peft: Optional[str] = None, + delta: Optional[str] = None, + autogptq: Optional[Union[bool, str]] = False, + gptqmodel: Optional[bool] = False, + gguf_file: Optional[str] = None, + mc_num: int = 1024, + remasking: str = "low_confidence", + mask_id: int = 126336, # LLaDA v1 default mask_id + is_check_greedy: bool = True, + assistant_prefix: Optional[str] = None, + **kwargs, + ) -> None: + super().__init__() + self.mc_num = mc_num + self.mask_id = mask_id + self.remasking = remasking + self.pretrained = pretrained + self.is_check_greedy = is_check_greedy + self.assistant_prefix = assistant_prefix + self.add_bos_token = add_bos_token + self.escape_until = escape_until + + if not isinstance(pretrained, str): + eval_logger.warning( + "`pretrained` model kwarg is not of type `str`. Many other model arguments may be ignored." + ) + assert not parallelize, ( + "`parallelize=True` is not compatible with passing pre-initialized model to `pretrained`" + ) + self._model = pretrained + self._device = self._model.device + self._config = self._model.config + gpus = 0 + else: + assert isinstance(device, str) + assert isinstance(pretrained, str) + assert isinstance(batch_size, (int, str)) + gpus = torch.cuda.device_count() + accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52)) + accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs]) + if accelerator.num_processes > 1: + self.accelerator = accelerator + if "npu" in accelerator.device.type: + gpus = torch.npu.device_count() + if not (parallelize or accelerator.num_processes > 1): + device_list = set( + ["cuda", "cpu"] + + [f"cuda:{i}" for i in range(gpus)] + + ["mps", "mps:0"] + + [f"npu:{i}" for i in range(gpus)] + ) + if device and device in device_list: + self._device = torch.device(device) + eval_logger.info(f"Using device '{device}'") + if device in ("mps", "mps:0") and version.parse( + torch.__version__ + ) < version.parse("2.1"): + raise RuntimeError( + f"mps requires torch >= 2.1. You have {torch.__version__}" + ) + else: + eval_logger.info("Device not specified") + eval_logger.info(f"Cuda Available? {torch.cuda.is_available()}") + self._device = ( + torch.device("cuda") + if torch.cuda.is_available() + else torch.device("cpu") + ) + else: + if device != "cuda": + eval_logger.info( + f"Using `accelerate launch` or `parallelize=True`, device '{device}' will be overridden when placing model." + ) + self._device = ( + self.accelerator.device + if hasattr(self, "accelerator") + else torch.device(device) + ) + revision = str(revision) + revision = revision + ("/" + subfolder if subfolder is not None else "") + self._get_config( + pretrained, + revision=revision, + trust_remote_code=trust_remote_code, + gguf_file=gguf_file, + ) + + self._get_backend( + config=self.config, backend=backend, trust_remote_code=trust_remote_code + ) + self._create_tokenizer( + pretrained, + tokenizer, + revision=revision, + trust_remote_code=trust_remote_code, + use_fast_tokenizer=use_fast_tokenizer, + gguf_file=gguf_file, + add_bos_token=add_bos_token, + ) + + if isinstance(pretrained, str): + self._create_model( + pretrained=pretrained, + revision=revision, + dtype=dtype, + trust_remote_code=trust_remote_code, + parallelize=parallelize, + gpus=gpus, + max_memory_per_gpu=max_memory_per_gpu, + max_cpu_memory=max_cpu_memory, + offload_folder=offload_folder, + peft=peft, + delta=delta, + autogptq=autogptq, + gptqmodel=gptqmodel, + gguf_file=gguf_file, + **kwargs, + ) + + if isinstance(self.model, torch.nn.Module): + self.model.eval() + self.model.tie_weights() + + self.truncation = truncation + self.logits_cache = logits_cache + self.vocab_size = self.tokenizer.vocab_size + self.tokenizer = configure_pad_token(self.tokenizer, model_config=self.config) + self.add_bos_token = add_bos_token + + if "gemma" in getattr(self.config, "model_type", ""): + self.add_bos_token = True + eval_logger.info( + f"Model type is '{self.config.model_type}', part of the Gemma family--a BOS token will be used." + ) + + self._max_length = max_length + self.pretrained = pretrained + self.delta = delta + self.peft = peft + self.revision = revision + self.batch_schedule = 1 + self.batch_sizes = {} + self.max_batch_size = max_batch_size + + if str(batch_size).startswith("auto"): + batch_size = batch_size.split(":") + self.batch_size_per_gpu = batch_size[0] + self.batch_schedule = float(batch_size[1]) if len(batch_size) > 1 else 1 + else: + self.batch_size_per_gpu = int(batch_size) + + if isinstance(pretrained, str): + if gpus >= 1 or str(self.device) == "mps": + if not (parallelize or autogptq or hasattr(self, "accelerator")): + try: + self.model.to(self.device) + except ValueError: + eval_logger.debug( + "Failed to place model onto specified device. This may be because the model is quantized via `bitsandbytes` or `device_map` is provided." + ) + if gpus > 1: + if hasattr(self, "accelerator") and self.accelerator.num_processes > 1: + if parallelize: + eval_logger.warning( + "You are both using a HF Accelerate `device_map` and launching via `accelerate launch`." + ) + elif gpus > self.accelerator.num_processes: + eval_logger.warning( + "WARNING: The number of total system GPUs does not match the number of spawned processes." + ) + self._device = torch.device(f"{self.accelerator.device}") + self._rank = self.accelerator.local_process_index + self._world_size = self.accelerator.num_processes + else: + self._rank = 0 + self._world_size = 1 + else: + self._rank = 0 + self._world_size = 1 + else: + eval_logger.warning( + "Passed an already-initialized model through `pretrained`, assuming single-process call." + ) + self._rank = 0 + self._world_size = 1 + + self.custom_prefix_token_id = prefix_token_id + if prefix_token_id is not None: + eval_logger.info( + f"Loglikelihood prefix token id used in evaluation: {self.prefix_token_id}" + ) + self.is_first_inference = True + + if HTSSampler is not None: + self.hts_sampler = HTSSampler(self.model, self.tokenizer, device=self.device) + eval_logger.info("HTSSampler initialized successfully.") + + # Copy all the property and helper methods from LLaDA2 + @property + def rank(self): + if hasattr(self, "_rank"): + return self._rank + if hasattr(self, "accelerator"): + return self.accelerator.local_process_index + return int(os.environ.get("LOCAL_RANK", 0)) + + @property + def world_size(self): + if hasattr(self, "_world_size"): + return self._world_size + if hasattr(self, "accelerator"): + return self.accelerator.num_processes + return int(os.environ.get("WORLD_SIZE", 1)) + + def _get_accelerate_args( + self, + parallelize: Optional[bool] = None, + device_map: Optional[str] = "auto", + max_memory_per_gpu: Optional[Union[int, str]] = None, + max_cpu_memory: Optional[Union[int, str]] = None, + offload_folder: Optional[str] = "./offload", + gpus: Optional[int] = None, + ) -> dict: + """Get accelerate arguments - same as LLaDA2""" + num_local_processes = int(os.environ.get("LOCAL_WORLD_SIZE", 1)) + if parallelize is None and gpus is not None and gpus > 1: + parallelize = True + args = {} + if parallelize: + max_memory_all_gpus = get_max_memory() + if "cpu" in max_memory_all_gpus: + del max_memory_all_gpus["cpu"] + max_memory_per_gpu_map = { + device_idx: max_memory_per_gpu for device_idx in range(len(max_memory_all_gpus)) + } if max_memory_per_gpu is not None else {k: v for k, v in max_memory_all_gpus.items()} + if hasattr(self, "accelerator"): + max_memory_per_gpu_map = { + k: v for k, v in max_memory_all_gpus.items() + if k % num_local_processes == self.accelerator.process_index % num_local_processes + } + args["max_memory"] = max_memory_per_gpu_map + args["device_map"] = "auto" + args["offload_folder"] = offload_folder + if max_cpu_memory is not None: + args["max_memory"]["cpu"] = max_cpu_memory + else: + args["device_map"] = {"": str(self.device)} + return args + + @property + def config(self): + return self._config + + @property + def model(self): + if hasattr(self, "accelerator"): + return self.accelerator.unwrap_model(self._model) + else: + return self._model + + @property + def eot_token_id(self): + return self.tokenizer.eos_token_id + + @property + def prefix_token_id(self): + if self.custom_prefix_token_id is not None: + return self.custom_prefix_token_id + if self.tokenizer.bos_token_id is not None: + return self.tokenizer.bos_token_id + return self.tokenizer.eos_token_id + + @property + def max_length(self): + if self._max_length: + return self._max_length + seqlen_config_attrs = ("n_positions", "max_position_embeddings", "n_ctx") + for attr in seqlen_config_attrs: + if hasattr(self.model.config, attr): + return getattr(self.model.config, attr) + if hasattr(self.tokenizer, "model_max_length"): + if self.tokenizer.model_max_length > 1e10: + return self._DEFAULT_MAX_LENGTH + return self.tokenizer.model_max_length + return self._DEFAULT_MAX_LENGTH + + @property + def max_gen_toks(self) -> int: + return 256 + + @property + def batch_size(self): + return self.batch_size_per_gpu + + @property + def device(self): + return self._device + + @property + def tokenizer_name(self) -> str: + return self.tokenizer.name_or_path.replace("/", "__") + + def _get_backend(self, config, backend, trust_remote_code): + """Get backend type - same as LLaDA2""" + assert backend in ["default", "causal", "seq2seq"] + if backend != "default": + self.backend = backend + eval_logger.info(f"Overrode HF model backend type, and using type '{self.backend}'") + else: + if getattr(config, "model_type") in MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES: + self.backend = "seq2seq" + elif getattr(self.config, "model_type") in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES: + self.backend = "causal" + else: + eval_logger.warning("HF model type is neither CausalLM nor Seq2SeqLM. Assuming CausalLM.") + self.backend = "causal" + + def _get_config(self, pretrained, revision, trust_remote_code, gguf_file): + """Get model config - same as LLaDA2""" + self._config = transformers.AutoConfig.from_pretrained( + pretrained, revision=revision, trust_remote_code=trust_remote_code + ) + + def _create_model(self, pretrained, revision, dtype, trust_remote_code, parallelize, + gpus, max_memory_per_gpu, max_cpu_memory, offload_folder, + peft, delta, autogptq, gptqmodel, gguf_file, **kwargs): + """Create model - same as LLaDA2""" + if autogptq or gptqmodel: + raise NotImplementedError("Quantization options are not implemented.") + model_dtype = get_dtype(dtype) + eval_logger.info(f"Loading model with dtype: {model_dtype}") + model_kwargs = kwargs if kwargs else {} + if not parallelize: + model_kwargs.update( + self._get_accelerate_args( + parallelize=parallelize, + gpus=gpus, + max_memory_per_gpu=max_memory_per_gpu, + max_cpu_memory=max_cpu_memory, + offload_folder=offload_folder + ) + ) + self._model = transformers.AutoModelForCausalLM.from_pretrained( + pretrained, revision=revision, torch_dtype=model_dtype, + trust_remote_code=trust_remote_code, **model_kwargs + ) + if peft: + from peft import PeftModel + eval_logger.info(f"Loading PEFT model from {peft}") + self._model = PeftModel.from_pretrained(self._model, peft, torch_dtype=model_dtype) + if not parallelize: + self._model = self._model.to(self.device) + self._model = self._model.to(torch.bfloat16) + self._model.eval() + + def _create_tokenizer(self, pretrained, tokenizer, revision, trust_remote_code, + use_fast_tokenizer, gguf_file, add_bos_token): + """Create tokenizer - same as LLaDA2""" + kwargs = { + "revision": revision, + "trust_remote_code": trust_remote_code, + "use_fast": use_fast_tokenizer + } + if add_bos_token: + kwargs["add_bos_token"] = True + if tokenizer: + if isinstance(tokenizer, str): + self.tokenizer = transformers.AutoTokenizer.from_pretrained(tokenizer, **kwargs) + else: + self.tokenizer = tokenizer + else: + model_name = pretrained if isinstance(pretrained, str) else self.model.name_or_path + self.tokenizer = transformers.AutoTokenizer.from_pretrained(model_name, **kwargs) + + def tok_encode(self, string, left_truncate_len=None, add_special_tokens=None): + """Tokenize string - same as LLaDA2""" + special_tokens_kwargs = {} + if add_special_tokens is None: + if self.backend == "causal": + special_tokens_kwargs["add_special_tokens"] = self.add_bos_token + else: + special_tokens_kwargs["add_special_tokens"] = add_special_tokens + encoding = self.tokenizer.encode(string, **special_tokens_kwargs) + if left_truncate_len: + encoding = encoding[-left_truncate_len:] + return encoding + + def tok_batch_encode(self, strings, padding_side="left", left_truncate_len=None, truncation=False): + """Batch tokenize - same as LLaDA2""" + old_padding_side = self.tokenizer.padding_side + self.tokenizer.padding_side = padding_side + add_special_tokens = {"add_special_tokens": self.add_bos_token} if self.backend == "causal" else {} + encoding = self.tokenizer( + strings, truncation=truncation, padding="longest", + return_tensors="pt", **add_special_tokens + ) + if left_truncate_len and encoding["input_ids"].size(1) > left_truncate_len: + eval_logger.warning(f"Left-truncating from {encoding['input_ids'].size(1)} to {left_truncate_len} tokens.") + encoding["input_ids"] = encoding["input_ids"][:, -left_truncate_len:] + encoding["attention_mask"] = encoding["attention_mask"][:, -left_truncate_len:] + self.tokenizer.padding_side = old_padding_side + return encoding["input_ids"].to(self.device), encoding["attention_mask"].to(self.device) + + def tok_decode(self, tokens, skip_special_tokens=False): + """Decode tokens - same as LLaDA2""" + return self.tokenizer.decode(tokens, skip_special_tokens=skip_special_tokens) + + def _model_call(self, inps, attn_mask=None, labels=None): + """Model forward call - same as LLaDA2""" + with torch.no_grad(): + if self.backend == "seq2seq": + return self.model(input_ids=inps, attention_mask=attn_mask, labels=labels).logits + else: + return self.model(inps, attention_mask=attn_mask).logits + + def _loglikelihood_tokens(self, requests, **kwargs) -> List[Tuple[float, bool]]: + raise NotImplementedError + + def loglikelihood_rolling( + self, requests: List[Instance], disable_tqdm: bool = False + ) -> List[float]: + raise NotImplementedError + + def loglikelihood(self, requests): + raise NotImplementedError + + def generate_until(self, requests: List[Instance]) -> List[str]: + """Generate until - adapted for LLaDA v1 """ + res = [] + gen_kwargs = requests[0].args[1] + use_hts = gen_kwargs.get("use_hts", False) + + realtime_output = gen_kwargs.get("realtime_output", "realtime_hts_results.jsonl") + baseline_realtime_output = gen_kwargs.get("realtime_output", "realtime_baseline_results.jsonl") + + if not use_hts and "realtime_output" not in gen_kwargs: + baseline_realtime_output = "realtime_baseline_results.jsonl" + + if not use_hts: + bar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Running Baseline (LLaDA v1)") + + for req in requests: + prompt_text = req.args[0] + local_gen_kwargs = req.args[1] if len(req.args) > 1 else {} + + context_enc, _ = self.tok_batch_encode([prompt_text]) + + final_codes, stats = self.hts_sampler.generate_hts( + prompt_text=prompt_text, + input_ids=context_enc, + initial_N=1, + final_K=1, + hts_survivor_k=1, + hts_mode=False, + hts_start_pct=0.0, + hts_end_pct=0.0, + decay_factor=1.5, + pruning_interval=0, + reward_mode="confidence", + task_type=local_gen_kwargs.get("task_type", "code"), + steps=int(local_gen_kwargs.get("steps", 32)), + gen_length=int(local_gen_kwargs.get("gen_length", 512)), + block_length=int(local_gen_kwargs.get("block_length", 32)), + temperature=float(local_gen_kwargs.get("temperature", 0.0)), + top_p=float(local_gen_kwargs.get("top_p", 0.95)), + top_k=local_gen_kwargs.get("top_k", None), + threshold=float(local_gen_kwargs.get("threshold", 0.85)), + mask_id=self.mask_id, + eos_id=self.eot_token_id, + until=local_gen_kwargs.get("until", []), + ) + + processed_codes = [] + for code in final_codes: + code = code.strip() + if not self.escape_until: + until_terms = local_gen_kwargs.get("until", []) + for term in until_terms: + if len(term) > 0 and term in code: + code = code.split(term)[0] + processed_codes.append(code) + + final_choice = processed_codes[0] if processed_codes else "" + res.append(final_choice) + + target_val = getattr(req, "target", None) + if target_val is None or target_val == "N/A": + if "test" in req.doc and "entry_point" in req.doc: + target_val = req.doc["test"] + "\ncheck(" + req.doc["entry_point"] + ")" + else: + target_val = req.doc.get("answer", req.doc.get("solution", "N/A")) + + output_dir = os.path.dirname(baseline_realtime_output) + if output_dir: + os.makedirs(output_dir, exist_ok=True) + with open(baseline_realtime_output, "a", encoding="utf-8") as f: + all_resps = [[code] for code in processed_codes] + output_data = { + "doc": req.doc, + "target": target_val, + "resps": all_resps, + "prompt": prompt_text, + "entropy_history": stats.get("entropy_history", []), + "pruning_history": stats.get("pruning_history", []), + "final_scores": stats.get("final_scores", []), + "all_trajectories": stats.get("all_trajectories", []), + "nfe": stats.get("nfe", 0), + "first_block_nfe": stats.get("first_block_nfe", 0), + "svf_calls": stats.get("svf_calls", 0), + "total_steps": stats.get("total_steps", 0), + "num_gen_blocks": stats.get("num_gen_blocks", []), + "steps_per_block": stats.get("steps_per_block", []) + } + f.write(json.dumps(output_data, ensure_ascii=False) + "\n") + f.flush() + + bar.update(1) + bar.close() + + else: + bar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Running HTS+SVF (LLaDA v1)") + for req in requests: + prompt_text = req.args[0] + local_gen_kwargs = req.args[1] if len(req.args) > 1 else {} + context_enc, _ = self.tok_batch_encode([prompt_text]) + + p_interval = int(local_gen_kwargs.get("pruning_interval", 0)) + + final_codes, stats = self.hts_sampler.generate_hts( + prompt_text=prompt_text, + input_ids=context_enc, + initial_N=int(local_gen_kwargs.get("hts_N", 4)), + final_K=int(local_gen_kwargs.get("final_K", 1)), + hts_survivor_k=int(local_gen_kwargs.get("hts_survivor_k", 4)), + hts_mode=local_gen_kwargs.get("hts_mode", True), + hts_start_pct=float(local_gen_kwargs.get("hts_start_pct", 0.1)), + hts_end_pct=float(local_gen_kwargs.get("hts_end_pct", 0.6)), + decay_factor=float(local_gen_kwargs.get("decay_factor", 1.5)), + pruning_interval=p_interval, + reward_mode=local_gen_kwargs.get("reward_mode", "svf"), + task_type=local_gen_kwargs.get("task_type", "code"), + steps=int(local_gen_kwargs.get("steps", 32)), + gen_length=int(local_gen_kwargs.get("gen_length", 512)), + block_length=int(local_gen_kwargs.get("block_length", 32)), + temperature=float(local_gen_kwargs.get("temperature", 0.7)), + top_p=float(local_gen_kwargs.get("top_p", 0.95)), + top_k=local_gen_kwargs.get("top_k", None), + threshold=float(local_gen_kwargs.get("threshold", 0.85)), + mask_id=self.mask_id, + eos_id=self.eot_token_id, + until=local_gen_kwargs.get("until", []), + ) + + processed_codes = [] + for code in final_codes: + code = code.strip() + if not self.escape_until: + until_terms = local_gen_kwargs.get("until", []) + for term in until_terms: + if len(term) > 0 and term in code: + code = code.split(term)[0] + processed_codes.append(code) + + final_choice = processed_codes[0] + res.append(final_choice) + + target_val = getattr(req, "target", None) + if target_val is None or target_val == "N/A": + if "test" in req.doc and "entry_point" in req.doc: + target_val = req.doc["test"] + "\ncheck(" + req.doc["entry_point"] + ")" + else: + target_val = req.doc.get("answer", req.doc.get("solution", "N/A")) + + output_dir = os.path.dirname(realtime_output) + if output_dir: + os.makedirs(output_dir, exist_ok=True) + with open(realtime_output, "a", encoding="utf-8") as f: + all_resps = [[code] for code in processed_codes] + output_data = { + "doc": req.doc, + "target": target_val, + "resps": all_resps, + "prompt": prompt_text, + "entropy_history": stats.get("entropy_history", []), + "pruning_history": stats.get("pruning_history", []), + "final_scores": stats.get("final_scores", []), + "all_trajectories": stats.get("all_trajectories", []), + "nfe": stats.get("nfe", 0), + "first_block_nfe": stats.get("first_block_nfe", 0), + "svf_calls": stats.get("svf_calls", 0), + "total_steps": stats.get("total_steps", 0), + "num_gen_blocks": stats.get("num_gen_blocks", []), + "steps_per_block": stats.get("steps_per_block", []) + } + f.write(json.dumps(output_data, ensure_ascii=False) + "\n") + f.flush() + + bar.update(1) + bar.close() + + return res + + def apply_chat_template( + self, chat_history: List[Dict[str, str]], add_generation_prompt: bool = True + ) -> str: + """Apply chat template - same as LLaDA2""" + chat_templated = self.tokenizer.apply_chat_template( + chat_history, tokenize=False, add_generation_prompt=add_generation_prompt + ) + if self.assistant_prefix: + chat_templated += self.assistant_prefix + return chat_templated diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/models/__init__.py b/Prism/LLaDA/LLaDA_Prism/dllm_eval/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..89a01de1866ac6d741c08d46d1a3ad8857c902ee --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/dllm_eval/models/__init__.py @@ -0,0 +1,19 @@ +from . import ( + LLaDA, + huggingface, +) +# from .configuration_llada import LLaDAConfig +# from .modeling_llada import LLaDAModelLM + + +try: + # enable hf hub transfer if available + import hf_transfer # type: ignore # noqa + import huggingface_hub.constants # type: ignore + + huggingface_hub.constants.HF_HUB_ENABLE_HF_TRANSFER = True +except ImportError: + pass + + +# __all__ = ['LLaDAConfig', 'LLaDAModelLM'] diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/models/dummy.py b/Prism/LLaDA/LLaDA_Prism/dllm_eval/models/dummy.py new file mode 100644 index 0000000000000000000000000000000000000000..4702a36cb29809c9dd08c516b99e74e71ffcc166 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/dllm_eval/models/dummy.py @@ -0,0 +1,41 @@ +import random + +from tqdm import tqdm + +from dllm_eval.api.model import LM +from dllm_eval.api.registry import register_model + + +@register_model("dummy") +class DummyLM(LM): + def __init__(self) -> None: + super().__init__() + + @classmethod + def create_from_arg_string(cls, arg_string, additional_config=None): + return cls() + + def loglikelihood(self, requests, disable_tqdm: bool = False): + res = [] + + for _ in tqdm(requests, disable=disable_tqdm): + res.append((-random.random(), False)) + + return res + + def generate_until(self, requests, disable_tqdm: bool = False): + res = [] + + for request in tqdm(requests, disable=disable_tqdm): + res.append("lol") + assert request.arguments[0].strip() != "" + + return res + + def loglikelihood_rolling(self, requests, disable_tqdm: bool = False): + res = [] + + for _ in tqdm(requests, disable=disable_tqdm): + res.append(-random.random()) + + return res diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/models/hts_sampler.py b/Prism/LLaDA/LLaDA_Prism/dllm_eval/models/hts_sampler.py new file mode 100644 index 0000000000000000000000000000000000000000..99a3ac049a840ecea3d1301fd6cdc1669f91daa9 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/dllm_eval/models/hts_sampler.py @@ -0,0 +1,337 @@ +import torch +import torch.nn.functional as F +import numpy as np +from .verifier import CodeVerifier +import logging +import re +import math + +logger = logging.getLogger(__name__) + +class HTSSampler: + def __init__(self, model, tokenizer, device="cuda"): + self.model = model + self.tokenizer = tokenizer + self.device = device + self.verifier = CodeVerifier(model, tokenizer, device) + + def _get_num_transfer_tokens(self, block_length, steps): + if steps == 0: return torch.tensor([], dtype=torch.int64) + base = block_length // steps + remainder = block_length % steps + num_transfer_tokens = torch.full((steps,), base, dtype=torch.int64) + num_transfer_tokens[:remainder] += 1 + return num_transfer_tokens + + def _sample_with_temperature(self, logits, temperature, top_k, top_p): + logits = logits.to(torch.float32) + + orig_probs = torch.softmax(logits, dim=-1) + x0_p, _ = torch.max(orig_probs, dim=-1) + + if temperature > 0.0: + noise = torch.rand_like(logits, dtype=torch.float32) + gumbel_noise = -torch.log(-torch.log(noise + 1e-10) + 1e-10) + logits = logits / temperature + gumbel_noise + + if top_k is not None and top_k > 0: + indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, None] + logits[indices_to_remove] = -float('Inf') + + x0 = torch.argmax(logits, dim=-1) + + return x0, x0_p + + def _safe_scalar(self, val): + if isinstance(val, torch.Tensor): + if val.numel() > 1: return val.mean().item() + return val.item() + return float(val) + + def _analyze_structure(self, text, task_type="code"): + score = 0.0 + stripped = text.strip() + if task_type == "code": + if len(stripped) < 5: return -0.1 + keywords = ["return", "print", "yield", "lambda", "class ", "def "] + if any(k in stripped for k in keywords): score += 0.05 + if ":" in stripped: score += 0.02 + if " " in text: score += 0.03 + elif task_type == "math": + if "\\boxed{" in stripped: score += 0.1 + if "The answer is" in stripped: score += 0.05 + if len(stripped) < 10: return -0.1 + if "Step" in stripped and stripped.count("Step") > 15: score -= 0.2 + return score + + def _chunked_forward(self, x, chunk_size=96, slice_indices=None): + total_batch = x.shape[0] + logits_list = [] + for i in range(0, total_batch, chunk_size): + end_idx = min(i + chunk_size, total_batch) + sub_x = x[i:end_idx] + sub_mask = torch.ones_like(sub_x, device=self.device) + with torch.no_grad(): + outputs = self.model(input_ids=sub_x, attention_mask=sub_mask) + sub_logits = outputs.logits + if slice_indices is not None: + s_start, s_end = slice_indices + sub_logits = sub_logits[:, s_start:s_end, :] + logits_list.append(sub_logits.detach().clone()) + return torch.cat(logits_list, dim=0) + + def _branch_and_resample(self, x, conf_scores, survivor_indices, target_width, mask_id, + prompt_length, resample_window=6, task_type="code"): + num_survivors = len(survivor_indices) + if num_survivors == 0: return x[:target_width].clone(), conf_scores[:target_width].clone() + + if task_type == "math": resample_window = 2 + elif task_type == "reasoning": resample_window = 6 + elif task_type == "code": resample_window = 2 + + base_repeat = target_width // num_survivors + remainder = target_width % num_survivors + new_x_list = [] + new_conf_list = [] + + for i in range(num_survivors): + count = base_repeat + (1 if i < remainder else 0) + if count == 0: continue + + survivor_x = x[survivor_indices[i]] + survivor_conf = conf_scores[survivor_indices[i]] + + new_x_list.append(survivor_x.unsqueeze(0)) + new_conf_list.append(survivor_conf.unsqueeze(0)) + + if count > 1: + gen_part = survivor_x[prompt_length:] + gen_conf = survivor_conf[prompt_length:] + non_mask_indices = (gen_part != mask_id).nonzero(as_tuple=True)[0] + + for _ in range(count - 1): + perturbed_x = survivor_x.clone() + perturbed_conf = survivor_conf.clone() + + if len(non_mask_indices) > 0: + pool_size = min(resample_window * 3, len(non_mask_indices)) + current_token_confs = gen_conf[non_mask_indices] + + _, candidate_indices = torch.topk(current_token_confs, k=pool_size, largest=False) + + num_to_perturb = min(resample_window, pool_size) + rand_indices = torch.randperm(pool_size, device=self.device)[:num_to_perturb] + selected_sub_indices = candidate_indices[rand_indices] + + target_indices_in_x = prompt_length + non_mask_indices[selected_sub_indices] + perturbed_x[target_indices_in_x] = mask_id + perturbed_conf[target_indices_in_x] = 0.0 + + new_x_list.append(perturbed_x.unsqueeze(0)) + new_conf_list.append(perturbed_conf.unsqueeze(0)) + + return torch.cat(new_x_list, dim=0), torch.cat(new_conf_list, dim=0) + + @torch.no_grad() + def generate_hts(self, prompt_text, input_ids, problem_data=None, + initial_N=1, final_K=1, survivor_K=None, + prune_step_pct=0.0, reward_mode="confidence", + temperature=0.7, block_length=32, steps=64, gen_length=1024, + top_p=0.95, top_k=None, minimal_topk=1, threshold=0.9, + eos_id=156892, mask_id=156895, + hts_mode=False, hts_start_pct=0.1, hts_end_pct=0.6, decay_factor=1.5, + hts_survivor_k=4, task_type="code", until=None, pruning_interval=0): + + input_ids = input_ids.to(self.device) + if input_ids.shape[0] == 1: input_ids = input_ids.repeat(initial_N, 1) + + schedule_map = {} + ts_start, tr_end = 0, 0 + if not hts_mode: + final_K_list = [final_K] if not isinstance(final_K, list) else final_K + prune_pct_list = [prune_step_pct] if not isinstance(prune_step_pct, list) else prune_step_pct + survivor_K_list = final_K_list if survivor_K is None else ([survivor_K] if not isinstance(survivor_K, list) else survivor_K) + if len(survivor_K_list) < len(final_K_list): survivor_K_list.extend(final_K_list[len(survivor_K_list):]) + for pct, width, parents in zip(prune_pct_list, final_K_list, survivor_K_list): + if pct > 0: + s = int(steps * pct) + schedule_map[s] = (width, parents) + else: + final_K_list = [final_K] if not isinstance(final_K, int) else [final_K] + ts_start, tr_end = int(steps * hts_start_pct), int(steps * hts_end_pct) + + steps = min(steps, gen_length // minimal_topk) + prompt_length = input_ids.shape[1] + num_blocks = (prompt_length + gen_length + block_length - 1) // block_length + total_length = num_blocks * block_length + + x = torch.full((initial_N, total_length), mask_id, dtype=torch.long, device=self.device) + x[:, :prompt_length] = input_ids.clone() + + conf_scores = torch.zeros((initial_N, total_length), dtype=torch.float32, device=self.device) + conf_scores[:, :prompt_length] = 1.0 + + prefill_blocks = prompt_length // block_length + num_gen_blocks = max(1, num_blocks - prefill_blocks) + current_bsz = initial_N + + next_allowed_pruning_step = ts_start if hts_mode else 0 + + stats = { + "initial_n": initial_N, "final_k": final_K_list[-1], + "pruning_history": [], "entropy_history": [], "nfe": 0.0, + "svf_calls": 0, "final_scores": [], "total_steps": steps, + "first_block_nfe": 0.0, "num_gen_blocks": [], "steps_per_block": [] + } + + for num_block in range(prefill_blocks, num_blocks): + stats["num_gen_blocks"].append(num_block) + + window_end = (num_block + 1) * block_length + schedule = self._get_num_transfer_tokens(block_length, steps) + + steps_this_block = 0 + for step in range(steps): + steps_this_block += 1 + cur_full_x = x[:current_bsz, :] + + perform_pruning = False + num_parents_to_select = 0 + + if hts_mode and step >= next_allowed_pruning_step and step < tr_end: + target_width = max(final_K_list[-1], math.ceil(initial_N * (decay_factor ** -(step - ts_start)))) + if current_bsz > target_width: + perform_pruning = True + num_parents_to_select = hts_survivor_k + elif not hts_mode and step in schedule_map: + target_width, num_parents_to_select = schedule_map[step] + if current_bsz > target_width: perform_pruning = True + + if perform_pruning: + stats["nfe"] += current_bsz + if num_block == prefill_blocks: + stats["first_block_nfe"] += current_bsz + + stats["svf_calls"] += current_bsz + + gen_logits = self._chunked_forward(cur_full_x, chunk_size=64, slice_indices=(prompt_length, total_length)) + rough_ids = torch.argmax(gen_logits, dim=-1) + rough_codes_snippet = self.tokenizer.batch_decode(rough_ids, skip_special_tokens=True) + candidates = [] + for i in range(current_bsz): + full_code = rough_codes_snippet[i] + s = self._safe_scalar(self.verifier.get_reward(prompt_text, full_code, mode=reward_mode, problem_data=problem_data, current_logits=gen_logits[i] if reward_mode != "svf" else None, task_type=task_type)) + s += self._analyze_structure(full_code, task_type=task_type) + clean_content = full_code.strip().replace(" ", "").replace("\n", "") + candidates.append({'score': s, 'idx': i, 'key': hash(clean_content[:200] + clean_content[-200:])}) + + stats["pruning_history"].append({"step": step, "scores": [c['score'] for c in candidates]}) + candidates.sort(key=lambda x: x['score'], reverse=True) + + selected_indices, seen_keys = [], set() + for cand in candidates: + if len(selected_indices) >= num_parents_to_select: break + if cand['key'] not in seen_keys: + selected_indices.append(cand['idx']); seen_keys.add(cand['key']) + + if len(selected_indices) < num_parents_to_select: + for cand in candidates: + if len(selected_indices) >= num_parents_to_select: break + if cand['idx'] not in selected_indices: selected_indices.append(cand['idx']) + + top_indices = torch.tensor(selected_indices, device=self.device) + x, conf_scores = self._branch_and_resample(x, conf_scores, top_indices, target_width, mask_id, prompt_length, task_type=task_type) + + current_bsz = target_width + cur_full_x = x[:current_bsz, :] + next_allowed_pruning_step = step + 1 + pruning_interval + + active_mask = x[:current_bsz, window_end-block_length:window_end] == mask_id + if active_mask.sum() == 0: break + + stats["nfe"] += current_bsz + if num_block == prefill_blocks: + stats["first_block_nfe"] += current_bsz + + active_logits = self._chunked_forward(cur_full_x, chunk_size=32, slice_indices=(window_end-block_length, window_end)) + + active_logits[:, :, eos_id] = -1e10 + + with torch.no_grad(): + if len(stats["entropy_history"]) < 32: + probs_for_stats = torch.softmax(active_logits.float(), dim=-1) + entropy_per_branch = (-(probs_for_stats * torch.log(probs_for_stats + 1e-10)).sum(dim=-1).mean(dim=-1)).cpu().numpy().tolist() + stats["entropy_history"].append(entropy_per_branch) + + x0, x0_p = self._sample_with_temperature(active_logits, temperature, top_k, top_p) + + num_transfer = schedule[step].item() + confidence = torch.where(active_mask, x0_p, -torch.inf) + transfer_idx = torch.zeros_like(x0, dtype=torch.bool) + + for b in range(current_bsz): + k_transfer = min(num_transfer, active_mask[b].sum().item()) + active_indices = torch.where(active_mask[b])[0] + if (confidence[b] > threshold).sum().item() >= k_transfer: + conf_indices = torch.where((confidence[b] > threshold) & active_mask[b])[0]; transfer_idx[b, conf_indices] = True + elif len(active_indices) > 0: + _, topk_indices = torch.topk(confidence[b][active_indices], k=min(k_transfer, len(active_indices))); transfer_idx[b, active_indices[topk_indices]] = True + + if transfer_idx.any(): + x[:current_bsz, window_end-block_length:window_end][transfer_idx] = x0[transfer_idx] + conf_scores[:current_bsz, window_end-block_length:window_end][transfer_idx] = x0_p[transfer_idx] + + if task_type in ["math", "reasoning"]: + for b in range(current_bsz): + gen_span = x[b, prompt_length:window_end] + text_snippet = self.tokenizer.decode(gen_span, skip_special_tokens=True) + should_stop = False + if task_type == "reasoning" and ("###" in text_snippet): + should_stop = True + if task_type == "math" and ("\\boxed{" in text_snippet and "}" in text_snippet.split("\\boxed{")[-1]): + should_stop = True + + if should_stop: + non_mask_indices = (gen_span != mask_id).nonzero(as_tuple=True)[0] + if len(non_mask_indices) > 0: + last_idx = non_mask_indices[-1].item() + if last_idx + 1 < len(gen_span): + x[b, prompt_length + last_idx + 1 : window_end] = eos_id + if window_end < total_length: + x[b, window_end:] = eos_id + conf_scores[b, window_end:] = 1.0 + + for b in range(current_bsz): + gen_window = x[b, prompt_length:window_end] + eos_indices = (gen_window == eos_id).nonzero(as_tuple=True)[0] + if len(eos_indices) > 0: + first_eos_idx = eos_indices[0].item() + if first_eos_idx + 1 < len(gen_window): + x[b, prompt_length + first_eos_idx + 1 : window_end] = eos_id + + stats["steps_per_block"].append(steps_this_block) + x = x[:current_bsz] + + stats["nfe"] = int(round(stats["nfe"])) + stats["first_block_nfe"] = int(round(stats["first_block_nfe"])) + + final_gen_tokens = x[:current_bsz, prompt_length:] + final_codes = self.tokenizer.batch_decode(final_gen_tokens, skip_special_tokens=True) + final_candidates = [] + + stats["svf_calls"] += len(final_codes) + + for i in range(len(final_codes)): + txt = final_codes[i] + if until: + for term in until: + if term in txt: txt = txt.split(term)[0] + s = self._safe_scalar(self.verifier.get_reward(prompt_text, txt, mode=reward_mode, task_type=task_type)) + s += self._analyze_structure(txt, task_type) + final_candidates.append({'resp': txt, 'score': s}) + + final_candidates.sort(key=lambda x: x['score'], reverse=True) + stats["final_scores"] = [c['score'] for c in final_candidates] + stats["all_trajectories"] = [{"rank": i+1, "resp": c['resp'], "score": c['score']} for i, c in enumerate(final_candidates)] + + return [c['resp'] for c in final_candidates], stats \ No newline at end of file diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/models/huggingface.py b/Prism/LLaDA/LLaDA_Prism/dllm_eval/models/huggingface.py new file mode 100644 index 0000000000000000000000000000000000000000..bf6e1e99e20aeed5b20f7cd2d7a8f9b76155330a --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/dllm_eval/models/huggingface.py @@ -0,0 +1,1489 @@ +import copy +import logging +import os +from datetime import timedelta +from pathlib import Path +from typing import Any, Dict, List, Literal, Optional, Tuple, Union + +import jinja2 +import torch +import torch.nn.functional as F +import transformers +from accelerate import ( + Accelerator, + InitProcessGroupKwargs, + find_executable_batch_size, +) +from accelerate.utils import get_max_memory +from huggingface_hub import HfApi +from packaging import version +from peft import PeftModel +from peft import __version__ as PEFT_VERSION +from tqdm import tqdm +from transformers.models.auto.modeling_auto import ( + MODEL_FOR_CAUSAL_LM_MAPPING_NAMES, + MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES, +) + +from dllm_eval import utils +from dllm_eval.api.instance import Instance +from dllm_eval.api.model import TemplateLM +from dllm_eval.api.registry import register_model +from dllm_eval.models.utils import ( + Collator, + clear_torch_cache, + configure_pad_token, + get_dtype, + handle_stop_sequences, + pad_and_concat, + stop_sequences_criteria, +) + + +eval_logger = logging.getLogger(__name__) + + +@register_model("hf-auto", "hf", "huggingface") +class HFLM(TemplateLM): + """ + An abstracted Huggingface model class. Enables usage with both models of + `transformers.AutoModelForCausalLM` and `transformers.AutoModelForSeq2SeqLM` classes. + + Supports data-parallel multi-GPU with HF Accelerate. + """ + + AUTO_MODEL_CLASS = None + _DEFAULT_MAX_LENGTH = 2048 + + def __init__( + self, + pretrained: Union[str, transformers.PreTrainedModel], + backend: Literal["default", "causal", "seq2seq"] = "default", + # override whether the model should be treated as decoder-only (causal) or encoder-decoder (seq2seq) + revision: Optional[str] = "main", + subfolder: str = "", + tokenizer: Optional[ + Union[ + str, + transformers.PreTrainedTokenizer, + transformers.PreTrainedTokenizerFast, + ] + ] = None, + truncation: Optional[bool] = False, + logits_cache: bool = True, + max_length: Optional[int] = None, + device: Optional[str] = "cuda", + dtype: Optional[Union[str, torch.dtype]] = "auto", + softmax_dtype: Optional[Union[str, torch.dtype]] = None, + batch_size: Optional[Union[int, str]] = 1, + max_batch_size: Optional[int] = 64, + trust_remote_code: Optional[bool] = False, + use_fast_tokenizer: Optional[bool] = True, + add_bos_token: Optional[bool] = False, + prefix_token_id: Optional[int] = None, + # arguments used for splitting a model across GPUs naively. + # only used if `parallelize=True`. + parallelize: Optional[bool] = False, + max_memory_per_gpu: Optional[Union[int, str]] = None, + max_cpu_memory: Optional[Union[int, str]] = None, + offload_folder: Optional[Union[str, os.PathLike]] = "./offload", + # PEFT, delta weights and quantization options + peft: Optional[str] = None, + delta: Optional[str] = None, + autogptq: Optional[Union[bool, str]] = False, + gptqmodel: Optional[bool] = False, + gguf_file: Optional[str] = None, + **kwargs, + ) -> None: + super().__init__() + # optionally: take in an already-initialized transformers.PreTrainedModel + if not isinstance(pretrained, str): + eval_logger.warning( + "`pretrained` model kwarg is not of type `str`. Many other model arguments may be ignored. Please do not launch via accelerate or use `parallelize=True` if passing an existing model this way." + ) + assert not parallelize, ( + "`parallelize=True` is not compatible with passing pre-initialized model to `pretrained`" + ) + self._model = pretrained + self._device = self._model.device + self._config = self._model.config + gpus = 0 + + else: + assert isinstance(device, str) + assert isinstance(pretrained, str) + assert isinstance(batch_size, (int, str)) + + gpus = torch.cuda.device_count() + accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52)) + accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs]) + if accelerator.num_processes > 1: + self.accelerator = accelerator + + if "npu" in accelerator.device.type: + gpus = torch.npu.device_count() + + # using one process with no model parallelism + if not (parallelize or accelerator.num_processes > 1): + # use user-passed device + device_list = set( + ["cuda", "cpu"] + + [f"cuda:{i}" for i in range(gpus)] + + ["mps", "mps:0"] + + [f"npu:{i}" for i in range(gpus)] + ) + if device and device in device_list: + self._device = torch.device(device) + eval_logger.info(f"Using device '{device}'") + if device in ("mps", "mps:0") and version.parse( + torch.__version__ + ) < version.parse("2.1"): + raise RuntimeError( + f"mps requires torch >= 2.1. You have {torch.__version__}" + ) + else: + eval_logger.info("Device not specified") + eval_logger.info(f"Cuda Available? {torch.cuda.is_available()}") + self._device = ( + torch.device("cuda") + if torch.cuda.is_available() + else torch.device("cpu") + ) + else: # Parallelism managed by accelerate + if device != "cuda": + eval_logger.info( + f"Using `accelerate launch` or `parallelize=True`, device '{device}' will be overridden when placing model." + ) + # TODO: include in warning that `load_in_8bit` etc. affect this too + self._device = ( + self.accelerator.device + if hasattr(self, "accelerator") + else torch.device(device) + ) + + revision = str(revision) # cast to string if not already one + + self._get_config( + pretrained, + revision=revision, + trust_remote_code=trust_remote_code, + gguf_file=gguf_file, + subfolder=subfolder, + ) + + # determine which of 'causal' and 'seq2seq' backends to use for HF models + self._get_backend( + config=self.config, backend=backend, trust_remote_code=trust_remote_code + ) + + # load tokenizer so we know tokenizer vocabulary size before loading model and PEFT + self._create_tokenizer( + pretrained, + tokenizer, + revision=revision, + subfolder=subfolder, + trust_remote_code=trust_remote_code, + use_fast_tokenizer=use_fast_tokenizer, + gguf_file=gguf_file, + add_bos_token=add_bos_token, + ) + + # if we passed `pretrained` as a string, initialize our model now + if isinstance(pretrained, str): + self._create_model( + pretrained=pretrained, + revision=revision, + dtype=dtype, + trust_remote_code=trust_remote_code, + parallelize=parallelize, + gpus=gpus, + max_memory_per_gpu=max_memory_per_gpu, + max_cpu_memory=max_cpu_memory, + offload_folder=offload_folder, + peft=peft, + delta=delta, + autogptq=autogptq, + gptqmodel=gptqmodel, + gguf_file=gguf_file, + quantization_config=getattr(self.config, "quantization_config", None), + subfolder=subfolder, + **kwargs, + ) + + # access self._model through self.model property outside this method + if isinstance(self.model, torch.nn.Module): + self.model.eval() + self.model.tie_weights() + + self.truncation = truncation + self.logits_cache = logits_cache + self.vocab_size = self.tokenizer.vocab_size + # select (or create) a pad token to use + self.tokenizer = configure_pad_token(self.tokenizer, model_config=self.config) + + self.add_bos_token = add_bos_token + if "gemma" in getattr(self.config, "model_type", ""): + self.add_bos_token = True + eval_logger.info( + f"Model type is '{self.config.model_type}', part of the Gemma family--a BOS token will be used as Gemma underperforms without it." + ) + + self._max_length = max_length + self.pretrained = pretrained + self.delta = delta + self.peft = peft + self.revision = revision + self.batch_schedule = 1 + self.batch_sizes = {} + self.max_batch_size = max_batch_size + self.softmax_dtype = ( + get_dtype(softmax_dtype) if softmax_dtype is not None else None + ) + + if str(batch_size).startswith("auto"): + batch_size = batch_size.split(":") + self.batch_size_per_gpu = batch_size[0] + self.batch_schedule = float(batch_size[1]) if len(batch_size) > 1 else 1 + else: + self.batch_size_per_gpu = int(batch_size) + + if isinstance(pretrained, str): + if gpus >= 1 or str(self.device) == "mps": + # TODO: can remove this whole snippet except in the mps case, perhaps? + if not (parallelize or autogptq or hasattr(self, "accelerator")): + # place model onto device requested manually, + # if not using HF Accelerate or device_map + # or any other option that preloads model onto device + try: + self.model.to(self.device) + except ValueError: + eval_logger.debug( + "Failed to place model onto specified device. This may be because the model is quantized via `bitsandbytes` or `device_map` is provided. If the desired GPU is being used, this message is safe to ignore." + ) + # multigpu data-parallel support when launched with accelerate + if gpus > 1: + if accelerator.num_processes > 1: + if parallelize: + eval_logger.warning( + "You are both using a HF Accelerate `device_map` (`--model_args parallelize=True`) and launching via `accelerate launch`. This will attempt to do model and data parallelism depending on the resources available." + ) + elif gpus > accelerator.num_processes: + eval_logger.warning( + "WARNING: The number of total system GPUs does not match the number of spawned processes. " + "If you would like to use data parallelism, please launch the script " + "with 'accelerate launch *script*'. " + f"Current run will proceed with {accelerator.num_processes} devices." + ) + if self.accelerator.is_local_main_process: + eval_logger.info( + f"Using {gpus} devices with data parallelism" + ) + + self._device = torch.device(f"{accelerator.device}") + self.accelerator = accelerator + + self._rank = self.accelerator.local_process_index + self._world_size = self.accelerator.num_processes + else: + # if we aren't launching via accelerate, ditch + self._rank = 0 + self._world_size = 1 + else: + # if a PreTrainedModel was passed into HFLM, we forgo distributed setup. + eval_logger.warning( + "Passed an already-initialized model through `pretrained`, assuming single-process call to evaluate() or custom distributed integration" + ) + self._rank = 0 + self._world_size = 1 + + self.custom_prefix_token_id = prefix_token_id + if prefix_token_id is not None: + eval_logger.info( + f"Loglikelihood prefix token id used in evaluation: {self.prefix_token_id}" + ) + + def _get_accelerate_args( + self, + parallelize: Optional[bool] = None, + device_map: Optional[str] = "auto", + max_memory_per_gpu: Optional[Union[int, str]] = None, + max_cpu_memory: Optional[Union[int, str]] = None, + offload_folder: Optional[str] = "./offload", + gpus: Optional[int] = None, + ) -> dict: + """Returns the kwargs needed to apply `accelerate` in `AutoModel.from_pretrained`.""" + num_local_processes = int(os.environ.get("LOCAL_WORLD_SIZE", 1)) + num_machines = int(os.environ.get("WORLD_SIZE", 0)) // num_local_processes + if ( + num_machines == 0 + and hasattr(self, "accelerator") + and self.accelerator is not None + ): + eval_logger.info( + "We are not in a distributed setting for accelerate. Setting model_parallel to False." + ) + parallelize = False + + if parallelize is None: + # If parallelism is unset by the user, we automatically assign model parallelism + # if enough extra GPUs are available + max_memory_all_gpus = get_max_memory() + # We just want gpu, not cpu, max memory + if "cpu" in max_memory_all_gpus: + del max_memory_all_gpus["cpu"] + parallelize = bool(num_local_processes < len(max_memory_all_gpus)) + eval_logger.info( + f"Setting model parallel to {parallelize} since " + f"the number of local processes is {num_local_processes} " + f"and the number of GPUs is {len(max_memory_all_gpus)}" + ) + + args = {} + if parallelize: # Model parallelism will be used + max_memory = {} + if max_memory_per_gpu is not None: # Using the provided memory requirements + max_memory_per_gpu_map = { + device_idx: max_memory_per_gpu for device_idx in range(gpus) + } + else: # Estimating the possible memory requirements + max_memory_all_gpus = get_max_memory() + if "cpu" in max_memory_all_gpus: + del max_memory_all_gpus["cpu"] + if not hasattr(self, "accelerator"): + max_memory_per_gpu_map = { + k: v for k, v in max_memory_all_gpus.items() + } + else: + # use only 1 / num_processes of the GPUs if we are running under accelerate launch + max_memory_per_gpu_map = { + k: v + for k, v in max_memory_all_gpus.items() + if k % num_local_processes + == (self.accelerator.process_index % num_local_processes) + } + args["max_memory"] = max_memory_per_gpu_map + args["device_map"] = "auto" if device_map is None else device_map + eval_logger.info( + f"Model parallel was set to True, setting max memory per GPU to {max_memory_per_gpu_map} and device map to {args.get('device_map')}" + ) + + if max_cpu_memory is not None: + max_memory["cpu"] = max_cpu_memory + + args["offload_folder"] = offload_folder + elif ( + device_map is None + ): # No model parallelism, we use the default provided device for our model + if hasattr(self, "accelerator"): + device_map = {"": f"{self.accelerator.device}"} + else: + device_map = {"": str(self.device)} + args["max_memory"] = None + args["device_map"] = device_map + eval_logger.info( + f"Model parallel was set to False, max memory was not set, and device map was set to {device_map}" + ) + else: + args["max_memory"] = None + args["device_map"] = None + eval_logger.info("Model parallel was set to False.") + + return args + + @property + def config(self): + # return the associated transformers.AutoConfig for the given pretrained model. + return self._config + + @property + def model(self): + # returns the model, unwrapping it if using Accelerate + if hasattr(self, "accelerator"): + return self.accelerator.unwrap_model(self._model) + else: + return self._model + + @property + def eot_token_id(self): + # we use EOT because end of *text* is more accurate for what we're doing than end of *sentence* + return self.tokenizer.eos_token_id + + @property + def prefix_token_id(self): + # it is used as prefix for loglikelihood + if self.custom_prefix_token_id is not None: + return self.custom_prefix_token_id + if self.tokenizer.bos_token_id is not None: + return self.tokenizer.bos_token_id + return self.tokenizer.eos_token_id + + @property + def max_length(self): + if self._max_length: # if max length manually set, return it + return self._max_length + seqlen_config_attrs = ("n_positions", "max_position_embeddings", "n_ctx") + for attr in seqlen_config_attrs: + if hasattr(self.model.config, attr): + return getattr(self.model.config, attr) + if hasattr(self.tokenizer, "model_max_length"): + if self.tokenizer.model_max_length == 1000000000000000019884624838656: + return self._DEFAULT_MAX_LENGTH + return self.tokenizer.model_max_length + return self._DEFAULT_MAX_LENGTH + + @property + def max_gen_toks(self) -> int: + return 256 + + @property + def batch_size(self): + return self.batch_size_per_gpu + + @property + def device(self): + return self._device + + @property + def rank(self): + return self._rank + + @property + def world_size(self): + return self._world_size + + @property + def tokenizer_name(self) -> str: + return self.tokenizer.name_or_path.replace("/", "__") + + def _get_backend( + self, + config: Union[transformers.PretrainedConfig, transformers.AutoConfig], + backend: Literal["default", "causal", "seq2seq"] = "default", + trust_remote_code: Optional[bool] = False, + ) -> None: + """ + Helper method during initialization. + Determines the backend ("causal" (decoder-only) or "seq2seq" (encoder-decoder)) model type to be used. + sets `self.AUTO_MODEL_CLASS` appropriately if not already set. + + **If not calling HFLM.__init__() or HFLM._get_backend() within a subclass of HFLM, + user must set `self.backend` to be either "causal" or "seq2seq" manually!** + """ + + assert backend in ["default", "causal", "seq2seq"] + + if backend != "default": + # if we've settled on non-default backend, use that manually + if backend == "causal": + self.backend = backend + elif backend == "seq2seq": + self.backend = backend + eval_logger.info( + f"Overrode HF model backend type, and using type '{self.backend}'" + ) + else: + # determine and use the default HF backend for this model, based on its config + metadata. + if ( + getattr(config, "model_type") + in MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES + ): + # first check if model type is listed under seq2seq models, since some + # models like MBart are listed in both seq2seq and causal mistakenly in HF transformers. + # these special cases should be treated as seq2seq models. + self.backend = "seq2seq" + eval_logger.debug(f"Using model type '{self.backend}'") + elif ( + getattr(self.config, "model_type") in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES + ): + self.backend = "causal" + eval_logger.debug(f"Using model type '{self.backend}'") + else: + if not trust_remote_code: + eval_logger.warning( + "HF model type is neither marked as CausalLM or Seq2SeqLM. \ + This is expected if your model requires `trust_remote_code=True` but may be an error otherwise." + "Setting backend to causal" + ) + # if model type is neither in HF transformers causal or seq2seq model registries + # then we default to assuming AutoModelForCausalLM + self.backend = "causal" + eval_logger.info( + f"Model type cannot be determined. Using default model type '{self.backend}'" + ) + + if self.AUTO_MODEL_CLASS is None: + if self.backend == "causal": + self.AUTO_MODEL_CLASS = transformers.AutoModelForCausalLM + elif self.backend == "seq2seq": + self.AUTO_MODEL_CLASS = transformers.AutoModelForSeq2SeqLM + + def _get_config( + self, + pretrained: str, + revision: str = "main", + trust_remote_code: bool = False, + gguf_file: Optional[str] = None, + subfolder: str = "", + ) -> None: + """Return the model config for HuggingFace models""" + self._config = transformers.AutoConfig.from_pretrained( + pretrained, + revision=revision, + trust_remote_code=trust_remote_code, + gguf_file=gguf_file, + subfolder=subfolder, + ) + + def _create_model( + self, + pretrained: str, + revision: Optional[str] = "main", + dtype: Optional[Union[str, torch.dtype]] = "auto", + trust_remote_code: Optional[bool] = False, + # arguments used for splitting a model across GPUs naively. + # only used if `parallelize=True`. + # (accelerate naive PP (device_map) options) + parallelize: Optional[bool] = False, + gpus: Optional[int] = None, + max_memory_per_gpu: Optional[Union[int, str]] = None, + max_cpu_memory: Optional[Union[int, str]] = None, + offload_folder: Optional[str] = "./offload", + # PEFT, delta weights and quantization options + peft: Optional[str] = None, + delta: Optional[str] = None, + autogptq: Optional[Union[bool, str]] = False, + gptqmodel: Optional[bool] = False, + gguf_file: Optional[str] = None, + quantization_config: Optional[Dict[str, Any]] = None, + subfolder: str = "", + **kwargs, + ) -> None: + """ + Initializes an HF or HF-compatible PreTrainedModel from scratch + inside HFLM, using the kwargs passed into self.__init__(). + + Also handles functionality such as AutoGPTQ usage and PEFT wrapping. + + For future similar extensions to AutoGPTQ that are not core to HF's ecosystem, + (such as PyTorch models that are nearly, but not quite, fully mirroring + HF's public interface relied on in this HFLM class) + please consider subclassing HFLM and overriding this and other methods as needed. + """ + + model_kwargs = kwargs if kwargs else {} + + model_kwargs.update( + self._get_accelerate_args( + parallelize=parallelize, + device_map=kwargs.get("device_map", None), + max_memory_per_gpu=max_memory_per_gpu, + max_cpu_memory=max_cpu_memory, + offload_folder=offload_folder, + gpus=gpus, + ) + ) + + if not autogptq and not gptqmodel: + if model_kwargs.get("load_in_4bit", None): + assert transformers.__version__ >= "4.30.0", ( + "load_in_4bit requires transformers >= 4.30.0" + ) + if transformers.__version__ >= "4.30.0": + if model_kwargs.get("load_in_4bit", None): + if model_kwargs.get("bnb_4bit_compute_dtype", None): + model_kwargs["bnb_4bit_compute_dtype"] = get_dtype( + model_kwargs["bnb_4bit_compute_dtype"] + ) + + self._model = self.AUTO_MODEL_CLASS.from_pretrained( + pretrained, + revision=revision, + torch_dtype=get_dtype(dtype), + trust_remote_code=trust_remote_code, + gguf_file=gguf_file, + quantization_config=quantization_config, + subfolder=subfolder, + **model_kwargs, + ) + else: + if autogptq and gptqmodel: + raise ValueError( + "Cannot use both 'autogptq' and 'gptqmodel' options at the same time." + ) + + if autogptq: + try: + from auto_gptq import AutoGPTQForCausalLM + except ModuleNotFoundError as exception: + raise type(exception)( + "Tried to load auto_gptq, but auto-gptq is not installed ", + "please install auto-gptq via pip install lm-eval[gptq] or pip install -e .[gptq]", + ) + + self._model = AutoGPTQForCausalLM.from_quantized( + pretrained, + trust_remote_code=trust_remote_code, + model_basename=None if autogptq is True else Path(autogptq).stem, + use_safetensors=True + if autogptq is True + else autogptq.endswith(".safetensors"), + **model_kwargs, + ) + + if gptqmodel: + try: + from gptqmodel import GPTQModel + except ModuleNotFoundError as exception: + raise type(exception)( + "Tried to load gptqmodel, but gptqmodel is not installed ", + "please install gptqmodel via `pip install gptqmodel --no-build-isolation` or `pip install lm-eval[gptqmodel] --no-build-isolation`", + ) + + self._model = GPTQModel.from_quantized( + pretrained, trust_remote_code=trust_remote_code, **model_kwargs + ) + + if peft and delta: + raise ValueError( + "Cannot use both 'peft' and 'delta' options at the same time." + ) + + if peft: + if model_kwargs.get("load_in_4bit", None): + if version.parse(PEFT_VERSION) < version.parse("0.4.0"): + raise AssertionError("load_in_4bit requires peft >= 0.4.0") + if self._model.config.vocab_size != len(self.tokenizer): + # resize model for LoRAs with added tokens + eval_logger.info( + f"Model config indicates vocab_size='{self._model.config.vocab_size}', but found tokenizer with vocab size '{len(self.tokenizer)}'. Resizing model embedding layer..." + ) + self._model.resize_token_embeddings(len(self.tokenizer)) + self._model = PeftModel.from_pretrained( + self._model, peft, revision=revision + ) + elif delta: + if autogptq: + eval_logger.warning( + "Delta weights might trigger unexpected behavior when used with AutoGPTQ." + ) + _model_delta = self.AUTO_MODEL_CLASS.from_pretrained( + delta, + revision=revision, + torch_dtype=get_dtype(dtype), + trust_remote_code=trust_remote_code, + **model_kwargs, + ) + for name, param in self._model.state_dict().items(): + try: + param.data += _model_delta.state_dict()[name] + except KeyError: + raise KeyError(f"Delta model is missing weights for layer: {name}") + except Exception as e: + raise RuntimeError( + f"Failed to add delta weights to layer {name}. Error: {e}" + ) + + del _model_delta + + return None + + def _create_tokenizer( + self, + pretrained: Union[str, transformers.PreTrainedModel], + tokenizer: Optional[ + Union[ + str, + transformers.PreTrainedTokenizer, + transformers.PreTrainedTokenizerFast, + ] + ], + revision: Optional[str] = "main", + trust_remote_code: Optional[bool] = False, + use_fast_tokenizer: Optional[bool] = True, + gguf_file: Optional[str] = None, + add_bos_token: Optional[bool] = False, + subfolder: Optional[str] = "", + ) -> None: + """ + Helper method during initialization. + + Create a tokenizer object corresponding to the correct + tokenizer for value of `pretrained`, or use the pre-initialized tokenizer passed. + """ + kwargs = { + "revision": revision, + "trust_remote_code": trust_remote_code, + } + + # gguf format embeds tokenizer and is not compatible with hf tokenizer `use_fast` param + if gguf_file is not None: + kwargs["gguf_file"] = gguf_file + else: + kwargs["use_fast"] = use_fast_tokenizer + + if add_bos_token: + kwargs["add_bos_token"] = True + + if subfolder: + kwargs["subfolder"] = subfolder + + if tokenizer: + if isinstance(tokenizer, str): + self.tokenizer = transformers.AutoTokenizer.from_pretrained( + tokenizer, **kwargs + ) + else: + assert isinstance( + tokenizer, transformers.PreTrainedTokenizer + ) or isinstance(tokenizer, transformers.PreTrainedTokenizerFast) + self.tokenizer = tokenizer + else: + # Get tokenizer based on 'pretrained' + if isinstance(pretrained, str): + model_name = pretrained + else: + # get the HF hub name via accessor on model + model_name = self.model.name_or_path + self.tokenizer = transformers.AutoTokenizer.from_pretrained( + model_name, **kwargs + ) + return None + + def _detect_batch_size(self, requests=None, pos: int = 0): + if requests: + _, context_enc, continuation_enc = requests[pos] + max_length = len( + (context_enc + continuation_enc)[-(self.max_length + 1) :][:-1] + ) + max_context_enc = len(context_enc[-(self.max_length + 1) :]) + max_cont_enc = len(continuation_enc[-(self.max_length + 1) :]) + else: + max_length = self.max_length + max_context_enc = max_length + max_cont_enc = max_length + + # if OOM, then halves batch_size and tries again + @find_executable_batch_size(starting_batch_size=self.max_batch_size) + def forward_batch(batch_size): + if self.backend == "seq2seq": + length = max(max_context_enc, max_cont_enc) + batched_conts = torch.ones( + (batch_size, length), device=self.device + ).long() + test_batch = torch.ones((batch_size, length), device=self.device).long() + call_kwargs = { + "attn_mask": test_batch, + "labels": batched_conts, + } + else: + call_kwargs = {} + test_batch = torch.ones( + (batch_size, max_length), device=self.device + ).long() + for _ in range(5): + out = F.log_softmax( # noqa: F841 + self._model_call(test_batch, **call_kwargs), + dim=-1, + dtype=self.softmax_dtype, + ) + + return batch_size + + try: + batch_size = forward_batch() + except RuntimeError as e: + if "No executable batch size found" in str(e): + batch_size = 1 + else: + raise + + if self.world_size > 1: + # if multi-GPU, always take minimum over all selected batch sizes + max_rnk_bs = torch.tensor([batch_size], device=self.device) + gathered = ( + self.accelerator.gather(max_rnk_bs).cpu().detach().numpy().tolist() + ) + batch_size = min(gathered) + clear_torch_cache() + return batch_size + + clear_torch_cache() + return batch_size + + def tok_encode( + self, string: str, left_truncate_len=None, add_special_tokens=None + ) -> List[int]: + """ """ + # default for None - empty dict, use predefined tokenizer param + # used for all models except for CausalLM or predefined value + special_tokens_kwargs = {} + + # by default for CausalLM - false or self.add_bos_token is set + if add_special_tokens is None: + if self.backend == "causal": + special_tokens_kwargs = { + "add_special_tokens": False or self.add_bos_token + } + # otherwise the method explicitly defines the value + else: + special_tokens_kwargs = {"add_special_tokens": add_special_tokens} + + encoding = self.tokenizer.encode(string, **special_tokens_kwargs) + + # left-truncate the encoded context to be at most `left_truncate_len` tokens long + if left_truncate_len: + encoding = encoding[-left_truncate_len:] + + return encoding + + def tok_batch_encode( + self, + strings: List[str], + padding_side: str = "left", + left_truncate_len: int = None, + truncation: bool = False, + ) -> Tuple[torch.Tensor, torch.Tensor]: + # encode a batch of strings. converts to tensors and pads automatically, unlike tok_encode. + old_padding_side = self.tokenizer.padding_side + self.tokenizer.padding_side = padding_side + + add_special_tokens = {} + if self.backend == "causal": + add_special_tokens = {"add_special_tokens": False or self.add_bos_token} + + encoding = self.tokenizer( + strings, + truncation=truncation, + padding="longest", + return_tensors="pt", + **add_special_tokens, + ) + if left_truncate_len: + original_lengths = encoding["input_ids"].size(1) + if original_lengths > left_truncate_len: + eval_logger.warn( + f"Left truncation applied. Original sequence length was {original_lengths}, " + f"truncating to last {left_truncate_len} tokens. Some content will be lost.", + ) + encoding["input_ids"] = encoding["input_ids"][:, -left_truncate_len:] + encoding["attention_mask"] = encoding["attention_mask"][ + :, -left_truncate_len: + ] + self.tokenizer.padding_side = old_padding_side + + return encoding["input_ids"], encoding["attention_mask"] + + def tok_decode(self, tokens, skip_special_tokens=True): + return self.tokenizer.decode(tokens, skip_special_tokens=skip_special_tokens) + + def _model_call(self, inps, attn_mask=None, labels=None): + """ + :param inps: torch.Tensor + A torch tensor of shape [batch, (sequence_ctx + sequence_cont)] or of shape + [batch, sequence_ctx]. the size of sequence may vary from call to call + :param attn_mask: torch.Tensor, optional + A torch tensor of shape [batch, (sequence_ctx + sequence_cont)]. Only passed + (and must be passed) if self.AUTO_MODEL_CLASS is transformers.AutoModelForSeq2SeqLM + :param labels: torch.Tensor, optional + A torch tensor of shape [batch, (sequence_ctx + sequence_cont)]. Only passed + (and must be passed) if self.AUTO_MODEL_CLASS is transformers.AutoModelForSeq2SeqLM + :return + A torch tensor of shape [batch, sequence, vocab] with the + logits returned from the model's decoder + """ + with torch.no_grad(): + if attn_mask is not None or labels is not None: + assert attn_mask is not None and labels is not None + assert self.AUTO_MODEL_CLASS == transformers.AutoModelForSeq2SeqLM + return self.model( + input_ids=inps, attention_mask=attn_mask, labels=labels + ).logits + else: + assert self.AUTO_MODEL_CLASS in ( + transformers.AutoModelForCausalLM, + transformers.AutoModelForVision2Seq, + ) + return self.model(inps).logits + + def _model_generate(self, context, max_length, stop, **generation_kwargs): + # temperature = 0.0 if not set + # if do_sample is false and temp==0.0: + # remove temperature, as do_sample=False takes care of this + # and we don't want a warning from HF + generation_kwargs["temperature"] = generation_kwargs.get("temperature", 0.0) + do_sample = generation_kwargs.get("do_sample", None) + + # The temperature has to be a strictly positive float -- if it is 0.0, use greedy decoding strategies + if generation_kwargs.get("temperature") == 0.0 and do_sample is None: + generation_kwargs["do_sample"] = do_sample = False + + if do_sample is False and generation_kwargs.get("temperature") == 0.0: + generation_kwargs.pop("temperature") + # build stopping criteria + stopping_criteria = stop_sequences_criteria( + self.tokenizer, stop, context.shape[1], context.shape[0] + ) + return self.model.generate( + input_ids=context, + max_length=max_length, + stopping_criteria=stopping_criteria, + pad_token_id=self.tokenizer.pad_token_id, + use_cache=True, + **generation_kwargs, + ) + + def _select_cont_toks( + self, logits: torch.Tensor, contlen: int = None, inplen: int = None + ) -> torch.Tensor: + if self.backend == "causal": + assert contlen and inplen, ( + "Must pass input len and cont. len to select scored logits for causal LM" + ) + # discard right-padding. + # also discard the input/context tokens. we'll only score continuations. + logits = logits[inplen - contlen : inplen] + elif self.backend == "seq2seq": + assert contlen and not inplen, ( + "Selecting scored logits for Seq2SeqLM requires only cont. len" + ) + # only discard right-padding. + # the logits input to this fn only contain decoder-side tokens. + logits = logits[:contlen] + + return logits + + def loglikelihood_rolling( + self, requests: List[Instance], disable_tqdm: bool = False + ) -> List[float]: + adaptive_batch_size = None + if self.batch_size == "auto": + # using rolling window with maximum context + print("Passed argument batch_size = auto. Detecting largest batch size") + batch_size = self._detect_batch_size() + print(f"Determined Largest batch size: {batch_size}") + adaptive_batch_size = batch_size + + # First, collect all windows from all requests + all_windows = [] # List of (request_idx, window) tuples + request_window_counts = [] # Track number of windows per request + + for req_idx, (string,) in enumerate( + tqdm( + [req.args for req in requests], + disable=(disable_tqdm or (self.rank != 0)), + ) + ): + rolling_token_windows: List[Tuple[List[int], List[int]]] = list( + map( + utils.make_disjoint_window, + utils.get_rolling_token_windows( + token_list=self.tok_encode(string), + prefix_token=self.prefix_token_id, + max_seq_len=self.max_length, + context_len=1, + ), + ) + ) + + # TODO: Right now, we pass single EOT token to the Encoder and the full context to the decoder, in seq2seq case + windows = [(None,) + x for x in rolling_token_windows] + + # Store windows with their request index + all_windows.extend((req_idx, window) for window in windows) + request_window_counts.append(len(windows)) + + # Handle distributed case padding + pad_amnt = 0 + if self.world_size > 1: + mytensor = torch.tensor(len(all_windows), device=self.device) + gathered = self.accelerator.gather(mytensor).cpu().detach().numpy().tolist() + pad_amnt = max(gathered) - gathered[self.rank] + if pad_amnt > 0: + all_windows += pad_amnt * [all_windows[0]] + + all_nlls = [] + batch_size = adaptive_batch_size or self.batch_size + for i in range(0, len(all_windows), batch_size): + batch = all_windows[i : i + batch_size] + # Extract just the windows for processing, keeping track of request indices + batch_indices, batch_windows = zip(*batch) + + batch_nlls = self._loglikelihood_tokens( + requests=batch_windows, + disable_tqdm=False, + override_bs=len(batch_windows), + ) + # Store results with their request indices + all_nlls.extend(zip(batch_indices, batch_nlls)) + + # Remove padding if necessary + if (self.world_size > 1) and (pad_amnt > 0): + all_nlls = all_nlls[:-pad_amnt] + + # Reconstruct per-request loglikelihoods + loglikelihoods = [] + current_idx = 0 + for window_count in request_window_counts: + # Get all nlls for this request + request_nlls = all_nlls[current_idx : current_idx + window_count] + # Sum up the nlls for this request (discarding is_greedy) + request_total = sum(nll[0] for _, nll in request_nlls) + loglikelihoods.append(request_total) + current_idx += window_count + + string = requests[len(loglikelihoods) - 1].args[0] + self.cache_hook.add_partial( + "loglikelihood_rolling", (string,), request_total + ) + + return loglikelihoods + + def _batch_scheduler(self, pos, n_reordered_requests): + sched = pos // int(len(n_reordered_requests) / self.batch_schedule) + if sched in self.batch_sizes: + return self.batch_sizes[sched] + if (len(self.batch_sizes) > 1) and ( + self.batch_sizes[sched - 1] == self.max_batch_size + ): + # if previous batch size is already maximal, skip recomputation + self.batch_sizes[sched] = self.max_batch_size + return self.batch_sizes[sched] + print( + f"Passed argument batch_size = auto:{self.batch_schedule}. Detecting largest batch size" + ) + self.batch_sizes[sched] = self._detect_batch_size(n_reordered_requests, pos) + print(f"Determined largest batch size: {self.batch_sizes[sched]}") + return self.batch_sizes[sched] + + def _loglikelihood_tokens( + self, + requests: List[Tuple[Tuple[str, str], List[int], List[int]]], + disable_tqdm: bool = False, + override_bs: int = None, + ) -> List[Tuple[float, bool]]: + # TODO: implement some kind of efficient-request-middleware that lumps together requests with the same context + res = [] + + def _collate(req: Tuple[Tuple[str, str], List[int], List[int]]): + """Defines the key for the sorted method""" + # the negative sign on len(toks) sorts descending - this has a few advantages: + # - time estimates will always be over not underestimates, which is more useful for planning + # - to know the size of a batch when going through the list, you know the first one is always the batch + # padded context length. this is useful to simplify the batching logic and more importantly to make + # automatic adaptive batches much much easier to implement + # - any OOMs will happen right away rather than near the end + + toks = req[1] + req[2] + return -len(toks), tuple(toks) + + def _lookup_one_token_cont(req: Tuple[Tuple[str, str], List[int], List[int]]): + """Defines the key to group and lookup one-token continuations""" + # Use with group_by="contexts" (optional)" + # allows for the creation of a lookup, so we can reuse logits in case of one-token continuations. + # speeds up some multiple-choice tasks proportionally to the number of choices. + # groups requests by context+continuation[:-1] and infer on one request/group. + return req[-2] + req[-1][:-1] + + re_ord = Collator( + requests, + sort_fn=_collate, + group_by="contexts" + if self.backend == "causal" and self.logits_cache + else None, + group_fn=_lookup_one_token_cont, + ) + + # automatic (variable) batch size detection for vectorization + # pull longest context sample from request + n_reordered_requests = len(re_ord) + batch_size = ( + self.batch_size + if self.batch_size != "auto" + else override_bs + if override_bs is not None + else 0 + ) + batch_fn = ( + self._batch_scheduler + if self.batch_size == "auto" + and n_reordered_requests > 0 + and not override_bs + else None + ) + + chunks = re_ord.get_batched(n=batch_size, batch_fn=batch_fn) + pbar = tqdm( + total=len(requests), + disable=(disable_tqdm or (self.rank != 0)), + desc="Running loglikelihood requests", + ) + for chunk in chunks: + inps = [] + cont_toks_list = [] + inplens = [] + + conts = [] + encoder_attns = [] + + padding_len_inp = None + padding_len_cont = None + # because vectorizing is annoying, we first convert each (context, continuation) pair to padded + # tensors, then we pack them together into a batch, call the model, and then pick it all apart + # again because vectorizing is annoying + + for _, context_enc, continuation_enc in chunk: + # sanity check + assert len(context_enc) > 0 + assert len(continuation_enc) > 0 + assert len(continuation_enc) <= self.max_length + + # how this all works (illustrated on a causal decoder-only setup): + # CTX CONT + # inp 0 1 2 3|4 5 6 7 8 9 <- last token is deleted by inp[:, :-1] + # model \ \ + # logits 1 2 3|4 5 6 7 8 9 <- the ctx half gets tossed out by the + # cont_toks 4 5 6 7 8 9 [:, -len(continuation_enc):, :self.vocab_size] slice + + # when too long to fit in context, truncate from the left + if self.backend == "causal": + total_length = len(context_enc) + len(continuation_enc) + if total_length > self.max_length + 1: + eval_logger.warning( + f"Combined length of context ({len(context_enc)}) and continuation ({len(continuation_enc)}) " + f"exceeds model's maximum length ({self.max_length}). " + f"Truncating {total_length - self.max_length + 1} tokens from the left." + ) + inp = torch.tensor( + (context_enc + continuation_enc)[-(self.max_length + 1) :][:-1], + dtype=torch.long, + device=self.device, + ) + (inplen,) = inp.shape + elif self.backend == "seq2seq": + inp = torch.tensor( + (context_enc)[-self.max_length :], + dtype=torch.long, + device=self.device, + ) + (inplen,) = inp.shape + + # build encoder attn masks + encoder_attns.append(torch.ones_like(inp)) + + cont = torch.tensor( + (continuation_enc)[-self.max_length :], + # TODO: left-shift these? + # TODO: our code assumes we never end up truncating conts for either model type + dtype=torch.long, + device=self.device, + ) + (contlen,) = cont.shape + + conts.append(cont) + + padding_len_cont = ( + max(padding_len_cont, contlen) + if padding_len_cont is not None + else contlen + ) + + padding_len_inp = ( + max(padding_len_inp, inplen) + if padding_len_inp is not None + else inplen + ) + + inps.append(inp) # [1, inp_length] + cont_toks_list.append(continuation_enc) + inplens.append(inplen) + + # create encoder attn mask and batched conts, if seq2seq + call_kwargs = {} + if self.backend == "causal": + batched_inps = pad_and_concat( + padding_len_inp, inps, padding_side="right" + ) # [batch, padding_len_inp] + elif self.backend == "seq2seq": + # TODO: left-pad encoder inps and mask? + batched_inps = pad_and_concat( + padding_len_inp, inps + ) # [batch, padding_len_inp] + batched_conts = pad_and_concat( + padding_len_cont, conts + ) # [batch, padding_len_cont] + batched_encoder_mask = pad_and_concat( + padding_len_inp, encoder_attns + ) # [batch, padding_len_inp] + call_kwargs = { + "attn_mask": batched_encoder_mask, + "labels": batched_conts, + } + + multi_logits = F.log_softmax( + self._model_call(batched_inps, **call_kwargs), + dim=-1, + dtype=self.softmax_dtype, + ) # [batch, padding_length (inp or cont), vocab] + + for (request_str, ctx_tokens, _), logits, inplen, cont_toks in zip( + chunk, multi_logits, inplens, cont_toks_list + ): + # Slice to original seq length + contlen = len(cont_toks) + # take only logits in the continuation + # (discard context toks if decoder-only ; discard right-padding) + # also discards + checks for "virtual tokens" in the causal LM's input window + # from prompt/prefix tuning tokens, if applicable + ctx_len = ( + inplen + (logits.shape[0] - padding_len_inp) + if self.backend == "causal" + else None + ) + logits = self._select_cont_toks(logits, contlen=contlen, inplen=ctx_len) + logits = logits.unsqueeze(0) # [1, seq, vocab] + + # Check if per-token argmax is exactly equal to continuation + greedy_tokens = logits.argmax(dim=-1) + + # check for one-token continuation cache hits. + # noop in case group_by != "contexts" or no cache hit and returns the + # original args. Otherwise, expands the logits batch dimension and yields each + # batch along with matching continuation tokens and prompt strings. + # logits -> [1, seq, vocab] + for request_str, cont_toks, logits in re_ord.get_cache( + req_str=request_str, + cxt_toks=ctx_tokens, + cont_toks=cont_toks, + logits=logits, + ): + cont_toks = torch.tensor( + cont_toks, dtype=torch.long, device=self.device + ).unsqueeze(0) # [1, seq] + # Use trailing slice [-cont_toks.shape[1]:] to handle variable length cont_len (but same ctx+cont[:-1]). + # i.e. continuations can be sliced at diff points. Collator ensures we have sufficient greedy_tokens + # by choosing key with longest cont if group_by="contexts". + max_equal = ( + greedy_tokens[:, -cont_toks.shape[1] :] == cont_toks + ).all() + + # Obtain log-probs at the corresponding continuation token indices + # last_token_slice = logits[:, -1, :].squeeze(0).tolist() + logits = torch.gather(logits, 2, cont_toks.unsqueeze(-1)).squeeze( + -1 + ) # [1, seq] + + # Answer: (log prob, is-exact-match) + answer = (float(logits.sum()), bool(max_equal)) + + res.append(answer) + + if request_str is not None: + # special case: loglikelihood_rolling produces a number of loglikelihood requests + # all with cache key None. instead do add_partial on the per-example level + # in the loglikelihood_rolling() function for those. + self.cache_hook.add_partial( + "loglikelihood", request_str, answer + ) + pbar.update(1) + + pbar.close() + + return re_ord.get_original(res) + + def generate_until( + self, requests: List[Instance], disable_tqdm: bool = False + ) -> List[str]: + res = [] + + def _collate(req: Tuple[str, dict]): + """Defines the key for the sorted method""" + # the negative sign on len(toks) sorts descending - this has a few advantages: + # - time estimates will always be over not underestimates, which is more useful for planning + # - to know the size of a batch when going through the list, you know the first one is always the batch + # padded context length. this is useful to simplify the batching logic and more importantly to make + # automatic adaptive batches much much easier to implement + # - any OOMs will happen right away rather than near the end + toks = self.tok_encode(req[0]) + return -len(toks), req[0] + + pbar = tqdm( + total=len(requests), + disable=(disable_tqdm or (self.rank != 0)), + desc="Running generate_until requests", + ) + adaptive_batch_size = None + if self.batch_size == "auto": + # using rolling window with maximum context + print("Passed argument batch_size = auto. Detecting largest batch size") + batch_size = self._detect_batch_size() + print(f"Determined Largest batch size: {batch_size}") + adaptive_batch_size = batch_size + # for each different set of kwargs, we execute all requests, by batch. + batch_size = ( + self.batch_size + if self.batch_size != "auto" + else adaptive_batch_size + if adaptive_batch_size is not None + else 0 + ) + batch_fn = ( + self._batch_scheduler + if self.batch_size == "auto" and not adaptive_batch_size + else None + ) + + # we group requests by their generation_kwargs, + # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling + # in the same batch. + # group_fn=lambda x: x[1] -> x=(context, gen_kwargs) + re_ords = Collator( + [reg.args for reg in requests], + sort_fn=_collate, + group_by="gen_kwargs", + group_fn=lambda x: x[1], + ) + chunks = re_ords.get_batched(n=batch_size, batch_fn=batch_fn) + eos = self.tok_decode(self.eot_token_id, skip_special_tokens=False) + for chunk in chunks: + contexts, all_gen_kwargs = zip(*chunk) + # we assume all gen kwargs in the batch are the same + # this is safe to assume because the `grouper` object ensures it. + gen_kwargs = all_gen_kwargs[0] + # unpack our keyword arguments. + if isinstance(gen_kwargs, dict): + kwargs = copy.deepcopy(gen_kwargs) # edge case for repeats > 1 + # add EOS token to stop sequences + until = handle_stop_sequences(kwargs.pop("until", None), eos=eos) + else: + raise ValueError( + f"Expected `kwargs` to be of type `dict` but got {type(gen_kwargs)}" + ) + if "max_gen_toks" in kwargs.keys(): + max_gen_toks = kwargs.pop("max_gen_toks") + else: + max_gen_toks = self.max_gen_toks + + # set the max length in tokens of inputs ("context_enc") + if self.backend == "causal": + # max len for inputs = max length, minus room to generate the max new tokens + max_ctx_len = self.max_length - max_gen_toks + assert max_ctx_len > 0, ( + f"Invalid configuration: requested max tokens to generate ({max_gen_toks}) must be less than model's maximum sequence length ({self.max_length})." + ) + elif self.backend == "seq2seq": + # max len for inputs = encoder's whole max_length + max_ctx_len = self.max_length + + # encode, pad, and truncate contexts for this batch + context_enc, attn_masks = self.tok_batch_encode( + contexts, + left_truncate_len=max_ctx_len, + truncation=self.truncation, + ) + context_enc = context_enc.to(self.device) + attn_masks = attn_masks.to(self.device) + + if "max_length" not in kwargs: + kwargs["max_length"] = context_enc.shape[1] + max_gen_toks + + # perform batched generation + cont = self._model_generate( + context=context_enc, + attention_mask=attn_masks, + stop=until, + **kwargs, + ) + + cont_toks_list = cont.tolist() + for cont_toks, context in zip(cont_toks_list, contexts): + # discard context + left-padding toks if using causal decoder-only LM + if self.backend == "causal": + cont_toks = cont_toks[context_enc.shape[1] :] + + s = self.tok_decode(cont_toks) + + # use secondary stop seqs to cut off should-have-been-stopped content post-hoc + for term in until: + if len(term) > 0: + # ignore '' separator, + # for seq2seq case where self.tok_decode(self.eot_token_id) = '' + s = s.split(term)[0] + + res.append(s) + + self.cache_hook.add_partial("generate_until", (context, gen_kwargs), s) + pbar.update(1) + # reorder this group of results back to original unsorted form + res = re_ords.get_original(res) + + pbar.close() + + return res + + def apply_chat_template( + self, chat_history: List[Dict[str, str]], add_generation_prompt: bool = True + ) -> str: + """ + Method to apply a chat template to a list of chat history between user and model. + """ + try: + chat_templated = self.tokenizer.apply_chat_template( + chat_history, + tokenize=False, + add_generation_prompt=add_generation_prompt, + continue_final_message=not add_generation_prompt, + ) + except jinja2.exceptions.TemplateError: + eval_logger.warning( + "Failed to apply chat template. removing the system role in chat history." + ) + chat_history = [msg for msg in chat_history if msg["role"] != "system"] + chat_templated = self.tokenizer.apply_chat_template( + chat_history, + tokenize=False, + add_generation_prompt=add_generation_prompt, + continue_final_message=not add_generation_prompt, + ) + + return chat_templated + + def get_model_info(self) -> dict: + """ + Method to get Hugging Face model information for experiment reproducibility. + """ + + def get_model_num_params(model) -> int: + if hasattr(model, "num_parameters"): + return model.num_parameters() + if hasattr(model, "parameters"): + return sum(p.numel() for p in model.parameters()) + else: + return -1 + + def get_model_dtype(model) -> str: + if hasattr(model, "dtype"): + return model.dtype + else: + return "" + + def get_model_sha(pretrained: str, revision: str) -> str: + try: + model_info = HfApi().model_info(repo_id=pretrained, revision=revision) + return model_info.sha + except Exception as e: + eval_logger.debug( + f"Failed to get model SHA for {pretrained} at revision {revision}. Error: {e}" + ) + return "" + + model_info = { + "model_num_parameters": get_model_num_params(self._model), + "model_dtype": get_model_dtype(self._model), + "model_revision": self.revision, + "model_sha": get_model_sha(self.pretrained, self.revision), + } + if self.peft: + model_info["peft_sha"] = get_model_sha(self.peft, self.revision) + if self.delta: + model_info["delta_sha"] = get_model_sha(self.delta, self.revision) + return model_info diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/models/utils.py b/Prism/LLaDA/LLaDA_Prism/dllm_eval/models/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..e17fa224b22fbbef442c94e13d4f7c237d3c647d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/dllm_eval/models/utils.py @@ -0,0 +1,854 @@ +import collections +import fnmatch +import gc +import itertools +import logging +import time +from functools import wraps +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Iterable, + Iterator, + List, + Literal, + Optional, + Tuple, + Type, + Union, +) + +import torch +import transformers + + +eval_logger = logging.getLogger(__name__) + + +if TYPE_CHECKING: + from PIL import Image + from transformers import PreTrainedTokenizerBase + from transformers.configuration_utils import PretrainedConfig + + +def chunks(iter, n: int = 0, fn=None): + """ + Divides an iterable into chunks of specified size or based on a given function. + Useful for batching + + Parameters: + - iter: The input iterable to be divided into chunks. + - n: An integer representing the size of each chunk. Default is 0. + - fn: A function that takes the current index and the iterable as arguments and returns the size of the chunk. Default is None. + + Returns: + An iterator that yields chunks of the input iterable. + + Example usage: + ``` + data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + for chunk in chunks(data, 3): + print(chunk) + ``` + Output: + ``` + [1, 2, 3] + [4, 5, 6] + [7, 8, 9] + [10] + ``` + """ + arr = [] + for i, x in enumerate(iter): + arr.append(x) + if len(arr) == (fn(i, iter) if fn else n): + yield arr + arr = [] + + if arr: + yield arr + + +class MultiChoice: + def __init__(self, choices) -> None: + self.choices = choices + + # Simple wildcard support (linux filename patterns) + def __contains__(self, values) -> bool: + for value in values.split(","): + if len(fnmatch.filter(self.choices, value)) == 0: + eval_logger.info("Available tasks to choose:") + for choice in self.choices: + eval_logger.info(f" - {choice}") + raise ValueError("'{}' is not in task list".format(value)) + return True + + def __iter__(self) -> Iterator: + for choice in self.choices: + yield choice + + +class Grouper: + """ + takes an array `arr` and function `fn` and returns a dictionary + with keys fn(ob) for each ob in `arr` and with values `self.arr[key]` a list of all + objects in `arr` satisfying `key == fn(ob)`. + """ + + def __init__(self, arr, fn) -> None: + # self.orig_arr = arr + self.size = len(arr) + arr = list(enumerate(arr)) + + def group_return_dict(arr, fn): + res = collections.defaultdict(list) + + for ob in arr: + res[fn(ob)].append(ob) + return res + + arr = group_return_dict(arr, lambda x: fn(x[1])) + + # self.arr has format Dict[Tuple[int, ]] + self.arr = arr + self._grouped = None + + def get_grouped(self): + # return the contents but not indices for our grouped dict. + if self._grouped: + return self._grouped + grouped = {} + for key in self.arr.keys(): + # drop the index from each element of self.arr + grouped[key] = [y[1] for y in self.arr[key]] + self._grouped = grouped + return grouped + + def get_original(self, grouped_dict): + # take in a grouped dictionary with e.g. results for each key listed + # in the same order as the instances in `self.arr`, and + # return the results in the same (single list) order as `self.orig_arr`. + res = [None] * self.size + cov = [False] * self.size + # orig = [None] * self.size + + assert grouped_dict.keys() == self.arr.keys() + + for key in grouped_dict.keys(): + for (ind, _), v in zip(self.arr[key], grouped_dict[key]): + res[ind] = v + cov[ind] = True + # orig[ind] = _ + + assert all(cov) + # assert orig == self.orig_arr + + return res + + +def pad_and_concat( + max_length: int, + tensors: List[torch.Tensor], + padding_side: Literal["right", "left"] = "right", +): + """ + Method for padding a list of tensors given the maximum tensor + length in the batch. Used for batching inputs and continuations in + seq2seq models. + """ + assert padding_side == "left" or padding_side == "right", ( + f"Unrecognized padding type: '{padding_side}' not 'left' or 'right'" + ) + + for i, tensor in enumerate(tensors): + if len(tensor.shape) == 2: + tensor = tensor.squeeze(0) # squeeze, in case passed [1, seq] size + tensor_len = tensor.shape[0] + if tensor_len < max_length: + if padding_side == "right": + # right-pad + tensors[i] = torch.cat( + [ + tensor, # [seq] + torch.zeros( + max_length - tensor_len, + dtype=torch.long, + device=tensor.device, + ), # [padding_length - seq] + ], + dim=0, + ).unsqueeze(0) + else: + # left-pad + tensors[i] = torch.cat( + [ + torch.zeros( + max_length - tensor_len, + dtype=torch.long, + device=tensor.device, + ), # [padding_length - seq] + tensor, # [seq] + ], + dim=0, + ).unsqueeze(0) + else: + tensors[i] = tensor.unsqueeze(0) + + return torch.cat(tensors, dim=0) + + +def clear_torch_cache() -> None: + gc.collect() + torch.cuda.empty_cache() + + +def get_dtype(dtype: Union[str, torch.dtype]) -> torch.dtype: + """Converts `dtype` from `str` to torch.dtype when possible. Does not use an instantiated HF AutoConfig""" + if isinstance(dtype, str) and dtype != "auto": + # Convert `str` args torch dtype: `float16` -> `torch.float16` + _torch_dtype = getattr(torch, dtype) + else: + _torch_dtype = dtype + return _torch_dtype + + +class MultiTokenEOSCriteria(transformers.StoppingCriteria): + """Criteria to stop on the specified multi-token sequence.""" + + def __init__( + self, + sequence: str, + tokenizer: transformers.PreTrainedTokenizer, + initial_decoder_input_length: int, + batch_size: int, + ) -> None: + self.initial_decoder_input_length = initial_decoder_input_length + self.done_tracker = [False] * batch_size + self.sequence = sequence + self.sequence_ids = tokenizer.encode(sequence, add_special_tokens=False) + # print(sequence, self.sequence_ids) + # we look back for 2 more tokens than it takes to encode our stop sequence + # because tokenizers suck, and a model might generate `['\n', '\n']` but our `sequence` is `['\n\n']` + # and we don't want to mistakenly not stop a generation because our + # (string) stop sequence was output in a different tokenization + + # NOTE: there is a minor danger that this will end up looking back 2 tokens into the past, into the inputs to the model, + # and stopping generation immediately as a result. With only 2 extra tokens of lookback, this risk is minimized + # Additionally, in lookback_ids_batch we should prevent ever looking back into the inputs as described. + self.sequence_id_len = len(self.sequence_ids) + 2 + self.tokenizer = tokenizer + + def __call__(self, input_ids, scores, **kwargs) -> bool: + # For efficiency, we compare the last n tokens where n is the number of tokens in the stop_sequence + lookback_ids_batch = input_ids[:, self.initial_decoder_input_length :] + + lookback_ids_batch = lookback_ids_batch[:, -self.sequence_id_len :] + + lookback_tokens_batch = self.tokenizer.batch_decode(lookback_ids_batch) + + for i, done in enumerate(self.done_tracker): + if not done: + self.done_tracker[i] = self.sequence in lookback_tokens_batch[i] + return False not in self.done_tracker + + +def stop_sequences_criteria( + tokenizer: transformers.PreTrainedTokenizer, + stop_sequences: List[str], + initial_decoder_input_length: int, + batch_size: int, +) -> transformers.StoppingCriteriaList: + return transformers.StoppingCriteriaList( + [ + *[ + MultiTokenEOSCriteria( + sequence, tokenizer, initial_decoder_input_length, batch_size + ) + for sequence in stop_sequences + ], + ] + ) + + +def undistribute(iterable): + """ + Undoes https://more-itertools.readthedocs.io/en/stable/api.html#more_itertools.distribute . + + Re-interleaves results that have been split using more_itertools.distribute: + >>> group_1, group_2 = distribute(2, [1, 2, 3, 4, 5, 6]) + >>> list(group_1) + [1, 3, 5] + >>> list(group_2) + [2, 4, 6] + >>> undistribute([group_1, group_2]) + [1, 2, 3, 4, 5, 6] + + Handles non-uniform component lengths: + + >>> children = distribute(3, [1, 2, 3, 4, 5, 6, 7]) + >>> [list(c) for c in children] + [[1, 4, 7], [2, 5], [3, 6]] + >>> undistribute(children) + [1, 2, 3, 4, 5, 6, 7] + + Also handles when some iterables are empty: + + >>> children = distribute(5, [1, 2, 3]) + >>> [list(c) for c in children] + [[1], [2], [3], [], []] + >>> undistribute(children) + [1, 2, 3] + + """ + + return [ + x + for x in itertools.chain.from_iterable( + itertools.zip_longest(*[list(x) for x in iterable]) + ) + if x is not None + ] + + +def retry_on_specific_exceptions( + on_exceptions: List[Type[Exception]], + max_retries: Optional[int] = None, + backoff_time: float = 3.0, + backoff_multiplier: float = 1.5, + on_exception_callback: Optional[Callable[[Exception, float], Any]] = None, +): + """Retry on an LLM Provider's rate limit error with exponential backoff + For example, to use for OpenAI, do the following: + ``` + from openai import RateLimitError + + # Recommend specifying max_retries to avoid infinite loops! + @retry_on_specific_exceptions([RateLimitError], max_retries=3) + def completion(...): + # Wrap OpenAI completion function here + ... + ``` + """ + + def decorator(func: Callable): + @wraps(func) + def wrapper(*args, **kwargs): + sleep_time = backoff_time + attempt = 0 + while max_retries is None or attempt < max_retries: + try: + return func(*args, **kwargs) + except tuple(on_exceptions) as e: + if on_exception_callback is not None: + on_exception_callback(e, sleep_time) + time.sleep(sleep_time) + sleep_time *= backoff_multiplier + attempt += 1 + + return wrapper + + return decorator + + +class Collator: + """ + A class for reordering and batching elements of an array. + + This class allows for sorting an array based on a provided sorting function, grouping elements based on a grouping function, and generating batches from the sorted and grouped data. + + Objects of this class have the group_by attribute which determines the method for grouping + the data while batching it. Three options include "gen_kwargs", "contexts", or None: + If group_by == "gen_kwargs" then requests will be grouped by gen_kwargs + If group_by == "contexts" then requests will be grouped by context + cont[:-1] + If None then requests will just be reordered by length descending. + """ + + def __init__( + self, + arr: List, + sort_fn: Callable = lambda x: x, + group_fn: Callable = lambda x: x[1], + group_by: Union[Literal["gen_kwargs", "contexts"], None] = None, + ) -> None: + self._group_by = group_by + # 0 indices are enumerated indices. Apply functions to original arr. + self._sort_fn = lambda x: sort_fn(x[1]) + self._group_fn = lambda x: group_fn(x[1]) + self._reorder_indices: List = [] + self._size = len(arr) + self._arr_with_indices: Union[Dict, Tuple[Tuple[int, Any], ...]] = tuple( + enumerate(arr) + ) # [indices, (arr)] + if self._group_by == "contexts": + self._group_by_context() + elif self._group_by == "gen_kwargs": + self._group_by_index() + + def _group_by_index(self) -> None: + """Group the elements of a list based on their indices.""" + self._arr_with_indices = self.group( + self._arr_with_indices, fn=self._group_fn, group_by="gen_kwargs" + ) + + def _group_by_context(self) -> None: + """Group the array with indices by context.""" + self._arr_with_indices = self.group( + self._arr_with_indices, fn=self._group_fn, group_by="contexts" + ) + + def get_batched(self, n: int = 1, batch_fn: Optional[Callable] = None) -> Iterator: + """ + Generates and yields batches from the reordered array. The method of grouping and batching + depends on the parameter `group_by`. + If `group_by` is set to "gen_kwargs", it will batch the + re-ordered values with same gen_kwargs for each batch. + If `group_by` is "contexts", it caches the requests by context before batching. + If `group_by` is neither "gen_kwargs" nor "contexts", it yields the reordered array + + Parameters: + - n (int): The size of each batch. Defaults to 1. + - batch_fn ([Callable[[int, Iterable], int]] | None): A function to determine the size of + each batch. Optional, defaults to None. + + Returns: + Iterator: An iterator over batches of reordered elements grouped as per the `group_by` + attribute. + + Yields: + List of batched elements according to the `group_by` attribute. + """ + if self._group_by == "gen_kwargs": + for ( + key, + values, + ) in self._arr_with_indices.items(): # type: ignore + values = self._reorder(values) + batch = self.get_chunks(values, n=n, fn=batch_fn) + yield from batch + elif self._group_by == "contexts": + # Get one sample from each key. + # Select longest continuation per group to ensure sufficient context logits + values = self._reorder( + [ + max(value, key=lambda x: len(x[1][-1])) + for value in self._arr_with_indices.values() + ] + ) + batch = self.get_chunks(values, n=n, fn=batch_fn) + yield from batch + else: + values = self._reorder(self._arr_with_indices) # type: ignore + batch = self.get_chunks(values, n=n, fn=batch_fn) + yield from batch + + def get_cache( + self, + req_str: Tuple[str, str] = None, + cxt_toks: List[int] = None, + cont_toks: List[int] = None, + logits: torch.Tensor = None, + ) -> Iterator[Tuple[Tuple[str, str], List[int], torch.Tensor]]: + """ + Retrieves cached single-token continuations and their associated arguments, updating indices as necessary. + + The behavior of this function varies depending on how the `group_by` attribute is set: + + - When `group_by` is "contexts": + The function identifies single-token continuations by checking for keys that equate to + [context+continuation][-1] and logs the indices for re-ordering. + In this mode, this function can work in two scenarios: + + 1. Cache Hit - Single Match: + If a single matching context-continuation pair is found in the cache, + the function yields the original arguments. + + 2. Cache Hit - Multiple Matches: + If multiple matching context-continuation pairs are found in the cache, + the function expands the logits batch dimension to match the number of cache hits. + It updates the original requests and continuation tokens. + + - When `group_by` is not set to "contexts": + This method yields the original arguments, logits and continuation tokens, + without checking for one-token continuations. + + Parameters: + - req_str (tuple[str, str]): Original strings used for CachingLM. + - cxt_toks (list[int]): Full context tokens used for lookup. + - cont_toks (list[int]): Continuation tokens for which logits were generated. + - logits (torch.Tensor [1, seq_length, vocab_size]): Logits generated by the model given context and continuation keys. + + Yields: + - Iterator: + - req_str (tuple[str, str]): strings used for CachingLM. + - cont_toks (list[int]) : continuation tokens. + - logits (torch.Tensor [1, seq_length, vocab_size]): The original logits (repeated cache hit times) + """ + if self._group_by == "contexts": + cache_hit: List[ + Tuple[int, Tuple[Tuple[str, str], List[int], List[int]]] + ] = self._arr_with_indices.pop(tuple(cxt_toks + cont_toks[:-1])) + if (cache_size := len(cache_hit)) == 1: + self._reorder_indices.extend(x[0] for x in cache_hit) + yield req_str, cont_toks, logits + else: + # If we have matching requests then expand the batch dimension (no-op) and + # yield each along with its corresponding args. + multilogits = logits.expand(cache_size, -1, -1).chunk(cache_size) + indices, req_str, cont_toks = zip( + *[(x[0], x[1][0], x[-1][-1]) for x in cache_hit] + ) + self._reorder_indices.extend(indices) + for c_key, cont_tok, logit in zip(req_str, cont_toks, multilogits): + yield c_key, cont_tok, logit + else: + yield req_str, cont_toks, logits + + def _reorder(self, arr: Union[List, Tuple[Tuple[int, Any], ...]]) -> Iterator: + """ + Reorders the elements in the array based on the sorting function. + + Parameters: + - arr (list | tuple[tuple[int, Any], ...]]): The array or iterable to be reordered. + + Yields: + Iterator + """ + arr = sorted(arr, key=self._sort_fn) + if not self._group_by == "contexts": + # If grouped by contexts then indices will be set in get_cache() + self._reorder_indices.extend([x[0] for x in arr]) + yield from [x[1] for x in arr] + + def get_original(self, newarr: List) -> List: + """ + Restores the original order of elements from the reordered list. + + Parameters: + - newarr (list): The reordered array. + + Returns: + list: The array with elements restored to their original order. + """ + res = [None] * self._size + cov = [False] * self._size + + for ind, v in zip(self._reorder_indices, newarr): + res[ind] = v + cov[ind] = True + + assert all(cov) + + return res + + def __len__(self): + return self._size + + @staticmethod + def group( + arr: Iterable, + fn: Callable, + group_by: Literal["gen_kwargs", "contexts"] = "gen_kwargs", + ) -> dict: + """ + Groups elements of an iterable based on a provided function. + + + The `group_by` parameter determines the method of grouping. + If `group_by` is "contexts", the elements are grouped by [context + cont][:-1]. + If `group_by` is "gen_kwargs", the elements are grouped based on the gen_kwargs dict. + + Parameters: + - arr (Iterable): The iterable to be grouped. + - fn (Callable): The function to determine the grouping. + - values (bool): If True, returns the values of the group. Defaults to False. + + Returns: + Iterator: An iterable of grouped elements. + """ + res = collections.defaultdict(list) + for ob in arr: + # where ob == [context + cont] + if group_by == "contexts": + res[tuple(fn(ob))].append(ob) + else: + try: + hashable_dict = tuple( + ( + key, + tuple(value) + if isinstance(value, collections.abc.Iterable) + else value, + ) + for key, value in sorted(fn(ob).items()) + ) + res[hashable_dict].append(ob) + except (TypeError, AttributeError): + res[tuple(fn(ob))].append(ob) + return res + + @staticmethod + def get_chunks(_iter, n: int = 0, fn=None): + """ + Divides an iterable into chunks of specified size or based on a given function. + Useful for batching + + Parameters: + - iter: The input iterable to be divided into chunks. + - n: An integer representing the size of each chunk. Default is 0. + - fn: A function that takes the current index and the iterable as arguments and returns the size of the chunk. Default is None. + + Returns: + An iterator that yields chunks of the input iterable. + + Example usage: + ``` + data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + for chunk in chunks(data, 3): + print(chunk) + ``` + Output: + ``` + [1, 2, 3] + [4, 5, 6] + [7, 8, 9] + [10] + ``` + """ + arr = [] + _iter = tuple(_iter) + for i, x in enumerate(_iter): + arr.append(x) + if len(arr) == (fn(i, _iter) if fn else n): + yield arr + arr = [] + + if arr: + yield arr + + +def configure_pad_token( + tokenizer: "PreTrainedTokenizerBase", + model_config: Optional["PretrainedConfig"] = None, +) -> "PreTrainedTokenizerBase": + """ + This function checks if the (Hugging Face) tokenizer has a padding token and sets it if not present. + Some tokenizers require special handling. + + Args: + tokenizer: The tokenizer for which the padding token is to be handled. + model_config: The configuration of the model. Default is None. + + Returns: + The tokenizer after the padding token has been handled. + + Raises: + AssertionError: If the tokenizer is of type RWKVWorldTokenizer or Rwkv5Tokenizer and the padding token id is not 0. + """ + if tokenizer.pad_token: + pass + elif tokenizer.unk_token: + tokenizer.pad_token_id = tokenizer.unk_token_id + elif tokenizer.eos_token: + tokenizer.pad_token_id = tokenizer.eos_token_id + else: + # handle special cases + if model_config and getattr(model_config, "model_type", None) == "qwen": + # Qwen's trust_remote_code tokenizer does not allow for adding special tokens + tokenizer.pad_token = "<|endoftext|>" + elif ( + tokenizer.__class__.__name__ == "RWKVWorldTokenizer" + or tokenizer.__class__.__name__ == "Rwkv5Tokenizer" + ): + # The RWKV world tokenizer, does not allow for adding special tokens / setting the pad token (which is set as 0) + # The additional tokenizer name check is needed, as there exists rwkv4 models with neox tokenizer + # --- + # Note that the world tokenizer class name, might change in the future for the final huggingface merge + # https://github.com/huggingface/transformers/pull/26963 + assert tokenizer.pad_token_id == 0 + else: + tokenizer.add_special_tokens({"pad_token": "<|pad|>"}) + + return tokenizer + + +def replace_placeholders( + string: str, default_placeholder: str, image_token: str, max_images: int +): + """ + A utility function used for local multimodal models. It locates all `placeholder` string + occurrences in the given input `string_` and replaces the first `max_count` instances with + `replacement`, and all subsequent occurrences with the empty string. + + This is used to replace placeholder tags by model-specific image tokens like <|image_pad|> + and to allow for only the first `max_count` images to be passed to a model if desired. + + :param string: The original string containing placeholders. + :param default_placeholder: The placeholder text to be replaced. + :param image_token: The token to replace the placeholder with. + :param max_images: The maximum number of replacements to make. + :return: The string with placeholders replaced. + """ + count = 0 + result = [] + + parts = string.split(default_placeholder) + for part in parts[:-1]: # Iterate through all but the last part + result.append(part) + if count < max_images: + result.append(image_token) + count += 1 + elif default_placeholder != image_token: + result.append(default_placeholder) + + # Add the last part of the string + result.append(parts[-1]) + return "".join(result) + + +def flatten_image_list(images: List[List]): + """ + Takes in a list of lists of images, and returns a single list of all images in order. + Used for some multimodal models like Llava-1.5 which expects this flattened-list format for its image processor. + + :param images: A list of lists of PIL images. + :return: a list of PIL images, via concatenating all the sub-lists in order. + """ + return [image for image_list in images for image in image_list] + + +def handle_stop_sequences( + until: Union[str, List[str], None], eos: Optional[str] +) -> List[str]: + """Ensures that the `until` parameter is a list of stop sequences and includes the EOS token.""" + if isinstance(until, str): + until = [until] + elif until is None: + until = [] + elif not isinstance(until, list): + raise ValueError( + f"Expected `kwargs['until']` to be of type Union[str,list] but got {until}" + ) + + if eos is not None and eos not in until: + until.append(eos) + return until + + +def resize_image( + image: "Image.Image", + width: Optional[int] = None, + height: Optional[int] = None, + max_dimension: Optional[int] = None, + keep_aspect_ratio: bool = True, + resample_filter: Union[int, str] = "Image.BICUBIC", + min_width: int = 1, + min_height: int = 1, +) -> "Image.Image": + """ + Resizes a PIL Image object with flexible options. + + Args: + image: The PIL Image object to resize. + width: Target width in pixels. + height: Target height in pixels. + max_dimension: Maximum size for the longer dimension of the image. + keep_aspect_ratio: If True (default) and both width and height are provided, + the image is resized to fit within these dimensions while + maintaining its aspect ratio. If False, the image is stretched + to the exact width and height. + resample_filter: The resampling filter to use for resizing. + Defaults to Image.BICUBIC. + min_width: Minimum width for the resized image. Defaults to 1. + min_height: Minimum height for the resized image. Defaults to 1. + + Returns: + The resized PIL Image object. If no resize parameters are provided + or if the image already meets the criteria, the original image is returned. + + Order of precedence for resizing: + 1. If width AND height are provided: + - If keep_aspect_ratio is True: Fits image within bounds, preserving aspect ratio. + - If keep_aspect_ratio is False: Resizes to exact dimensions (may distort). + 2. Else if only width is provided: Calculates height proportionally. + 3. Else if only height is provided: Calculates width proportionally. + 4. Else if max_dimension is provided: Resizes the longest side to max_dimension + and scales the other side proportionally. + 5. If none of the above are provided, returns the original image. + """ + original_width, original_height = image.size + + # If no arguments are provided, return the original image + if width is None and height is None and max_dimension is None: + return image + + new_width = original_width + new_height = original_height + + if width is not None and height is not None: + # No resize needed if image is already smaller than target dimensions + if original_width <= width and original_height <= height: + return image + + if keep_aspect_ratio: + # Calculate the ratio to fit within the target dimensions + ratio = min(width / original_width, height / original_height) + new_width = int(original_width * ratio) + new_height = int(original_height * ratio) + else: + # Stretch to exact dimensions + new_width = width + new_height = height + elif width is not None: + # No resize needed if width is already smaller + if original_width <= width: + return image + # Calculate height proportionally + new_width = width + new_height = int((original_height / original_width) * new_width) + elif height is not None: + # No resize needed if height is already smaller + if original_height <= height: + return image + # Calculate width proportionally + new_height = height + new_width = int((original_width / original_height) * new_height) + elif max_dimension is not None: + # No resize needed if both dimensions are smaller than max_dimension + if max(original_height, original_width) <= max_dimension: + return image + + if original_width > original_height: + # Width is the longer side + new_width = max_dimension + new_height = int((original_height / original_width) * new_width) + else: + # Height is the longer side or sides are equal + new_height = max_dimension + new_width = int((original_width / original_height) * new_height) + + # Ensure dimensions are at least minimum values + new_width = max(min_width, new_width) + new_height = max(min_height, new_height) + + # Perform the resize operation with the calculated dimensions + return image.resize((new_width, new_height), resample_filter) + + +def truncate_tokens( + tokens: List[int], + max_length: int, + tokenizer: "PreTrainedTokenizerBase", + strategy: str = "left", +): + if strategy == "left": + return tokens[-max_length:] + elif strategy == "right": + return tokens[:max_length] + elif strategy == "middle": + # Truncate the middle of the sequence + left_length = max_length // 2 + right_length = max_length - left_length + return tokens[:left_length] + tokens[-right_length:] + return None diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/models/verifier.py b/Prism/LLaDA/LLaDA_Prism/dllm_eval/models/verifier.py new file mode 100644 index 0000000000000000000000000000000000000000..1a89831e5ee670740bc3833b09849115971545a2 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/dllm_eval/models/verifier.py @@ -0,0 +1,154 @@ +import torch +import logging +import ast +import re +import numpy as np +import textwrap + +logger = logging.getLogger(__name__) + +class CodeVerifier: + def __init__(self, model, tokenizer, device="cuda"): + self.model = model + self.tokenizer = tokenizer + self.device = device + + self.yes_ids, self.no_ids = [], [] + for t in ["Yes", " Yes", "YES"]: + ids = self.tokenizer.encode(t, add_special_tokens=False) + if len(ids) > 0: self.yes_ids.append(ids[-1]) + for t in ["No", " No", "NO"]: + ids = self.tokenizer.encode(t, add_special_tokens=False) + if len(ids) > 0: self.no_ids.append(ids[-1]) + + self.yes_ids = list(set(self.yes_ids)) + self.no_ids = list(set(self.no_ids)) + + def _extract_python_code(self, text): + text = text.strip() + match = re.search(r"```python\s*(.*?)```", text, re.DOTALL) + if match: return match.group(1) + match_generic = re.search(r"```\s*(.*?)```", text, re.DOTALL) + if match_generic: return match_generic.group(1) + return text + + def check_syntax(self, code_str): + clean_code = self._extract_python_code(code_str) + try: + if len(clean_code.strip()) < 5: return False + ast.parse(clean_code) + return True + except: + return False + + def compute_confidence(self, logits): + if logits is None: return 0.0 + probs = torch.softmax(logits, dim=-1) + max_probs, _ = torch.max(probs, dim=-1) + log_probs = torch.log(max_probs + 1e-10) + return torch.exp(torch.mean(log_probs)).item() + + def svf_score(self, prompt, code_str, task_type="code"): + + max_len = 2000 + if len(code_str) > max_len: + if task_type == "reasoning": + truncated_code = code_str[:500] + "\n...[truncated]...\n" + code_str[-(max_len-500):] + else: + truncated_code = code_str[-max_len:] + else: + truncated_code = code_str + + if task_type == "code": + prompt_template = f""" + You are an expert programming contest judge. Your task is to evaluate a generated solution for a given problem based on correctness, efficiency, and adherence to constraints. + + [Problem Statement] + {prompt} + [/Problem Statement] + + [Proposed Python Solution] + ```python + {truncated_code} + ``` + [/Proposed Python Solution] + + **Analysis Steps:** + 1. Correctness: Does the core algorithm correctly solve the problem? + 2. Efficiency: Is the time complexity acceptable for the given constraints? + 3. Edge Cases & Constraints: Does the code handle all rules and edge cases? + + **Conclusion**: Based on your analysis, is the solution likely to be fully correct? Answer with a single word: Yes or No. + **Answer:** """ + + elif task_type == "math": + prompt_template = f""" + You are an expert mathematician and competition judge. Your task is to evaluate a proposed mathematical solution for a given problem based on its logical rigor and accuracy. + + [Math Problem] + {prompt} + [/Math Problem] + + [Proposed Mathematical Solution] + {truncated_code} + [/Proposed Mathematical Solution] + + **Analysis Steps:** + 1. Reasoning Validity: Are the logical steps and mathematical properties applied correctly? + 2. Calculation Accuracy: Are the intermediate calculations or algebraic manipulations accurate? + 3. Goal Alignment: Does the current reasoning path directly lead toward the final answer required by the problem? + + **Conclusion**: Based on your analysis, is this solution path sound and likely to result in the correct final answer? Answer with a single word: Yes or No. + **Answer:** """ + + elif task_type == "reasoning": + prompt_template = f""" + You are an expert reading comprehension and faithfulness judge. Your task is to evaluate a generated answer based on the provided context and question. + + [Context and Question] + {prompt} + [/Context and Question] + + [Proposed Answer] + {truncated_code} + [/Proposed Answer] + + **Analysis Steps :** + 1. Faithfulness: Is the answer an exact, literal span from the context? + 2. Relevance: Does the answer directly address the specific question asked without hallucinating external information? + 3. Accuracy: Does the provided context strictly support this answer? + + **Conclusion**: Based on your analysis, is the answer fully faithful to the context and correct? Answer with a single word: Yes or No. + **Answer:** """ + + else: + prompt_template = f"Is the following answer correct?\nQuestion: {prompt}\nAnswer: {truncated_code}\nAnswer Yes or No.\nAnswer:" + + verify_text = textwrap.dedent(prompt_template).strip() + input_ids = self.tokenizer(verify_text, return_tensors="pt").input_ids.to(self.device) + + max_pos = getattr(self.model.config, "max_position_embeddings", + getattr(self.model.config, "n_positions", + getattr(self.model.config, "max_sequence_length", 20480))) + + if input_ids.shape[1] > max_pos - 16: + logger.warning("Verifier input is too long, truncating from the left.") + input_ids = input_ids[:, -(max_pos - 16):] + + with torch.no_grad(): + outputs = self.model(input_ids) + logits = outputs.logits[0, -1, :] + + yes_score = max((logits[i].item() for i in self.yes_ids if i < logits.shape[-1]), default=-float('inf')) + no_score = max((logits[i].item() for i in self.no_ids if i < logits.shape[-1]), default=-float('inf')) + + if yes_score == -float('inf') and no_score == -float('inf'): return 0.5 + + probs = torch.softmax(torch.tensor([yes_score, no_score]), dim=0) + return probs[0].item() + + def get_reward(self, prompt, code_str, mode="confidence", problem_data=None, current_logits=None, task_type="code"): + if mode == "svf": + return self.svf_score(prompt, code_str, task_type=task_type) + else: + return self.compute_confidence(current_logits) \ No newline at end of file diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/models/verifier_qwen7B.py b/Prism/LLaDA/LLaDA_Prism/dllm_eval/models/verifier_qwen7B.py new file mode 100644 index 0000000000000000000000000000000000000000..ed8677cbd9156bdfc1a456143664b2c1cd829f63 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/dllm_eval/models/verifier_qwen7B.py @@ -0,0 +1,166 @@ +import torch +import logging +import ast +import re +import numpy as np +import textwrap +from transformers import AutoModelForCausalLM, AutoTokenizer + +logger = logging.getLogger(__name__) + +class CodeVerifier: + def __init__(self, model, tokenizer, device="cuda"): + reward_model_path = "PATH_TO_YOUR_Qwen2-7B" + + self.model = model + self.tokenizer = tokenizer + self.device = device + + logger.info(f"Loading Qwen2-7B reward model from {reward_model_path}...") + + self.reward_tokenizer = AutoTokenizer.from_pretrained( + reward_model_path, + trust_remote_code=True + ) + + if self.reward_tokenizer.pad_token_id is None: + self.reward_tokenizer.pad_token_id = 151643 + self.reward_model = AutoModelForCausalLM.from_pretrained( + reward_model_path, + device_map="auto", + torch_dtype=torch.bfloat16, + trust_remote_code=True + ).eval() + + self.yes_ids, self.no_ids = [], [] + for t in ["Yes", " Yes", "YES"]: + ids = self.reward_tokenizer.encode(t, add_special_tokens=False) + if len(ids) > 0: self.yes_ids.append(ids[-1]) + for t in ["No", " No", "NO"]: + ids = self.reward_tokenizer.encode(t, add_special_tokens=False) + if len(ids) > 0: self.no_ids.append(ids[-1]) + + self.yes_ids = list(set(self.yes_ids)) + self.no_ids = list(set(self.no_ids)) + + def _extract_python_code(self, text): + text = text.strip() + match = re.search(r"```python\s*(.*?)```", text, re.DOTALL) + if match: return match.group(1) + match_generic = re.search(r"```\s*(.*?)```", text, re.DOTALL) + if match_generic: return match_generic.group(1) + return text + + def check_syntax(self, code_str): + clean_code = self._extract_python_code(code_str) + try: + if len(clean_code.strip()) < 5: return False + ast.parse(clean_code) + return True + except: + return False + + def compute_confidence(self, logits): + if logits is None: return 0.0 + probs = torch.softmax(logits, dim=-1) + max_probs, _ = torch.max(probs, dim=-1) + log_probs = torch.log(max_probs + 1e-10) + return torch.exp(torch.mean(log_probs)).item() + + def svf_score(self, prompt, code_str, task_type="code"): + max_char_len = 4000 + if len(code_str) > max_char_len: + if task_type == "reasoning": + truncated_code = code_str[:1000] + "\n...[truncated]...\n" + code_str[-(max_char_len-1000):] + else: + truncated_code = code_str[-max_char_len:] + else: + truncated_code = code_str + + if task_type == "code": + prompt_template = f""" + You are an expert programming contest judge. Your task is to evaluate a generated solution for a given problem based on correctness, efficiency, and adherence to constraints. + + [Problem Statement] + {prompt} + [/Problem Statement] + + [Proposed Python Solution] + ```python + {truncated_code} + ``` + [/Proposed Python Solution] + + **Analysis Steps:** + 1. Correctness: Does the core algorithm correctly solve the problem? + 2. Efficiency: Is the time complexity acceptable for the given constraints? + 3. Edge Cases & Constraints: Does the code handle all rules and edge cases? + + **Conclusion**: Based on your analysis, is the solution likely to be fully correct? Answer with a single word: Yes or No. + **Answer:** """ + elif task_type == "math": + prompt_template = f""" + You are an expert mathematician and competition judge. Your task is to evaluate a proposed mathematical solution for a given problem based on its logical rigor and accuracy. + + [Math Problem] + {prompt} + [/Math Problem] + + [Proposed Mathematical Solution] + {truncated_code} + [/Proposed Mathematical Solution] + + **Analysis Steps:** + 1. Reasoning Validity: Are the logical steps and mathematical properties applied correctly? + 2. Calculation Accuracy: Are the intermediate calculations or algebraic manipulations accurate? + 3. Goal Alignment: Does the current reasoning path directly lead toward the final answer required by the problem? + + **Conclusion**: Based on your analysis, is this solution path sound and likely to result in the correct final answer? Answer with a single word: Yes or No. + **Answer:** """ + elif task_type == "reasoning": + prompt_template = f""" + You are an expert reading comprehension and faithfulness judge. Your task is to evaluate a generated answer based on the provided context and question. + + [Context and Question] + {prompt} + [/Context and Question] + + [Proposed Answer] + {truncated_code} + [/Proposed Answer] + + **Analysis Steps :** + 1. Faithfulness: Is the answer an exact, literal span from the context? + 2. Relevance: Does the answer directly address the specific question asked without hallucinating external information? + 3. Accuracy: Does the provided context strictly support this answer? + + **Conclusion**: Based on your analysis, is the answer fully faithful to the context and correct? Answer with a single word: Yes or No. + **Answer:** """ + else: + prompt_template = f"Is the following answer correct?\nQuestion: {prompt}\nAnswer: {truncated_code}\nAnswer Yes or No.\nAnswer:" + + verify_text = textwrap.dedent(prompt_template).strip() + input_ids = self.reward_tokenizer(verify_text, return_tensors="pt").input_ids.to(self.reward_model.device) + + max_pos = getattr(self.reward_model.config, "max_position_embeddings", 8192) + + if input_ids.shape[1] > max_pos - 10: + input_ids = input_ids[:, -(max_pos - 10):] + + with torch.no_grad(): + outputs = self.reward_model(input_ids) + logits = outputs.logits[0, -1, :] + + yes_score = max((logits[i].item() for i in self.yes_ids if i < logits.shape[-1]), default=-float('inf')) + no_score = max((logits[i].item() for i in self.no_ids if i < logits.shape[-1]), default=-float('inf')) + + if yes_score == -float('inf') and no_score == -float('inf'): return 0.5 + + probs = torch.softmax(torch.tensor([yes_score, no_score]), dim=0) + return probs[0].item() + + def get_reward(self, prompt, code_str, mode="confidence", problem_data=None, current_logits=None, task_type="code"): + if mode == "svf": + return self.svf_score(prompt, code_str, task_type=task_type) + else: + return self.compute_confidence(current_logits) \ No newline at end of file diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/models/verifier_qwen8B.py b/Prism/LLaDA/LLaDA_Prism/dllm_eval/models/verifier_qwen8B.py new file mode 100644 index 0000000000000000000000000000000000000000..a11ca4b893b258546712ea15c69d21767a88b575 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/dllm_eval/models/verifier_qwen8B.py @@ -0,0 +1,166 @@ +import torch +import logging +import ast +import re +import numpy as np +import textwrap +from transformers import AutoModelForCausalLM, AutoTokenizer + +logger = logging.getLogger(__name__) + +class CodeVerifier: + def __init__(self, model, tokenizer, device="cuda"): + self.model = model + self.tokenizer = tokenizer + self.device = device + + reward_model_path = "PATH_TO_YOUR_Qwen3-8B" + logger.info(f"Loading Qwen3-8B as reward model from {reward_model_path}...") + + self.reward_tokenizer = AutoTokenizer.from_pretrained( + reward_model_path, + trust_remote_code=True + ) + + if self.reward_tokenizer.pad_token_id is None: + self.reward_tokenizer.pad_token_id = 151643 + + self.reward_model = AutoModelForCausalLM.from_pretrained( + reward_model_path, + device_map="auto", + torch_dtype=torch.bfloat16, + trust_remote_code=True + ).eval() + + self.yes_ids, self.no_ids = [], [] + for t in ["Yes", " Yes", "YES"]: + ids = self.reward_tokenizer.encode(t, add_special_tokens=False) + if len(ids) > 0: self.yes_ids.append(ids[-1]) + for t in ["No", " No", "NO"]: + ids = self.reward_tokenizer.encode(t, add_special_tokens=False) + if len(ids) > 0: self.no_ids.append(ids[-1]) + + self.yes_ids = list(set(self.yes_ids)) + self.no_ids = list(set(self.no_ids)) + + def _extract_python_code(self, text): + text = text.strip() + match = re.search(r"```python\s*(.*?)```", text, re.DOTALL) + if match: return match.group(1) + match_generic = re.search(r"```\s*(.*?)```", text, re.DOTALL) + if match_generic: return match_generic.group(1) + return text + + def check_syntax(self, code_str): + clean_code = self._extract_python_code(code_str) + try: + if len(clean_code.strip()) < 5: return False + ast.parse(clean_code) + return True + except: + return False + + def compute_confidence(self, logits): + if logits is None: return 0.0 + probs = torch.softmax(logits, dim=-1) + max_probs, _ = torch.max(probs, dim=-1) + log_probs = torch.log(max_probs + 1e-10) + return torch.exp(torch.mean(log_probs)).item() + + def svf_score(self, prompt, code_str, task_type="code"): + max_char_len = 4000 + if len(code_str) > max_char_len: + if task_type == "reasoning": + truncated_code = code_str[:1000] + "\n...[truncated]...\n" + code_str[-(max_char_len-1000):] + else: + truncated_code = code_str[-max_char_len:] + else: + truncated_code = code_str + + if task_type == "code": + prompt_template = f""" + You are an expert programming contest judge. Your task is to evaluate a generated solution for a given problem based on correctness, efficiency, and adherence to constraints. + + [Problem Statement] + {prompt} + [/Problem Statement] + + [Proposed Python Solution] + ```python + {truncated_code} + ``` + [/Proposed Python Solution] + + **Analysis Steps:** + 1. Correctness: Does the core algorithm correctly solve the problem? + 2. Efficiency: Is the time complexity acceptable for the given constraints? + 3. Edge Cases & Constraints: Does the code handle all rules and edge cases? + + **Conclusion**: Based on your analysis, is the solution likely to be fully correct? Answer with a single word: Yes or No. + **Answer:** """ + elif task_type == "math": + prompt_template = f""" + You are an expert mathematician and competition judge. Your task is to evaluate a proposed mathematical solution for a given problem based on its logical rigor and accuracy. + + [Math Problem] + {prompt} + [/Math Problem] + + [Proposed Mathematical Solution] + {truncated_code} + [/Proposed Mathematical Solution] + + **Analysis Steps:** + 1. Reasoning Validity: Are the logical steps and mathematical properties applied correctly? + 2. Calculation Accuracy: Are the intermediate calculations or algebraic manipulations accurate? + 3. Goal Alignment: Does the current reasoning path directly lead toward the final answer required by the problem? + + **Conclusion**: Based on your analysis, is this solution path sound and likely to result in the correct final answer? Answer with a single word: Yes or No. + **Answer:** """ + elif task_type == "reasoning": + prompt_template = f""" + You are an expert reading comprehension and faithfulness judge. Your task is to evaluate a generated answer based on the provided context and question. + + [Context and Question] + {prompt} + [/Context and Question] + + [Proposed Answer] + {truncated_code} + [/Proposed Answer] + + **Analysis Steps :** + 1. Faithfulness: Is the answer an exact, literal span from the context? + 2. Relevance: Does the answer directly address the specific question asked without hallucinating external information? + 3. Accuracy: Does the provided context strictly support this answer? + + **Conclusion**: Based on your analysis, is the answer fully faithful to the context and correct? Answer with a single word: Yes or No. + **Answer:** """ + else: + prompt_template = f"Is the following answer correct?\nQuestion: {prompt}\nAnswer: {truncated_code}\nAnswer Yes or No.\nAnswer:" + + verify_text = textwrap.dedent(prompt_template).strip() + + input_ids = self.reward_tokenizer(verify_text, return_tensors="pt").input_ids.to(self.reward_model.device) + + max_pos = getattr(self.reward_model.config, "max_position_embeddings", 40960) + if input_ids.shape[1] > max_pos - 10: + input_ids = input_ids[:, -(max_pos - 10):] + + with torch.no_grad(): + outputs = self.reward_model(input_ids) + logits = outputs.logits[0, -1, :] + + yes_score = max((logits[i].item() for i in self.yes_ids if i < logits.shape[-1]), default=-float('inf')) + no_score = max((logits[i].item() for i in self.no_ids if i < logits.shape[-1]), default=-float('inf')) + + if yes_score == -float('inf') and no_score == -float('inf'): return 0.5 + + probs = torch.softmax(torch.tensor([yes_score, no_score]), dim=0) + return probs[0].item() + + def get_reward(self, prompt, code_str, mode="confidence", problem_data=None, current_logits=None, task_type="code"): + if mode == "svf": + return self.svf_score(prompt, code_str, task_type=task_type) + else: + return self.compute_confidence(current_logits) \ No newline at end of file diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/prompts/__init__.py b/Prism/LLaDA/LLaDA_Prism/dllm_eval/prompts/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a0c2ce897dcde522ac82d0cbe0e06db1e02b1b72 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/dllm_eval/prompts/__init__.py @@ -0,0 +1,128 @@ +import ast +import logging +import os +from typing import Dict + +from dllm_eval import utils + + +eval_logger = logging.getLogger(__name__) + +# Prompt library. +# Stores prompts in a dictionary indexed by 2 levels: +# prompt category name, and prompt name. +# This allows us to access prompts +PROMPT_REGISTRY: Dict[str, Dict[str, str]] = { + "qa-basic": { + "question-newline-answer": "Question: {{question}}\nAnswer:", + "q-newline-a": "Q: {{question}}\nA:", + }, +} + + +def get_prompt(prompt_id: str, dataset_name: str = None, subset_name: str = None): + # unpack prompt name + category_name, prompt_name = prompt_id.split(":") + if subset_name is None: + dataset_full_name = dataset_name + else: + dataset_full_name = f"{dataset_name}-{subset_name}" + eval_logger.info(f"Loading prompt from {category_name} for {dataset_full_name}") + if category_name == "promptsource": + try: + from promptsource.templates import DatasetTemplates + except ModuleNotFoundError as exception: + raise type(exception)( + "Tried to load a Promptsource template, but promptsource is not installed ", + "please install promptsource via pip install lm-eval[promptsource] or pip install -e .[promptsource]", + ) + try: + if subset_name is None: + prompts = DatasetTemplates(dataset_name=dataset_name) + else: + prompts = DatasetTemplates( + dataset_name=dataset_name, subset_name=subset_name + ) + except Exception: + raise ValueError(f"{dataset_name} and {subset_name} not found") + if prompt_name in prompts.all_template_names: + return prompts[prompt_name] + else: + raise ValueError( + f"{prompt_name} not in prompt list {prompts.all_template_names}" + ) + elif ".yaml" in category_name: + import yaml + + with open(category_name, "rb") as file: + prompt_yaml_file = yaml.full_load(file) + + prompt_string = prompt_yaml_file["prompts"][prompt_name] + return PromptString(prompt_string) + else: + try: + return PROMPT_REGISTRY[category_name][prompt_name] + except Exception: + raise ValueError( + f"expected only a single `:` as separator between \ + prompt category and name, but got `{prompt_id}` instead" + ) + + +def load_prompt_list( + use_prompt: str, dataset_name=None, subset_name=None, yaml_path=None, **kwargs +): + category_name, prompt_name = use_prompt.split(":") + + if category_name == "promptsource": + from promptsource.templates import DatasetTemplates + + if subset_name is None: + prompts = DatasetTemplates(dataset_name=dataset_name) + else: + prompts = DatasetTemplates( + dataset_name=dataset_name, subset_name=subset_name + ) + + prompt_list = utils.pattern_match(prompt_name, prompts.all_template_names) + + elif ".yaml" in category_name: + import yaml + + if yaml_path is not None: + category_name = os.path.realpath(os.path.join(yaml_path, category_name)) + + with open(category_name, "rb") as file: + prompt_yaml_file = yaml.full_load(file) + + prompt_list = utils.pattern_match( + prompt_name, prompt_yaml_file["prompts"].keys() + ) + + # category_name, *prompt_name = use_prompt.split(":") + # TODO allow to multiple prompt naming + # if len(prompt_name) > 1: + # prompt_list = [] + # for prompt in prompt_name: + # prompt_list.append(utils.pattern_match(prompt_name, prompts.all_template_names)) + # else: + # prompt_list = utils.pattern_match(prompt_name, prompts.all_template_names) + return [":".join([category_name, prompt]) for prompt in prompt_list] + + +class PromptString: + def __init__(self, prompt_string): + self.prompt_string = prompt_string + + def apply(self, doc): + doc_to_text = self.prompt_string["doc_to_text"] + doc_to_target = self.prompt_string["doc_to_target"] + + # TODO need a way to process doc_to_choice + if "doc_to_choice" in self.prompt_string: + raise NotImplementedError("Not yet implemented to accept doc_to_choice") + + text_string = utils.apply_template(doc_to_text, doc) + target_string = utils.apply_template(doc_to_target, doc) + + return [text_string, target_string] diff --git a/Prism/LLaDA/LLaDA_Prism/dllm_eval/tasks/__init__.py b/Prism/LLaDA/LLaDA_Prism/dllm_eval/tasks/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..73d896452e06c2cc2909c290de70dcf87b0c6f90 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/dllm_eval/tasks/__init__.py @@ -0,0 +1,670 @@ +import collections +import inspect +import logging +import os +from functools import partial +from typing import Dict, List, Mapping, Optional, Union + +from dllm_eval import utils +from dllm_eval.api.group import ConfigurableGroup, GroupConfig +from dllm_eval.api.task import ConfigurableTask, Task +from dllm_eval.evaluator_utils import get_subtask_list + + +GROUP_ONLY_KEYS = list(GroupConfig().to_dict().keys()) + +eval_logger = logging.getLogger(__name__) + + +class TaskManager: + """TaskManager indexes all tasks from the default `dllm_eval/tasks/` + and an optional directory if provided. + + """ + + def __init__( + self, + verbosity: Optional[str] = None, + include_path: Optional[Union[str, List]] = None, + include_defaults: bool = True, + metadata: Optional[dict] = None, + ) -> None: + if verbosity is not None: + utils.setup_logging(verbosity) + self.include_path = include_path + self.metadata = metadata + self._task_index = self.initialize_tasks( + include_path=include_path, include_defaults=include_defaults + ) + self._all_tasks = sorted(list(self._task_index.keys())) + + self._all_groups = sorted( + [x for x in self._all_tasks if self._task_index[x]["type"] == "group"] + ) + self._all_subtasks = sorted( + [ + x + for x in self._all_tasks + if self._task_index[x]["type"] in ["task", "python_task"] + ] + ) + self._all_tags = sorted( + [x for x in self._all_tasks if self._task_index[x]["type"] == "tag"] + ) + + self.task_group_map = collections.defaultdict(list) + + def initialize_tasks( + self, + include_path: Optional[Union[str, List]] = None, + include_defaults: bool = True, + ) -> dict[str, dict]: + """Creates a dictionary of tasks indexes. + + :param include_path: Union[str, List] = None + An additional path to be searched for tasks recursively. + Can provide more than one such path as a list. + :param include_defaults: bool = True + If set to false, default tasks (those in dllm_eval/tasks/) are not indexed. + return + Dictionary of task names as key and task metadata + """ + if include_defaults: + all_paths = [os.path.dirname(os.path.abspath(__file__)) + "/"] + else: + all_paths = [] + if include_path is not None: + if isinstance(include_path, str): + include_path = [include_path] + all_paths.extend(include_path) + + task_index = {} + for task_dir in all_paths: + tasks = self._get_task_and_group(task_dir) + task_index = {**tasks, **task_index} + + return task_index + + @property + def all_tasks(self): + return self._all_tasks + + @property + def all_groups(self): + return self._all_groups + + @property + def all_subtasks(self): + return self._all_subtasks + + @property + def all_tags(self): + return self._all_tags + + @property + def task_index(self): + return self._task_index + + def list_all_tasks( + self, list_groups=True, list_tags=True, list_subtasks=True + ) -> str: + from pytablewriter import MarkdownTableWriter + + def sanitize_path(path): + # don't print full path if we are within the dllm_eval/tasks dir ! + # if we aren't though, provide the full path. + if "dllm_eval/tasks/" in path: + return "dllm_eval/tasks/" + path.split("dllm_eval/tasks/")[-1] + else: + return path + + group_table = MarkdownTableWriter() + group_table.headers = ["Group", "Config Location"] + gt_values = [] + for g in self.all_groups: + path = self.task_index[g]["yaml_path"] + if path == -1: + path = "---" + else: + path = sanitize_path(path) + gt_values.append([g, path]) + group_table.value_matrix = gt_values + + tag_table = MarkdownTableWriter() + tag_table.headers = ["Tag"] + tag_table.value_matrix = [[t] for t in self.all_tags] + + subtask_table = MarkdownTableWriter() + subtask_table.headers = ["Task", "Config Location", "Output Type"] + st_values = [] + for t in self.all_subtasks: + path = self.task_index[t]["yaml_path"] + + output_type = "" + + # read the yaml file to determine the output type + if path != -1: + config = utils.load_yaml_config(path, mode="simple") + if "output_type" in config: + output_type = config["output_type"] + elif ( + "include" in config + ): # if no output type, check if there is an include with an output type + include_path = path.split("/")[:-1] + config["include"] + include_config = utils.load_yaml_config(include_path, mode="simple") + if "output_type" in include_config: + output_type = include_config["output_type"] + + if path == -1: + path = "---" + else: + path = sanitize_path(path) + st_values.append([t, path, output_type]) + subtask_table.value_matrix = st_values + + result = "\n" + if list_groups: + result += group_table.dumps() + "\n\n" + if list_tags: + result += tag_table.dumps() + "\n\n" + if list_subtasks: + result += subtask_table.dumps() + "\n\n" + return result + + def match_tasks(self, task_list: list[str]) -> list[str]: + return utils.pattern_match(task_list, self.all_tasks) + + def _name_is_registered(self, name: str) -> bool: + if name in self.all_tasks: + return True + return False + + def _name_is_task(self, name: str) -> bool: + if self._name_is_registered(name) and (self.task_index[name]["type"] == "task"): + return True + return False + + def _name_is_tag(self, name: str) -> bool: + if self._name_is_registered(name) and (self.task_index[name]["type"] == "tag"): + return True + return False + + def _name_is_group(self, name: str) -> bool: + if self._name_is_registered(name) and ( + self.task_index[name]["type"] == "group" + ): + return True + return False + + def _name_is_python_task(self, name: str) -> bool: + if self._name_is_registered(name) and ( + self.task_index[name]["type"] == "python_task" + ): + return True + return False + + def _config_is_task(self, config: dict) -> bool: + if ("task" in config) and isinstance(config["task"], str): + return True + return False + + def _config_is_group(self, config: dict) -> bool: + if ("task" in config) and isinstance(config["task"], list): + return True + return False + + def _config_is_python_task(self, config: dict) -> bool: + if "class" in config: + return True + return False + + def _get_yaml_path(self, name: str): + if name not in self.task_index: + raise ValueError + return self.task_index[name]["yaml_path"] + + def _get_config(self, name): + if name not in self.task_index: + raise ValueError + yaml_path = self._get_yaml_path(name) + if yaml_path == -1: + return {} + else: + return utils.load_yaml_config(yaml_path, mode="full") + + def _get_tasklist(self, name): + if self._name_is_task(name): + raise ValueError + return self.task_index[name]["task"] + + def _process_alias(self, config, group=None): + # If the group is not the same as the original + # group which the group alias was intended for, + # Set the group_alias to None instead. + if ("group_alias" in config) and ("group" in config) and group is not None: + if config["group"] != group: + config["group_alias"] = None + return config + + def _class_has_config_in_constructor(self, cls): + constructor = getattr(cls, "__init__", None) + return ( + "config" in inspect.signature(constructor).parameters + if constructor + else False + ) + + def _load_individual_task_or_group( + self, + name_or_config: Optional[Union[str, dict]] = None, + parent_name: Optional[str] = None, + update_config: Optional[dict] = None, + ) -> Mapping: + def _load_task(config, task): + if "include" in config: + config = { + **utils.load_yaml_config( + yaml_path=None, + yaml_config={"include": config.pop("include")}, + mode="full", + ), + **config, + } + if self._config_is_python_task(config): + if self._class_has_config_in_constructor(config["class"]): + task_object = config["class"](config=config) + else: + task_object = config["class"]() + if isinstance(task_object, ConfigurableTask): + # very scuffed: set task name here. TODO: fixme? + task_object.config.task = task + else: + if self.metadata is not None: + config["metadata"] = config.get("metadata", {}) | self.metadata + else: + config["metadata"] = config.get("metadata", {}) + task_object = ConfigurableTask(config=config) + + return {task: task_object} + + def _get_group_and_subtask_from_config( + config: dict, + ) -> tuple[ConfigurableGroup, list[str]]: + if self.metadata is not None: + config["metadata"] = config.get("metadata", {}) | self.metadata + group_name = ConfigurableGroup(config=config) + subtask_list = [] + for task in group_name.config["task"]: + if isinstance(task, str) and self._name_is_tag(task): + subtask_list.extend(self._get_tasklist(task)) + else: + subtask_list.append(task) + return group_name, subtask_list + + def _process_group_config( + config: dict, update_config: dict = None + ) -> tuple[dict, dict]: + if update_config is not None: + config = {**config, **update_config} + _update_config = { + k: v for k, v in config.items() if k not in GROUP_ONLY_KEYS + } + if not bool(_update_config): + _update_config = None + + group_config = {k: v for k, v in config.items() if k in GROUP_ONLY_KEYS} + return group_config, _update_config + + if isinstance(name_or_config, str): + if update_config is not None: + # Process name_or_config as a dict instead + name_or_config = {"task": name_or_config, **update_config} + elif self._name_is_task(name_or_config) or self._name_is_python_task( + name_or_config + ): + task_config = self._get_config(name_or_config) + return _load_task(task_config, task=name_or_config) + else: + subtask_list = self._get_tasklist(name_or_config) + if subtask_list == -1: + group_config = self._get_config(name_or_config) + group_config, update_config = _process_group_config(group_config) + group_name, subtask_list = _get_group_and_subtask_from_config( + group_config + ) + else: + if self._name_is_tag(name_or_config): + fn = partial( + self._load_individual_task_or_group, + update_config=name_or_config + if isinstance(name_or_config, dict) + else None, + ) + return dict( + collections.ChainMap(*map(fn, reversed(subtask_list))) + ) + else: + group_name = ConfigurableGroup( + config={"group": name_or_config, "task": subtask_list} + ) + + if isinstance(name_or_config, dict): + if self._config_is_task(name_or_config): + name = name_or_config.pop("task") + if update_config is not None: + name_or_config = {**name_or_config, **update_config} + # If the name is registered as a group + if self._name_is_group(name): + group_config = self._get_config(name) + + group_config, update_config = _process_group_config( + group_config, name_or_config + ) + group_name, subtask_list = _get_group_and_subtask_from_config( + group_config + ) + elif self._name_is_tag(name): + subtask_list = self._get_tasklist(name) + fn = partial( + self._load_individual_task_or_group, + update_config=name_or_config, + ) + return dict(collections.ChainMap(*map(fn, reversed(subtask_list)))) + else: + if self._name_is_registered(name): + base_task_config = self._get_config(name) + + # Check if this is a duplicate. + if parent_name is not None: + num_duplicate = len( + list( + filter( + lambda x: x.startswith(name), + self.task_group_map[parent_name], + ) + ) + ) + if num_duplicate > 0: + name = f"{name}-{num_duplicate}" + self.task_group_map[parent_name].append(name) + + task_config = { + **base_task_config, + **name_or_config, + } + else: + task_config = name_or_config + return _load_task(task_config, task=name) + else: + group_config, update_config = _process_group_config(name_or_config) + group_name, subtask_list = _get_group_and_subtask_from_config( + group_config + ) + + fn = partial( + self._load_individual_task_or_group, + parent_name=group_name, + update_config=update_config, + ) + return { + group_name: dict(collections.ChainMap(*map(fn, reversed(subtask_list)))) + } + + def load_task_or_group(self, task_list: Optional[Union[str, list]] = None) -> dict: + """Loads a dictionary of task objects from a list + + :param task_list: Union[str, list] = None + Single string or list of string of task names to be loaded + + :return + Dictionary of task objects + """ + if isinstance(task_list, str): + task_list = [task_list] + + all_loaded_tasks = dict( + collections.ChainMap( + *map( + lambda task: self._load_individual_task_or_group(task), + task_list, + ) + ) + ) + return all_loaded_tasks + + def load_config(self, config: Dict): + return self._load_individual_task_or_group(config) + + def _get_task_and_group(self, task_dir: str): + """Creates a dictionary of tasks index with the following metadata, + - `type`, that can be either `task`, `python_task`, `group` or `tags`. + `task` refer to regular task configs, `python_task` are special + yaml files that only consists of `task` and `class` parameters. + `group` are group configs. `tags` are labels that can be assigned + to tasks to assist in sorting and calling tasks of certain themes. + - `yaml_path`, path to the yaml file. If the entry is a `group` that + was configured through a task config, the yaml_path will be -1 + and all subtasks will be listed in `task` (see below) + - `task`, reserved for entries with `type` as `group`. This will list + all subtasks. When a group config is created (as opposed to task + config having `group` parameter set), this will be set to -1 to + avoid recursive indexing. The whole list of subtasks will be loaded + at evaluation. + + :param task_dir: str + A directory to check for tasks + + :return + Dictionary of task names as key and task metadata + """ + + def _populate_tags_and_groups(config, task, tasks_and_groups, print_info): + # TODO: remove group in next release + if "tag" in config: + attr_list = config["tag"] + if isinstance(attr_list, str): + attr_list = [attr_list] + + for tag in attr_list: + if tag not in tasks_and_groups: + tasks_and_groups[tag] = { + "type": "tag", + "task": [task], + "yaml_path": -1, + } + elif tasks_and_groups[tag]["type"] != "tag": + eval_logger.info( + f"The tag '{tag}' is already registered as a group, this tag will not be registered. " + "This may affect tasks you want to call." + ) + break + else: + tasks_and_groups[tag]["task"].append(task) + + # TODO: remove group in next release + print_info = True + ignore_dirs = [ + "__pycache__", + ".ipynb_checkpoints", + ] + tasks_and_groups = collections.defaultdict() + for root, dirs, file_list in os.walk(task_dir): + dirs[:] = [d for d in dirs if d not in ignore_dirs] + for f in file_list: + if f.endswith(".yaml"): + yaml_path = os.path.join(root, f) + print(yaml_path) + config = utils.load_yaml_config(yaml_path, mode="simple") + if self._config_is_python_task(config): + # This is a python class config + task = config["task"] + tasks_and_groups[task] = { + "type": "python_task", + "yaml_path": yaml_path, + } + _populate_tags_and_groups( + config, task, tasks_and_groups, print_info + ) + elif self._config_is_group(config): + # This is a group config + tasks_and_groups[config["group"]] = { + "type": "group", + "task": -1, # This signals that + # we don't need to know + # the task list for indexing + # as it can be loaded + # when called. + "yaml_path": yaml_path, + } + + # # Registered the level 1 tasks from a group config + # for config in config["task"]: + # if isinstance(config, dict) and self._config_is_task(config): + # task = config["task"] + # tasks_and_groups[task] = { + # "type": "task", + # "yaml_path": yaml_path, + # } + + elif self._config_is_task(config): + # This is a task config + task = config["task"] + tasks_and_groups[task] = { + "type": "task", + "yaml_path": yaml_path, + } + _populate_tags_and_groups( + config, task, tasks_and_groups, print_info + ) + else: + eval_logger.debug(f"File {f} in {root} could not be loaded") + + return tasks_and_groups + + +def get_task_name_from_config(task_config: Dict[str, str]) -> str: + if "task" in task_config: + return task_config["task"] + if "dataset_name" in task_config: + return "{dataset_path}_{dataset_name}".format(**task_config) + else: + return "{dataset_path}".format(**task_config) + + +def get_task_name_from_object(task_object): + if hasattr(task_object, "config"): + return task_object._config["task"] + + # TODO: scrap this + # this gives a mechanism for non-registered tasks to have a custom name anyways when reporting + return ( + task_object.EVAL_HARNESS_NAME + if hasattr(task_object, "EVAL_HARNESS_NAME") + else type(task_object).__name__ + ) + + +def _check_duplicates(task_dict: dict) -> None: + """helper function solely used in validating get_task_dict output. + Takes the output of dllm_eval.evaluator_utils.get_subtask_list and + returns a list of all leaf subtasks contained within, and errors if any such leaf subtasks are + "oversubscribed" to several disjoint groups. + """ + subtask_names = [] + for key, value in task_dict.items(): + subtask_names.extend(value) + + duplicate_tasks = { + task_name for task_name in subtask_names if subtask_names.count(task_name) > 1 + } + + # locate the potentially problematic groups that seem to 'compete' for constituent subtasks + competing_groups = [ + group + for group in task_dict.keys() + if len(set(task_dict[group]).intersection(duplicate_tasks)) > 0 + ] + + if len(duplicate_tasks) > 0: + raise ValueError( + f"Found 1 or more tasks while trying to call get_task_dict() that were members of more than 1 called group: {list(duplicate_tasks)}. Offending groups: {competing_groups}. Please call groups which overlap their constituent tasks in separate evaluation runs." + ) + + +def get_task_dict( + task_name_list: Union[str, List[Union[str, Dict, Task]]], + task_manager: Optional[TaskManager] = None, +): + """Creates a dictionary of task objects from either a name of task, config, or prepared Task object. + + :param task_name_list: List[Union[str, Dict, Task]] + Name of model or LM object, see dllm_eval.models.get_model + :param task_manager: TaskManager = None + A TaskManager object that stores indexed tasks. If not set, + task_manager will load one. This should be set by the user + if there are additional paths that want to be included + via `include_path` + + :return + Dictionary of task objects + """ + + task_name_from_string_dict = {} + task_name_from_config_dict = {} + task_name_from_object_dict = {} + + if isinstance(task_name_list, str): + task_name_list = [task_name_list] + elif isinstance(task_name_list, list): + if not all([isinstance(task, (str, dict, Task)) for task in task_name_list]): + raise TypeError( + "Expected all list items to be of types 'str', 'dict', or 'Task', but at least one entry did not match." + ) + else: + raise TypeError( + f"Expected a 'str' or 'list' but received {type(task_name_list)}." + ) + + string_task_name_list = [task for task in task_name_list if isinstance(task, str)] + others_task_name_list = [ + task for task in task_name_list if not isinstance(task, str) + ] + if len(string_task_name_list) > 0: + if task_manager is None: + task_manager = TaskManager() + + task_name_from_string_dict = task_manager.load_task_or_group( + string_task_name_list + ) + + for task_element in others_task_name_list: + if isinstance(task_element, dict): + task_name_from_config_dict = { + **task_name_from_config_dict, + **task_manager.load_config(config=task_element), + } + + elif isinstance(task_element, Task): + task_name_from_object_dict = { + **task_name_from_object_dict, + get_task_name_from_object(task_element): task_element, + } + + if not set(task_name_from_string_dict.keys()).isdisjoint( + set(task_name_from_object_dict.keys()) + ): + raise ValueError + + final_task_dict = { + **task_name_from_string_dict, + **task_name_from_config_dict, + **task_name_from_object_dict, + } + + # behavior can get odd if one tries to invoke several groups that "compete" for the same task. + # (notably, because one could request several num_fewshot values at once in GroupConfig overrides for the subtask + # and we'd be unsure which to use and report.) + # we explicitly check and error in this case. + _check_duplicates(get_subtask_list(final_task_dict)) + + return final_task_dict