Spaces:

anonymous8
/

Rapid-Textual-Adversarial-Defense

Running

Rapid-Textual-Adversarial-Defense / textattack /attack_args.py

anonymous8/RPD-Demo

initial commit

4943752 over 1 year ago

No virus

35.1 kB

	"""
	AttackArgs Class
	================
	"""

	from dataclasses import dataclass, field
	import json
	import os
	import sys
	import time
	from typing import Dict, Optional

	import textattack
	from textattack.shared.utils import ARGS_SPLIT_TOKEN, load_module_from_file

	from .attack import Attack
	from .dataset_args import DatasetArgs
	from .model_args import ModelArgs

	ATTACK_RECIPE_NAMES = {
	"alzantot": "textattack.attack_recipes.GeneticAlgorithmAlzantot2018",
	"bae": "textattack.attack_recipes.BAEGarg2019",
	"bert-attack": "textattack.attack_recipes.BERTAttackLi2020",
	"faster-alzantot": "textattack.attack_recipes.FasterGeneticAlgorithmJia2019",
	"deepwordbug": "textattack.attack_recipes.DeepWordBugGao2018",
	"hotflip": "textattack.attack_recipes.HotFlipEbrahimi2017",
	"input-reduction": "textattack.attack_recipes.InputReductionFeng2018",
	"kuleshov": "textattack.attack_recipes.Kuleshov2017",
	"morpheus": "textattack.attack_recipes.MorpheusTan2020",
	"seq2sick": "textattack.attack_recipes.Seq2SickCheng2018BlackBox",
	"textbugger": "textattack.attack_recipes.TextBuggerLi2018",
	"textfooler": "textattack.attack_recipes.TextFoolerJin2019",
	"pwws": "textattack.attack_recipes.PWWSRen2019",
	"iga": "textattack.attack_recipes.IGAWang2019",
	"pruthi": "textattack.attack_recipes.Pruthi2019",
	"pso": "textattack.attack_recipes.PSOZang2020",
	"checklist": "textattack.attack_recipes.CheckList2020",
	"clare": "textattack.attack_recipes.CLARE2020",
	"a2t": "textattack.attack_recipes.A2TYoo2021",
	}


	BLACK_BOX_TRANSFORMATION_CLASS_NAMES = {
	"random-synonym-insertion": "textattack.transformations.RandomSynonymInsertion",
	"word-deletion": "textattack.transformations.WordDeletion",
	"word-swap-embedding": "textattack.transformations.WordSwapEmbedding",
	"word-swap-homoglyph": "textattack.transformations.WordSwapHomoglyphSwap",
	"word-swap-inflections": "textattack.transformations.WordSwapInflections",
	"word-swap-neighboring-char-swap": "textattack.transformations.WordSwapNeighboringCharacterSwap",
	"word-swap-random-char-deletion": "textattack.transformations.WordSwapRandomCharacterDeletion",
	"word-swap-random-char-insertion": "textattack.transformations.WordSwapRandomCharacterInsertion",
	"word-swap-random-char-substitution": "textattack.transformations.WordSwapRandomCharacterSubstitution",
	"word-swap-wordnet": "textattack.transformations.WordSwapWordNet",
	"word-swap-masked-lm": "textattack.transformations.WordSwapMaskedLM",
	"word-swap-hownet": "textattack.transformations.WordSwapHowNet",
	"word-swap-qwerty": "textattack.transformations.WordSwapQWERTY",
	}


	WHITE_BOX_TRANSFORMATION_CLASS_NAMES = {
	"word-swap-gradient": "textattack.transformations.WordSwapGradientBased"
	}


	CONSTRAINT_CLASS_NAMES = {
	#
	# Semantics constraints
	#
	"embedding": "textattack.constraints.semantics.WordEmbeddingDistance",
	"bert": "textattack.constraints.semantics.sentence_encoders.BERT",
	"infer-sent": "textattack.constraints.semantics.sentence_encoders.InferSent",
	"thought-vector": "textattack.constraints.semantics.sentence_encoders.ThoughtVector",
	"use": "textattack.constraints.semantics.sentence_encoders.UniversalSentenceEncoder",
	"muse": "textattack.constraints.semantics.sentence_encoders.MultilingualUniversalSentenceEncoder",
	"bert-score": "textattack.constraints.semantics.BERTScore",
	#
	# Grammaticality constraints
	#
	"lang-tool": "textattack.constraints.grammaticality.LanguageTool",
	"part-of-speech": "textattack.constraints.grammaticality.PartOfSpeech",
	"goog-lm": "textattack.constraints.grammaticality.language_models.GoogleLanguageModel",
	"gpt2": "textattack.constraints.grammaticality.language_models.GPT2",
	"learning-to-write": "textattack.constraints.grammaticality.language_models.LearningToWriteLanguageModel",
	"cola": "textattack.constraints.grammaticality.COLA",
	#
	# Overlap constraints
	#
	"bleu": "textattack.constraints.overlap.BLEU",
	"chrf": "textattack.constraints.overlap.chrF",
	"edit-distance": "textattack.constraints.overlap.LevenshteinEditDistance",
	"meteor": "textattack.constraints.overlap.METEOR",
	"max-words-perturbed": "textattack.constraints.overlap.MaxWordsPerturbed",
	#
	# Pre-transformation constraints
	#
	"repeat": "textattack.constraints.pre_transformation.RepeatModification",
	"stopword": "textattack.constraints.pre_transformation.StopwordModification",
	"max-word-index": "textattack.constraints.pre_transformation.MaxWordIndexModification",
	}


	SEARCH_METHOD_CLASS_NAMES = {
	"beam-search": "textattack.search_methods.BeamSearch",
	"greedy": "textattack.search_methods.GreedySearch",
	"ga-word": "textattack.search_methods.GeneticAlgorithm",
	"greedy-word-wir": "textattack.search_methods.GreedyWordSwapWIR",
	"pso": "textattack.search_methods.ParticleSwarmOptimization",
	}


	GOAL_FUNCTION_CLASS_NAMES = {
	#
	# Classification goal functions
	#
	"targeted-classification": "textattack.goal_functions.classification.TargetedClassification",
	"untargeted-classification": "textattack.goal_functions.classification.UntargetedClassification",
	"input-reduction": "textattack.goal_functions.classification.InputReduction",
	#
	# Text goal functions
	#
	"minimize-bleu": "textattack.goal_functions.text.MinimizeBleu",
	"non-overlapping-output": "textattack.goal_functions.text.NonOverlappingOutput",
	"text-to-text": "textattack.goal_functions.text.TextToTextGoalFunction",
	}


	@dataclass
	class AttackArgs:
	"""Attack arguments to be passed to :class:`~textattack.Attacker`.

	Args:
	num_examples (:obj:`int`, 'optional`, defaults to :obj:`10`):
	The number of examples to attack. :obj:`-1` for entire dataset.
	num_successful_examples (:obj:`int`, `optional`, defaults to :obj:`None`):
	The number of successful adversarial examples we want. This is different from :obj:`num_examples`
	as :obj:`num_examples` only cares about attacking `N` samples while :obj:`num_successful_examples` aims to keep attacking
	until we have `N` successful cases.

	.. note::
	If set, this argument overrides `num_examples` argument.
	num_examples_offset (:obj: `int`, `optional`, defaults to :obj:`0`):
	The offset index to start at in the dataset.
	attack_n (:obj:`bool`, `optional`, defaults to :obj:`False`):
	Whether to run attack until total of `N` examples have been attacked (and not skipped).
	shuffle (:obj:`bool`, `optional`, defaults to :obj:`False`):
	If :obj:`True`, we randomly shuffle the dataset before attacking. However, this avoids actually shuffling
	the dataset internally and opts for shuffling the list of indices of examples we want to attack. This means
	:obj:`shuffle` can now be used with checkpoint saving.
	query_budget (:obj:`int`, `optional`, defaults to :obj:`None`):
	The maximum number of model queries allowed per example attacked.
	If not set, we use the query budget set in the :class:`~textattack.goal_functions.GoalFunction` object (which by default is :obj:`float("inf")`).

	.. note::
	Setting this overwrites the query budget set in :class:`~textattack.goal_functions.GoalFunction` object.
	checkpoint_interval (:obj:`int`, `optional`, defaults to :obj:`None`):
	If set, checkpoint will be saved after attacking every `N` examples. If :obj:`None` is passed, no checkpoints will be saved.
	checkpoint_dir (:obj:`str`, `optional`, defaults to :obj:`"checkpoints"`):
	The directory to save checkpoint files.
	random_seed (:obj:`int`, `optional`, defaults to :obj:`765`):
	Random seed for reproducibility.
	parallel (:obj:`False`, `optional`, defaults to :obj:`False`):
	If :obj:`True`, run attack using multiple CPUs/GPUs.
	num_workers_per_device (:obj:`int`, `optional`, defaults to :obj:`1`):
	Number of worker processes to run per device in parallel mode (i.e. :obj:`parallel=True`). For example, if you are using GPUs and :obj:`num_workers_per_device=2`,
	then 2 processes will be running in each GPU.
	log_to_txt (:obj:`str`, `optional`, defaults to :obj:`None`):
	If set, save attack logs as a `.txt` file to the directory specified by this argument.
	If the last part of the provided path ends with `.txt` extension, it is assumed to the desired path of the log file.
	log_to_csv (:obj:`str`, `optional`, defaults to :obj:`None`):
	If set, save attack logs as a CSV file to the directory specified by this argument.
	If the last part of the provided path ends with `.csv` extension, it is assumed to the desired path of the log file.
	csv_coloring_style (:obj:`str`, `optional`, defaults to :obj:`"file"`):
	Method for choosing how to mark perturbed parts of the text. Options are :obj:`"file"`, :obj:`"plain"`, and :obj:`"html"`.
	:obj:`"file"` wraps perturbed parts with double brackets :obj:`[[ <text> ]]` while :obj:`"plain"` does not mark the text in any way.
	log_to_visdom (:obj:`dict`, `optional`, defaults to :obj:`None`):
	If set, Visdom logger is used with the provided dictionary passed as a keyword arguments to :class:`~textattack.loggers.VisdomLogger`.
	Pass in empty dictionary to use default arguments. For custom logger, the dictionary should have the following
	three keys and their corresponding values: :obj:`"env", "port", "hostname"`.
	log_to_wandb(:obj:`dict`, `optional`, defaults to :obj:`None`):
	If set, WandB logger is used with the provided dictionary passed as a keyword arguments to :class:`~textattack.loggers.WeightsAndBiasesLogger`.
	Pass in empty dictionary to use default arguments. For custom logger, the dictionary should have the following
	key and its corresponding value: :obj:`"project"`.
	disable_stdout (:obj:`bool`, `optional`, defaults to :obj:`False`):
	Disable displaying individual attack results to stdout.
	silent (:obj:`bool`, `optional`, defaults to :obj:`False`):
	Disable all logging (except for errors). This is stronger than :obj:`disable_stdout`.
	enable_advance_metrics (:obj:`bool`, `optional`, defaults to :obj:`False`):
	Enable calculation and display of optional advance post-hoc metrics like perplexity, grammar errors, etc.
	"""

	num_examples: int = 10
	num_successful_examples: int = None
	num_examples_offset: int = 0
	attack_n: bool = False
	shuffle: bool = False
	query_budget: int = None
	checkpoint_interval: int = None
	checkpoint_dir: str = "checkpoints"
	random_seed: int = 765 # equivalent to sum((ord(c) for c in "TEXTATTACK"))
	parallel: bool = False
	num_workers_per_device: int = 1
	log_to_txt: str = None
	log_to_csv: str = None
	log_summary_to_json: str = None
	csv_coloring_style: str = "file"
	log_to_visdom: dict = None
	log_to_wandb: dict = None
	disable_stdout: bool = False
	silent: bool = False
	enable_advance_metrics: bool = False
	metrics: Optional[Dict] = None

	def __post_init__(self):
	if self.num_successful_examples:
	self.num_examples = None
	if self.num_examples:
	assert (
	self.num_examples >= 0 or self.num_examples == -1
	), "`num_examples` must be greater than or equal to 0 or equal to -1."
	if self.num_successful_examples:
	assert (
	self.num_successful_examples >= 0
	), "`num_examples` must be greater than or equal to 0."

	if self.query_budget:
	assert self.query_budget > 0, "`query_budget` must be greater than 0."

	if self.checkpoint_interval:
	assert (
	self.checkpoint_interval > 0
	), "`checkpoint_interval` must be greater than 0."

	assert (
	self.num_workers_per_device > 0
	), "`num_workers_per_device` must be greater than 0."

	@classmethod
	def _add_parser_args(cls, parser):
	"""Add listed args to command line parser."""
	default_obj = cls()
	num_ex_group = parser.add_mutually_exclusive_group(required=False)
	num_ex_group.add_argument(
	"--num-examples",
	"-n",
	type=int,
	default=default_obj.num_examples,
	help="The number of examples to process, -1 for entire dataset.",
	)
	num_ex_group.add_argument(
	"--num-successful-examples",
	type=int,
	default=default_obj.num_successful_examples,
	help="The number of successful adversarial examples we want.",
	)
	parser.add_argument(
	"--num-examples-offset",
	"-o",
	type=int,
	required=False,
	default=default_obj.num_examples_offset,
	help="The offset to start at in the dataset.",
	)
	parser.add_argument(
	"--query-budget",
	"-q",
	type=int,
	default=default_obj.query_budget,
	help="The maximum number of model queries allowed per example attacked. Setting this overwrites the query budget set in `GoalFunction` object.",
	)
	parser.add_argument(
	"--shuffle",
	action="store_true",
	default=default_obj.shuffle,
	help="If `True`, shuffle the samples before we attack the dataset. Default is False.",
	)
	parser.add_argument(
	"--attack-n",
	action="store_true",
	default=default_obj.attack_n,
	help="Whether to run attack until `n` examples have been attacked (not skipped).",
	)
	parser.add_argument(
	"--checkpoint-dir",
	required=False,
	type=str,
	default=default_obj.checkpoint_dir,
	help="The directory to save checkpoint files.",
	)
	parser.add_argument(
	"--checkpoint-interval",
	required=False,
	type=int,
	default=default_obj.checkpoint_interval,
	help="If set, checkpoint will be saved after attacking every N examples. If not set, no checkpoints will be saved.",
	)
	parser.add_argument(
	"--random-seed",
	default=default_obj.random_seed,
	type=int,
	help="Random seed for reproducibility.",
	)
	parser.add_argument(
	"--parallel",
	action="store_true",
	default=default_obj.parallel,
	help="Run attack using multiple GPUs.",
	)
	parser.add_argument(
	"--num-workers-per-device",
	default=default_obj.num_workers_per_device,
	type=int,
	help="Number of worker processes to run per device.",
	)
	parser.add_argument(
	"--log-to-txt",
	nargs="?",
	default=default_obj.log_to_txt,
	const="",
	type=str,
	help="Path to which to save attack logs as a text file. Set this argument if you want to save text logs. "
	"If the last part of the path ends with `.txt` extension, the path is assumed to path for output file.",
	)
	parser.add_argument(
	"--log-to-csv",
	nargs="?",
	default=default_obj.log_to_csv,
	const="",
	type=str,
	help="Path to which to save attack logs as a CSV file. Set this argument if you want to save CSV logs. "
	"If the last part of the path ends with `.csv` extension, the path is assumed to path for output file.",
	)
	parser.add_argument(
	"--log-summary-to-json",
	nargs="?",
	default=default_obj.log_summary_to_json,
	const="",
	type=str,
	help="Path to which to save attack summary as a JSON file. Set this argument if you want to save attack results summary in a JSON. "
	"If the last part of the path ends with `.json` extension, the path is assumed to path for output file.",
	)
	parser.add_argument(
	"--csv-coloring-style",
	default=default_obj.csv_coloring_style,
	type=str,
	help='Method for choosing how to mark perturbed parts of the text in CSV logs. Options are "file" and "plain". '
	'"file" wraps text with double brackets `[[ <text> ]]` while "plain" does not mark any text. Default is "file".',
	)
	parser.add_argument(
	"--log-to-visdom",
	nargs="?",
	default=None,
	const='{"env": "main", "port": 8097, "hostname": "localhost"}',
	type=json.loads,
	help="Set this argument if you want to log attacks to Visdom. The dictionary should have the following "
	'three keys and their corresponding values: `"env", "port", "hostname"`. '
	'Example for command line use: `--log-to-visdom {"env": "main", "port": 8097, "hostname": "localhost"}`.',
	)
	parser.add_argument(
	"--log-to-wandb",
	nargs="?",
	default=None,
	const='{"project": "textattack"}',
	type=json.loads,
	help="Set this argument if you want to log attacks to WandB. The dictionary should have the following "
	'key and its corresponding value: `"project"`. '
	'Example for command line use: `--log-to-wandb {"project": "textattack"}`.',
	)
	parser.add_argument(
	"--disable-stdout",
	action="store_true",
	default=default_obj.disable_stdout,
	help="Disable logging attack results to stdout",
	)
	parser.add_argument(
	"--silent",
	action="store_true",
	default=default_obj.silent,
	help="Disable all logging",
	)
	parser.add_argument(
	"--enable-advance-metrics",
	action="store_true",
	default=default_obj.enable_advance_metrics,
	help="Enable calculation and display of optional advance post-hoc metrics like perplexity, USE distance, etc.",
	)

	return parser

	@classmethod
	def create_loggers_from_args(cls, args):
	"""Creates AttackLogManager from an AttackArgs object."""
	assert isinstance(
	args, cls
	), f"Expect args to be of type `{type(cls)}`, but got type `{type(args)}`."

	# Create logger
	attack_log_manager = textattack.loggers.AttackLogManager(args.metrics)

	# Get current time for file naming
	timestamp = time.strftime("%Y-%m-%d-%H-%M")

	# if '--log-to-txt' specified with arguments
	if args.log_to_txt is not None:
	if args.log_to_txt.lower().endswith(".txt"):
	txt_file_path = args.log_to_txt
	else:
	txt_file_path = os.path.join(args.log_to_txt, f"{timestamp}-log.txt")

	dir_path = os.path.dirname(txt_file_path)
	dir_path = dir_path if dir_path else "."
	if not os.path.exists(dir_path):
	os.makedirs(os.path.dirname(txt_file_path))

	color_method = "file"
	attack_log_manager.add_output_file(txt_file_path, color_method)

	# if '--log-to-csv' specified with arguments
	if args.log_to_csv is not None:
	if args.log_to_csv.lower().endswith(".csv"):
	csv_file_path = args.log_to_csv
	else:
	csv_file_path = os.path.join(args.log_to_csv, f"{timestamp}-log.csv")

	dir_path = os.path.dirname(csv_file_path)
	dir_path = dir_path if dir_path else "."
	if not os.path.exists(dir_path):
	os.makedirs(dir_path)

	color_method = (
	None if args.csv_coloring_style == "plain" else args.csv_coloring_style
	)
	attack_log_manager.add_output_csv(csv_file_path, color_method)

	# if '--log-summary-to-json' specified with arguments
	if args.log_summary_to_json is not None:
	if args.log_summary_to_json.lower().endswith(".json"):
	summary_json_file_path = args.log_summary_to_json
	else:
	summary_json_file_path = os.path.join(
	args.log_summary_to_json, f"{timestamp}-attack_summary_log.json"
	)

	dir_path = os.path.dirname(summary_json_file_path)
	dir_path = dir_path if dir_path else "."
	if not os.path.exists(dir_path):
	os.makedirs(os.path.dirname(summary_json_file_path))

	attack_log_manager.add_output_summary_json(summary_json_file_path)

	# Visdom
	if args.log_to_visdom is not None:
	attack_log_manager.enable_visdom(**args.log_to_visdom)

	# Weights & Biases
	if args.log_to_wandb is not None:
	attack_log_manager.enable_wandb(**args.log_to_wandb)

	# Stdout
	if not args.disable_stdout and not sys.stdout.isatty():
	attack_log_manager.disable_color()
	elif not args.disable_stdout:
	attack_log_manager.enable_stdout()

	return attack_log_manager


	@dataclass
	class _CommandLineAttackArgs:
	"""Attack args for command line execution. This requires more arguments to
	create ``Attack`` object as specified.

	Args:
	transformation (:obj:`str`, `optional`, defaults to :obj:`"word-swap-embedding"`):
	Name of transformation to use.
	constraints (:obj:`list[str]`, `optional`, defaults to :obj:`["repeat", "stopword"]`):
	List of names of constraints to use.
	goal_function (:obj:`str`, `optional`, defaults to :obj:`"untargeted-classification"`):
	Name of goal function to use.
	search_method (:obj:`str`, `optional`, defualts to :obj:`"greedy-word-wir"`):
	Name of search method to use.
	attack_recipe (:obj:`str`, `optional`, defaults to :obj:`None`):
	Name of attack recipe to use.
	.. note::
	Setting this overrides any previous selection of transformation, constraints, goal function, and search method.
	attack_from_file (:obj:`str`, `optional`, defaults to :obj:`None`):
	Path of `.py` file from which to load attack from. Use `<path>^<variable_name>` to specifiy which variable to import from the file.
	.. note::
	If this is set, it overrides any previous selection of transformation, constraints, goal function, and search method
	interactive (:obj:`bool`, `optional`, defaults to :obj:`False`):
	If `True`, carry attack in interactive mode.
	parallel (:obj:`bool`, `optional`, defaults to :obj:`False`):
	If `True`, attack in parallel.
	model_batch_size (:obj:`int`, `optional`, defaults to :obj:`32`):
	The batch size for making queries to the victim model.
	model_cache_size (:obj:`int`, `optional`, defaults to :obj:`2**18`):
	The maximum number of items to keep in the model results cache at once.
	constraint-cache-size (:obj:`int`, `optional`, defaults to :obj:`2**18`):
	The maximum number of items to keep in the constraints cache at once.
	"""

	transformation: str = "word-swap-embedding"
	constraints: list = field(default_factory=lambda: ["repeat", "stopword"])
	goal_function: str = "untargeted-classification"
	search_method: str = "greedy-word-wir"
	attack_recipe: str = None
	attack_from_file: str = None
	interactive: bool = False
	parallel: bool = False
	model_batch_size: int = 32
	model_cache_size: int = 2**18
	constraint_cache_size: int = 2**18

	@classmethod
	def _add_parser_args(cls, parser):
	"""Add listed args to command line parser."""
	default_obj = cls()
	transformation_names = set(BLACK_BOX_TRANSFORMATION_CLASS_NAMES.keys()) \| set(
	WHITE_BOX_TRANSFORMATION_CLASS_NAMES.keys()
	)
	parser.add_argument(
	"--transformation",
	type=str,
	required=False,
	default=default_obj.transformation,
	help='The transformation to apply. Usage: "--transformation {transformation}:{arg_1}={value_1},{arg_3}={value_3}". Choices: '
	+ str(transformation_names),
	)
	parser.add_argument(
	"--constraints",
	type=str,
	required=False,
	nargs="*",
	default=default_obj.constraints,
	help='Constraints to add to the attack. Usage: "--constraints {constraint}:{arg_1}={value_1},{arg_3}={value_3}". Choices: '
	+ str(CONSTRAINT_CLASS_NAMES.keys()),
	)
	goal_function_choices = ", ".join(GOAL_FUNCTION_CLASS_NAMES.keys())
	parser.add_argument(
	"--goal-function",
	"-g",
	default=default_obj.goal_function,
	help=f"The goal function to use. choices: {goal_function_choices}",
	)
	attack_group = parser.add_mutually_exclusive_group(required=False)
	search_choices = ", ".join(SEARCH_METHOD_CLASS_NAMES.keys())
	attack_group.add_argument(
	"--search-method",
	"--search",
	"-s",
	type=str,
	required=False,
	default=default_obj.search_method,
	help=f"The search method to use. choices: {search_choices}",
	)
	attack_group.add_argument(
	"--attack-recipe",
	"--recipe",
	"-r",
	type=str,
	required=False,
	default=default_obj.attack_recipe,
	help="full attack recipe (overrides provided goal function, transformation & constraints)",
	choices=ATTACK_RECIPE_NAMES.keys(),
	)
	attack_group.add_argument(
	"--attack-from-file",
	type=str,
	required=False,
	default=default_obj.attack_from_file,
	help="Path of `.py` file from which to load attack from. Use `<path>^<variable_name>` to specifiy which variable to import from the file.",
	)
	parser.add_argument(
	"--interactive",
	action="store_true",
	default=default_obj.interactive,
	help="Whether to run attacks interactively.",
	)
	parser.add_argument(
	"--model-batch-size",
	type=int,
	default=default_obj.model_batch_size,
	help="The batch size for making calls to the model.",
	)
	parser.add_argument(
	"--model-cache-size",
	type=int,
	default=default_obj.model_cache_size,
	help="The maximum number of items to keep in the model results cache at once.",
	)
	parser.add_argument(
	"--constraint-cache-size",
	type=int,
	default=default_obj.constraint_cache_size,
	help="The maximum number of items to keep in the constraints cache at once.",
	)

	return parser

	@classmethod
	def _create_transformation_from_args(cls, args, model_wrapper):
	"""Create `Transformation` based on provided `args` and
	`model_wrapper`."""

	transformation_name = args.transformation
	if ARGS_SPLIT_TOKEN in transformation_name:
	transformation_name, params = transformation_name.split(ARGS_SPLIT_TOKEN)

	if transformation_name in WHITE_BOX_TRANSFORMATION_CLASS_NAMES:
	transformation = eval(
	f"{WHITE_BOX_TRANSFORMATION_CLASS_NAMES[transformation_name]}(model_wrapper.model, {params})"
	)
	elif transformation_name in BLACK_BOX_TRANSFORMATION_CLASS_NAMES:
	transformation = eval(
	f"{BLACK_BOX_TRANSFORMATION_CLASS_NAMES[transformation_name]}({params})"
	)
	else:
	raise ValueError(
	f"Error: unsupported transformation {transformation_name}"
	)
	else:
	if transformation_name in WHITE_BOX_TRANSFORMATION_CLASS_NAMES:
	transformation = eval(
	f"{WHITE_BOX_TRANSFORMATION_CLASS_NAMES[transformation_name]}(model_wrapper.model)"
	)
	elif transformation_name in BLACK_BOX_TRANSFORMATION_CLASS_NAMES:
	transformation = eval(
	f"{BLACK_BOX_TRANSFORMATION_CLASS_NAMES[transformation_name]}()"
	)
	else:
	raise ValueError(
	f"Error: unsupported transformation {transformation_name}"
	)
	return transformation

	@classmethod
	def _create_goal_function_from_args(cls, args, model_wrapper):
	"""Create `GoalFunction` based on provided `args` and
	`model_wrapper`."""

	goal_function = args.goal_function
	if ARGS_SPLIT_TOKEN in goal_function:
	goal_function_name, params = goal_function.split(ARGS_SPLIT_TOKEN)
	if goal_function_name not in GOAL_FUNCTION_CLASS_NAMES:
	raise ValueError(
	f"Error: unsupported goal_function {goal_function_name}"
	)
	goal_function = eval(
	f"{GOAL_FUNCTION_CLASS_NAMES[goal_function_name]}(model_wrapper, {params})"
	)
	elif goal_function in GOAL_FUNCTION_CLASS_NAMES:
	goal_function = eval(
	f"{GOAL_FUNCTION_CLASS_NAMES[goal_function]}(model_wrapper)"
	)
	else:
	raise ValueError(f"Error: unsupported goal_function {goal_function}")
	if args.query_budget:
	goal_function.query_budget = args.query_budget
	goal_function.model_cache_size = args.model_cache_size
	goal_function.batch_size = args.model_batch_size
	return goal_function

	@classmethod
	def _create_constraints_from_args(cls, args):
	"""Create list of `Constraints` based on provided `args`."""

	if not args.constraints:
	return []

	_constraints = []
	for constraint in args.constraints:
	if ARGS_SPLIT_TOKEN in constraint:
	constraint_name, params = constraint.split(ARGS_SPLIT_TOKEN)
	if constraint_name not in CONSTRAINT_CLASS_NAMES:
	raise ValueError(f"Error: unsupported constraint {constraint_name}")
	_constraints.append(
	eval(f"{CONSTRAINT_CLASS_NAMES[constraint_name]}({params})")
	)
	elif constraint in CONSTRAINT_CLASS_NAMES:
	_constraints.append(eval(f"{CONSTRAINT_CLASS_NAMES[constraint]}()"))
	else:
	raise ValueError(f"Error: unsupported constraint {constraint}")

	return _constraints

	@classmethod
	def _create_attack_from_args(cls, args, model_wrapper):
	"""Given ``CommandLineArgs`` and ``ModelWrapper``, return specified
	``Attack`` object."""

	assert isinstance(
	args, cls
	), f"Expect args to be of type `{type(cls)}`, but got type `{type(args)}`."

	if args.attack_recipe:
	if ARGS_SPLIT_TOKEN in args.attack_recipe:
	recipe_name, params = args.attack_recipe.split(ARGS_SPLIT_TOKEN)
	if recipe_name not in ATTACK_RECIPE_NAMES:
	raise ValueError(f"Error: unsupported recipe {recipe_name}")
	recipe = eval(
	f"{ATTACK_RECIPE_NAMES[recipe_name]}.build(model_wrapper, {params})"
	)
	elif args.attack_recipe in ATTACK_RECIPE_NAMES:
	recipe = eval(
	f"{ATTACK_RECIPE_NAMES[args.attack_recipe]}.build(model_wrapper)"
	)
	else:
	raise ValueError(f"Invalid recipe {args.attack_recipe}")
	if args.query_budget:
	recipe.goal_function.query_budget = args.query_budget
	recipe.goal_function.model_cache_size = args.model_cache_size
	recipe.constraint_cache_size = args.constraint_cache_size
	return recipe
	elif args.attack_from_file:
	if ARGS_SPLIT_TOKEN in args.attack_from_file:
	attack_file, attack_name = args.attack_from_file.split(ARGS_SPLIT_TOKEN)
	else:
	attack_file, attack_name = args.attack_from_file, "attack"
	attack_module = load_module_from_file(attack_file)
	if not hasattr(attack_module, attack_name):
	raise ValueError(
	f"Loaded `{attack_file}` but could not find `{attack_name}`."
	)
	attack_func = getattr(attack_module, attack_name)
	return attack_func(model_wrapper)
	else:
	goal_function = cls._create_goal_function_from_args(args, model_wrapper)
	transformation = cls._create_transformation_from_args(args, model_wrapper)
	constraints = cls._create_constraints_from_args(args)
	if ARGS_SPLIT_TOKEN in args.search_method:
	search_name, params = args.search_method.split(ARGS_SPLIT_TOKEN)
	if search_name not in SEARCH_METHOD_CLASS_NAMES:
	raise ValueError(f"Error: unsupported search {search_name}")
	search_method = eval(
	f"{SEARCH_METHOD_CLASS_NAMES[search_name]}({params})"
	)
	elif args.search_method in SEARCH_METHOD_CLASS_NAMES:
	search_method = eval(
	f"{SEARCH_METHOD_CLASS_NAMES[args.search_method]}()"
	)
	else:
	raise ValueError(f"Error: unsupported attack {args.search_method}")

	return Attack(
	goal_function,
	constraints,
	transformation,
	search_method,
	constraint_cache_size=args.constraint_cache_size,
	)


	# This neat trick allows use to reorder the arguments to avoid TypeErrors commonly found when inheriting dataclass.
	# https://stackoverflow.com/questions/51575931/class-inheritance-in-python-3-7-dataclasses
	@dataclass
	class CommandLineAttackArgs(AttackArgs, _CommandLineAttackArgs, DatasetArgs, ModelArgs):
	@classmethod
	def _add_parser_args(cls, parser):
	"""Add listed args to command line parser."""
	parser = ModelArgs._add_parser_args(parser)
	parser = DatasetArgs._add_parser_args(parser)
	parser = _CommandLineAttackArgs._add_parser_args(parser)
	parser = AttackArgs._add_parser_args(parser)
	return parser