Spaces:
Running
Running
File size: 7,909 Bytes
f18cdf1 7e0dde7 f18cdf1 e414859 2b1a300 e414859 2b1a300 f18cdf1 7e0dde7 f18cdf1 e414859 f18cdf1 e414859 2b1a300 2b8c4c9 2b1a300 f18cdf1 2b8c4c9 f18cdf1 e414859 2b8c4c9 2b1a300 e414859 2b1a300 e414859 2b1a300 e414859 2b8c4c9 2b1a300 e414859 2b1a300 e414859 2b1a300 e414859 2b1a300 e414859 2b1a300 e414859 2b1a300 e414859 2b1a300 e414859 2b1a300 e414859 7e0dde7 e414859 2b1a300 e414859 2b1a300 7e0dde7 2b1a300 e414859 2b1a300 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 |
"""
aggregate.py - module for aggregating text from multiple sources/multiple parts of a single source.
Primary usage is through the BatchAggregator class.
How it works:
1. We tell the language model to do it.
2. The language model does it.
3. Yaay!
"""
import pprint as pp
import logging
import time
import torch
from transformers import GenerationConfig, pipeline
from utils import compare_model_size
# Setting up logging
logging.basicConfig(
level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
)
class BatchAggregator:
"""
BatchAggregator is a class for aggregating text from multiple sources.
Usage:
>>> from aggregate import BatchAggregator
>>> aggregator = BatchAggregator()
>>> agg = aggregator.infer_aggregate(["This is a test", "This is another test"])
>>> print(agg)
"""
GENERIC_CONFIG = GenerationConfig(
num_beams=8,
early_stopping=True,
do_sample=False,
min_new_tokens=32,
max_new_tokens=256,
repetition_penalty=1.1,
length_penalty=1.4,
no_repeat_ngram_size=4,
encoder_no_repeat_ngram_size=5,
)
CONFIGURED_MODELS = [
"pszemraj/bart-large-mnli-dolly_hhrlhf-v1",
"pszemraj/bart-base-instruct-dolly_hhrlhf",
"pszemraj/flan-t5-large-instruct-dolly_hhrlhf",
"pszemraj/flan-t5-base-instruct-dolly_hhrlhf",
] # these have generation configs defined for this task in their model repos
DEFAULT_INSTRUCTION = "Write a comprehensive yet concise summary that pulls together the main points of the following text:"
def __init__(
self,
model_name: str = "pszemraj/bart-large-mnli-dolly_hhrlhf-v1",
force_cpu: bool = False,
**kwargs,
):
"""
__init__ initializes the BatchAggregator class.
:param str model_name: model name to use, default: "pszemraj/bart-large-mnli-dolly_hhrlhf-v1"
:param bool force_cpu: force the model to run on CPU, default: False
"""
self.device = None
self.is_compiled = False
self.model_name = None
self.aggregator = None
self.force_cpu = force_cpu
self.logger = logging.getLogger(__name__)
self.init_model(model_name)
def init_model(self, model_name: str) -> None:
"""
Initialize the model.
:param model_name: The name of the model to use.
"""
# Free up memory
if torch.cuda.is_available():
torch.cuda.empty_cache()
self.logger.info(f"Setting model to {model_name}")
self.model_name = model_name
self.aggregator = self._create_pipeline(model_name)
self._configure_model()
# update the generation config with the specific tokenizer
tokenizer_params = {
"decoder_start_token_id": 0
if "t5" in model_name.lower()
else self.aggregator.tokenizer.eos_token_id,
"eos_token_id": 1
if "t5" in model_name.lower()
else self.aggregator.tokenizer.eos_token_id,
"pad_token_id": 0
if "t5" in model_name.lower()
else self.aggregator.tokenizer.pad_token_id,
}
self.update_generation_config(**tokenizer_params)
def _create_pipeline(
self, model_name: str = "pszemraj/bart-large-mnli-dolly_hhrlhf-v1"
) -> pipeline:
"""
_create_pipeline creates a pipeline for the model.
:param str model_name: model name to use, default: "pszemraj/bart-large-mnli-dolly_hhrlhf-v1"
:return pipeline: the pipeline for the model
:raises Exception: if the pipeline cannot be created
"""
self.device = 0 if torch.cuda.is_available() and not self.force_cpu else -1
try:
self.logger.info(
f"Creating pipeline with model {model_name} on device {self.device}"
)
return pipeline(
"text2text-generation",
model_name,
device=self.device,
torch_dtype=torch.float32,
)
except Exception as e:
self.logger.error(f"Failed to create pipeline: {e}")
raise
def _configure_model(self):
"""
Configure the model for generation.
"""
try:
self.aggregator.model = torch.compile(self.aggregator.model)
self.is_compiled = True
except Exception as e:
self.logger.warning(f"Could not compile model with Torch 2.0: {e}")
if self.model_name not in self.CONFIGURED_MODELS:
self.logger.info("Setting generation config to general defaults")
self._set_default_generation_config()
else:
try:
self.logger.info("Loading generation config from hub")
self.aggregator.model.generation_config = (
GenerationConfig.from_pretrained(self.model_name)
)
except Exception as e:
self.logger.warning(
f"Could not load generation config, using defaults: {e}"
)
self._set_default_generation_config()
self.logger.info(self.aggregator.model.generation_config.to_json_string())
def _set_default_generation_config(self):
"""
Set the default generation configuration for the model.
"""
self.aggregator.model.generation_config = self.GENERIC_CONFIG
if (
"large"
or "xl" in self.model_name.lower()
or compare_model_size(self.model_name, 500)
):
upd = {"num_beams": 4}
self.update_generation_config(**upd)
def update_generation_config(self, **kwargs):
"""
Update the generation configuration with the specified parameters.
Args:
**kwargs: The parameters to update in the generation configuration.
"""
self.logger.info(f"Updating generation config with {pp.pformat(kwargs)}")
self.aggregator.model.generation_config.update(**kwargs)
def update_loglevel(self, level: str = "INFO"):
"""
Update the log level.
Args:
level (str): The log level to set. Defaults to "INFO".
"""
self.logger.setLevel(level)
def infer_aggregate(
self,
text_list: list,
instruction: str = DEFAULT_INSTRUCTION,
**kwargs,
) -> str:
f"""
infer_aggregate - infers a consolidated summary from a list of texts.
Args:
text_list (list): The texts to summarize.
instruction (str): The instruction for the summary. Defaults to {self.DEFAULT_INSTRUCTION}.
**kwargs: Additional parameters to update in the generation configuration.
Returns:
The generated summary.
"""
joined_text = "\n".join(text_list)
prompt = f"{instruction}\n\n{joined_text}\n"
if kwargs:
self.update_generation_config(**kwargs)
st = time.perf_counter()
self.logger.info(f"inference on {len(text_list)} texts ...")
result = self.aggregator(
prompt,
generation_config=self.aggregator.model.generation_config,
)[0]["generated_text"]
self.logger.info(f"Done. runtime:\t{round(time.perf_counter() - st, 2)}s")
self.logger.info(
f"Input tokens:\t{self.count_tokens(prompt)}. Output tokens:\t{self.count_tokens(result)}"
)
self.logger.debug(f"Generated text:\n{result}")
return result
def count_tokens(self, text: str) -> int:
"""count the number of tokens in a text"""
return (
len(self.aggregator.tokenizer.encode(text, truncation=False, padding=False))
if text
else 0
)
|