Upload standard.py with huggingface_hub
Browse files- standard.py +12 -6
standard.py
CHANGED
@@ -5,11 +5,7 @@ from .dataclass import Field, InternalField, OptionalField
|
|
5 |
from .formats import Format, SystemFormat
|
6 |
from .logging_utils import get_logger
|
7 |
from .operator import SourceSequentialOperator, StreamingOperator
|
8 |
-
from .operators import
|
9 |
-
Augmentor,
|
10 |
-
NullAugmentor,
|
11 |
-
StreamRefiner,
|
12 |
-
)
|
13 |
from .recipe import Recipe
|
14 |
from .schema import ToUnitxtGroup
|
15 |
from .splitters import Sampler, SeparateSplit, SpreadSplit
|
@@ -67,7 +63,7 @@ class BaseRecipe(Recipe, SourceSequentialOperator):
|
|
67 |
if self.num_demos > 0:
|
68 |
if self.demos_pool_size is None or self.demos_pool_size < 1:
|
69 |
raise ValueError(
|
70 |
-
"When using demonstrations both num_demos and demos_pool_size should be assigned with
|
71 |
)
|
72 |
if self.demos_pool_size < self.num_demos:
|
73 |
raise ValueError(
|
@@ -114,6 +110,16 @@ class BaseRecipe(Recipe, SourceSequentialOperator):
|
|
114 |
def prepare(self):
|
115 |
self.steps = [
|
116 |
self.card.loader,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
117 |
]
|
118 |
|
119 |
if self.loader_limit:
|
|
|
5 |
from .formats import Format, SystemFormat
|
6 |
from .logging_utils import get_logger
|
7 |
from .operator import SourceSequentialOperator, StreamingOperator
|
8 |
+
from .operators import AddFields, Augmentor, NullAugmentor, StreamRefiner
|
|
|
|
|
|
|
|
|
9 |
from .recipe import Recipe
|
10 |
from .schema import ToUnitxtGroup
|
11 |
from .splitters import Sampler, SeparateSplit, SpreadSplit
|
|
|
63 |
if self.num_demos > 0:
|
64 |
if self.demos_pool_size is None or self.demos_pool_size < 1:
|
65 |
raise ValueError(
|
66 |
+
"When using demonstrations both num_demos and demos_pool_size should be assigned with positive integers."
|
67 |
)
|
68 |
if self.demos_pool_size < self.num_demos:
|
69 |
raise ValueError(
|
|
|
110 |
def prepare(self):
|
111 |
self.steps = [
|
112 |
self.card.loader,
|
113 |
+
AddFields(
|
114 |
+
fields={
|
115 |
+
"recipe_metadata": {
|
116 |
+
"card": self.card,
|
117 |
+
"template": self.template,
|
118 |
+
"system_prompt": self.system_prompt,
|
119 |
+
"format": self.format,
|
120 |
+
}
|
121 |
+
}
|
122 |
+
),
|
123 |
]
|
124 |
|
125 |
if self.loader_limit:
|