Upload processing_mists.py

processing_mists.py

@@ -44,6 +44,7 @@ class MistsProcessor(ProcessorMixin):
         torch_dtype: Optional[Union[str, torch.dtype]] = torch.float,
         time_series_padding: Union[bool, str] = False,
         time_series_max_length: Union[int, None] = None,
+        text_tokenize: bool = True,
     ) -> BatchFeature:
         if time_series is not None:
             time_series_values = self.feature_extractor(
@@ -55,9 +56,15 @@ class MistsProcessor(ProcessorMixin):
             )
         else:
             time_series_values = None
-        text_inputs = self.tokenizer(
-            text, return_tensors=return_tensors, padding=padding, truncation=truncation, max_length=max_length
-        )
+        if text is not None: 
+            if text_tokenize:
+                text_inputs = self.tokenizer(
+                    text, return_tensors=return_tensors, padding=padding, truncation=truncation, max_length=max_length
+                )
+            else:
+                text_inputs = {"text": text}
+        else:
+            text_inputs = None
 
         return BatchFeature(data={**text_inputs, **time_series_values})