When using TimeSeriesTransformerForPrediction, able to train model but not predict on test set
Author: brett1099
Pull request: no

Using the code from this article: https://huggingface.co/blog/time-series-transformers, I was able to successfully run the data they utilized. However when attempting with my own data in the same format, I am able to get to training the model successfully but get an error when creating predictions.

I have a dataset with three columns, "start" as the index, "item_id", and "target", as matching the code from the article.

`test_sample = test_dataset[0]
test_sample.keys()`
_OUT `dict_keys(['start', 'target', 'item_id', 'feat_static_cat'])`_


Here is the full error message:
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
Input In [59], in <cell line: 5>()
      3 forecasts = []
      5 for batch in test_dataloader:
----> 6     outputs = model.generate(
      7         static_categorical_features=batch["static_categorical_features"].to(device)
      8         if config.num_static_categorical_features > 0
      9         else None,
     10         static_real_features=batch["static_real_features"].to(device)
     11         if config.num_static_real_features > 0
     12         else None,
     13         past_time_features=batch["past_time_features"].to(device),
     14         past_values=batch["past_values"].to(device),
     15         future_time_features=batch["future_time_features"].to(device),
     16         past_observed_mask=batch["past_observed_mask"].to(device),
     17     )
     18     forecasts.append(outputs.sequences.cpu().numpy())

File ~/.local/lib/python3.8/site-packages/torch/autograd/grad_mode.py:27, in _DecoratorContextManager.__call__.<locals>.decorate_context(*args, **kwargs)
     24 @functools.wraps(func)
     25 def decorate_context(*args, **kwargs):
     26     with self.clone():
---> 27         return func(*args, **kwargs)

File ~/.local/lib/python3.8/site-packages/transformers/models/time_series_transformer/modeling_time_series_transformer.py:1760, in TimeSeriesTransformerForPrediction.generate(self, past_values, past_time_features, future_time_features, past_observed_mask, static_categorical_features, static_real_features, output_attentions, output_hidden_states)
   1661 @torch.no_grad()
   1662 def generate(
   1663     self,
   (...)
   1671     output_hidden_states: Optional[bool] = None,
   1672 ) -> SampleTSPredictionOutput:
   1673     r"""
   1674     Greedily generate sequences of sample predictions from a model with a probability distribution head.
   1675 
   (...)
   1758         multivariate predictions.
   1759     """
-> 1760     outputs = self(
   1761         static_categorical_features=static_categorical_features,
   1762         static_real_features=static_real_features,
   1763         past_time_features=past_time_features,
   1764         past_values=past_values,
   1765         past_observed_mask=past_observed_mask,
   1766         future_time_features=future_time_features,
   1767         future_values=None,
   1768         output_attentions=output_attentions,
   1769         output_hidden_states=output_hidden_states,
   1770         return_dict=True,
   1771         use_cache=True,
   1772     )
   1774     decoder = self.model.get_decoder()
   1775     enc_last_hidden = outputs.encoder_last_hidden_state

File ~/.local/lib/python3.8/site-packages/torch/nn/modules/module.py:1194, in Module._call_impl(self, *input, **kwargs)
   1190 # If we don't have any hooks, we want to skip the rest of the logic in
   1191 # this function, and just call forward.
   1192 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1193         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1194     return forward_call(*input, **kwargs)
   1195 # Do not call functions when jit is used
   1196 full_backward_hooks, non_full_backward_hooks = [], []

File ~/.local/lib/python3.8/site-packages/transformers/models/time_series_transformer/modeling_time_series_transformer.py:1603, in TimeSeriesTransformerForPrediction.forward(self, past_values, past_time_features, past_observed_mask, static_categorical_features, static_real_features, future_values, future_time_features, future_observed_mask, decoder_attention_mask, head_mask, decoder_head_mask, cross_attn_head_mask, encoder_outputs, past_key_values, output_hidden_states, output_attentions, use_cache, return_dict)
   1600 if future_values is not None:
   1601     use_cache = False
-> 1603 outputs = self.model(
   1604     past_values=past_values,
   1605     past_time_features=past_time_features,
   1606     past_observed_mask=past_observed_mask,
   1607     static_categorical_features=static_categorical_features,
   1608     static_real_features=static_real_features,
   1609     future_values=future_values,
   1610     future_time_features=future_time_features,
   1611     decoder_attention_mask=decoder_attention_mask,
   1612     head_mask=head_mask,
   1613     decoder_head_mask=decoder_head_mask,
   1614     cross_attn_head_mask=cross_attn_head_mask,
   1615     encoder_outputs=encoder_outputs,
   1616     past_key_values=past_key_values,
   1617     output_hidden_states=output_hidden_states,
   1618     output_attentions=output_attentions,
   1619     use_cache=use_cache,
   1620     return_dict=return_dict,
   1621 )
   1623 prediction_loss = None
   1624 params = None

File ~/.local/lib/python3.8/site-packages/torch/nn/modules/module.py:1194, in Module._call_impl(self, *input, **kwargs)
   1190 # If we don't have any hooks, we want to skip the rest of the logic in
   1191 # this function, and just call forward.
   1192 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1193         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1194     return forward_call(*input, **kwargs)
   1195 # Do not call functions when jit is used
   1196 full_backward_hooks, non_full_backward_hooks = [], []

File ~/.local/lib/python3.8/site-packages/transformers/models/time_series_transformer/modeling_time_series_transformer.py:1452, in TimeSeriesTransformerModel.forward(self, past_values, past_time_features, past_observed_mask, static_categorical_features, static_real_features, future_values, future_time_features, decoder_attention_mask, head_mask, decoder_head_mask, cross_attn_head_mask, encoder_outputs, past_key_values, output_hidden_states, output_attentions, use_cache, return_dict)
   1445     encoder_outputs = BaseModelOutput(
   1446         last_hidden_state=encoder_outputs[0],
   1447         hidden_states=encoder_outputs[1] if len(encoder_outputs) > 1 else None,
   1448         attentions=encoder_outputs[2] if len(encoder_outputs) > 2 else None,
   1449     )
   1451 dec_input = transformer_inputs[:, self.config.context_length :, ...]
-> 1452 decoder_outputs = self.decoder(
   1453     inputs_embeds=dec_input,
   1454     attention_mask=decoder_attention_mask,
   1455     encoder_hidden_states=encoder_outputs[0],
   1456     head_mask=decoder_head_mask,
   1457     cross_attn_head_mask=cross_attn_head_mask,
   1458     past_key_values=past_key_values,
   1459     use_cache=use_cache,
   1460     output_attentions=output_attentions,
   1461     output_hidden_states=output_hidden_states,
   1462     return_dict=return_dict,
   1463 )
   1465 if not return_dict:
   1466     return decoder_outputs + encoder_outputs + (loc, scale, static_feat)

File ~/.local/lib/python3.8/site-packages/torch/nn/modules/module.py:1194, in Module._call_impl(self, *input, **kwargs)
   1190 # If we don't have any hooks, we want to skip the rest of the logic in
   1191 # this function, and just call forward.
   1192 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1193         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1194     return forward_call(*input, **kwargs)
   1195 # Do not call functions when jit is used
   1196 full_backward_hooks, non_full_backward_hooks = [], []

File ~/.local/lib/python3.8/site-packages/transformers/models/time_series_transformer/modeling_time_series_transformer.py:1178, in TimeSeriesTransformerDecoder.forward(self, attention_mask, encoder_hidden_states, encoder_attention_mask, head_mask, cross_attn_head_mask, past_key_values, inputs_embeds, use_cache, output_attentions, output_hidden_states, return_dict)
   1167     layer_outputs = torch.utils.checkpoint.checkpoint(
   1168         create_custom_forward(decoder_layer),
   1169         hidden_states,
   (...)
   1175         None,
   1176     )
   1177 else:
-> 1178     layer_outputs = decoder_layer(
   1179         hidden_states,
   1180         attention_mask=attention_mask,
   1181         encoder_hidden_states=encoder_hidden_states,
   1182         encoder_attention_mask=encoder_attention_mask,
   1183         layer_head_mask=(head_mask[idx] if head_mask is not None else None),
   1184         cross_attn_layer_head_mask=(
   1185             cross_attn_head_mask[idx] if cross_attn_head_mask is not None else None
   1186         ),
   1187         past_key_value=past_key_value,
   1188         output_attentions=output_attentions,
   1189         use_cache=use_cache,
   1190     )
   1191 hidden_states = layer_outputs[0]
   1193 if use_cache:

File ~/.local/lib/python3.8/site-packages/torch/nn/modules/module.py:1194, in Module._call_impl(self, *input, **kwargs)
   1190 # If we don't have any hooks, we want to skip the rest of the logic in
   1191 # this function, and just call forward.
   1192 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1193         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1194     return forward_call(*input, **kwargs)
   1195 # Do not call functions when jit is used
   1196 full_backward_hooks, non_full_backward_hooks = [], []

File ~/.local/lib/python3.8/site-packages/transformers/models/time_series_transformer/modeling_time_series_transformer.py:611, in TimeSeriesTransformerDecoderLayer.forward(self, hidden_states, attention_mask, encoder_hidden_states, encoder_attention_mask, layer_head_mask, cross_attn_layer_head_mask, past_key_value, output_attentions, use_cache)
    609 # cross_attn cached key/values tuple is at positions 3,4 of present_key_value tuple
    610 cross_attn_past_key_value = past_key_value[-2:] if past_key_value is not None else None
--> 611 hidden_states, cross_attn_weights, cross_attn_present_key_value = self.encoder_attn(
    612     hidden_states=hidden_states,
    613     key_value_states=encoder_hidden_states,
    614     attention_mask=encoder_attention_mask,
    615     layer_head_mask=cross_attn_layer_head_mask,
    616     past_key_value=cross_attn_past_key_value,
    617     output_attentions=output_attentions,
    618 )
    619 hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)
    620 hidden_states = residual + hidden_states

File ~/.local/lib/python3.8/site-packages/torch/nn/modules/module.py:1194, in Module._call_impl(self, *input, **kwargs)
   1190 # If we don't have any hooks, we want to skip the rest of the logic in
   1191 # this function, and just call forward.
   1192 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1193         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1194     return forward_call(*input, **kwargs)
   1195 # Do not call functions when jit is used
   1196 full_backward_hooks, non_full_backward_hooks = [], []

File ~/.local/lib/python3.8/site-packages/transformers/models/time_series_transformer/modeling_time_series_transformer.py:371, in TimeSeriesTransformerAttention.forward(self, hidden_states, key_value_states, past_key_value, attention_mask, layer_head_mask, output_attentions)
    368     value_states = past_key_value[1]
    369 elif is_cross_attention:
    370     # cross_attentions
--> 371     key_states = self._shape(self.k_proj(key_value_states), -1, bsz)
    372     value_states = self._shape(self.v_proj(key_value_states), -1, bsz)
    373 elif past_key_value is not None:
    374     # reuse k, v, self_attention

File ~/.local/lib/python3.8/site-packages/torch/nn/modules/module.py:1194, in Module._call_impl(self, *input, **kwargs)
   1190 # If we don't have any hooks, we want to skip the rest of the logic in
   1191 # this function, and just call forward.
   1192 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1193         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1194     return forward_call(*input, **kwargs)
   1195 # Do not call functions when jit is used
   1196 full_backward_hooks, non_full_backward_hooks = [], []

File ~/.local/lib/python3.8/site-packages/torch/nn/modules/linear.py:114, in Linear.forward(self, input)
    113 def forward(self, input: Tensor) -> Tensor:
--> 114     return F.linear(input, self.weight, self.bias)

RuntimeError: CUDA error: CUBLAS_STATUS_EXECUTION_FAILED when calling `cublasLtMatmul( ltHandle, computeDesc.descriptor(), &alpha_val, mat1_ptr, Adesc.descriptor(), mat2_ptr, Bdesc.descriptor(), &beta_val, result_ptr, Cdesc.descriptor(), result_ptr, Cdesc.descriptor(), &heuristicResult.algo, workspace.data_ptr(), workspaceSize, at::cuda::getCurrentCUDAStream())`

Comments:
cc @kashif 

thanks @brett1099 for the issue... just having a look at the error to see if i can figure out the issue...

@brett1099  can you kindly confirm that the length of the target time series in the test set are bigger that the corresponding training set target arrays by exactly the `prediction_length`?

Hi @kashif , yes, I can confirm that is the case. Here are the details below from my code:

`train_dataset`
Dataset({
    features: ['start', 'target', 'item_id', 'feat_static_cat'],
    num_rows: 366
})

`test_dataset`
Dataset({
    features: ['start', 'target', 'item_id', 'feat_static_cat'],
    num_rows: 366
})

(limited from over 2k to 366 rows to match the article in case there was a memory issue)

`train_sample = train_dataset[0]`
`test_sample = test_dataset[0]`

`print(len(test_sample["target"]))`
811
`print(len(train_sample["target"]))`
755

`prediction_length = 56`

`len(train_df["target"])`
274546

`len(test_df["target"])`
295042

I checked that (295042 - 274546) / 366 = 56, the prediction length.

@brett1099 can you kindly for the purpose of debugging train and do inference on the CPU device and see what the error is?

@kashif , seems I am not able to successfully separate onto only the CPU device. Here is the code below attempting to run, specified device as cpu.

`from accelerate import Accelerator
from torch.optim import AdamW

accelerator = Accelerator()
#device = accelerator.device
device = "cpu"

model.to(device)
optimizer = AdamW(model.parameters(), lr=2e-4, betas=(0.9, 0.995), weight_decay=1e-2)

model, optimizer, train_dataloader = accelerator.prepare(
    model,
    optimizer,
    train_dataloader,
)

model.train()
for epoch in range(3):
    for idx, batch in enumerate(train_dataloader):
        optimizer.zero_grad()
        outputs = model(
            static_categorical_features=batch["static_categorical_features"].to(device)
            if config.num_static_categorical_features > 0
            else None,
            static_real_features=batch["static_real_features"].to(device)
            if config.num_static_real_features > 0
            else None,
            past_time_features=batch["past_time_features"].to(device),
            past_values=batch["past_values"].to(device),
            future_time_features=batch["future_time_features"].to(device),
            future_values=batch["future_values"].to(device),
            past_observed_mask=batch["past_observed_mask"].to(device),
            future_observed_mask=batch["future_observed_mask"].to(device),
        )
        loss = outputs.loss

        # Backpropagation
        accelerator.backward(loss)
        optimizer.step()

        if idx % 100 == 0:
            print(loss.item())

Error message: 
--------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
Input In [64], in <cell line: 18>()
     19 for idx, batch in enumerate(train_dataloader):
     20     optimizer.zero_grad()
---> 21     outputs = model(
     22         static_categorical_features=batch["static_categorical_features"]
     23         if config.num_static_categorical_features > 0
     24         else None,
     25         static_real_features=batch["static_real_features"]
     26         if config.num_static_real_features > 0
     27         else None,
     28         past_time_features=batch["past_time_features"],
     29         past_values=batch["past_values"],
     30         future_time_features=batch["future_time_features"],
     31         future_values=batch["future_values"],
     32         past_observed_mask=batch["past_observed_mask"],
     33         future_observed_mask=batch["future_observed_mask"],
     34     )
     35     loss = outputs.loss
     37     # Backpropagation

File ~/.local/lib/python3.8/site-packages/torch/nn/modules/module.py:1194, in Module._call_impl(self, *input, **kwargs)
   1190 # If we don't have any hooks, we want to skip the rest of the logic in
   1191 # this function, and just call forward.
   1192 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1193         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1194     return forward_call(*input, **kwargs)
   1195 # Do not call functions when jit is used
   1196 full_backward_hooks, non_full_backward_hooks = [], []

File ~/.local/lib/python3.8/site-packages/transformers/models/time_series_transformer/modeling_time_series_transformer.py:1603, in TimeSeriesTransformerForPrediction.forward(self, past_values, past_time_features, past_observed_mask, static_categorical_features, static_real_features, future_values, future_time_features, future_observed_mask, decoder_attention_mask, head_mask, decoder_head_mask, cross_attn_head_mask, encoder_outputs, past_key_values, output_hidden_states, output_attentions, use_cache, return_dict)
   1600 if future_values is not None:
   1601     use_cache = False
-> 1603 outputs = self.model(
   1604     past_values=past_values,
   1605     past_time_features=past_time_features,
   1606     past_observed_mask=past_observed_mask,
   1607     static_categorical_features=static_categorical_features,
   1608     static_real_features=static_real_features,
   1609     future_values=future_values,
   1610     future_time_features=future_time_features,
   1611     decoder_attention_mask=decoder_attention_mask,
   1612     head_mask=head_mask,
   1613     decoder_head_mask=decoder_head_mask,
   1614     cross_attn_head_mask=cross_attn_head_mask,
   1615     encoder_outputs=encoder_outputs,
   1616     past_key_values=past_key_values,
   1617     output_hidden_states=output_hidden_states,
   1618     output_attentions=output_attentions,
   1619     use_cache=use_cache,
   1620     return_dict=return_dict,
   1621 )
   1623 prediction_loss = None
   1624 params = None

File ~/.local/lib/python3.8/site-packages/torch/nn/modules/module.py:1194, in Module._call_impl(self, *input, **kwargs)
   1190 # If we don't have any hooks, we want to skip the rest of the logic in
   1191 # this function, and just call forward.
   1192 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1193         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1194     return forward_call(*input, **kwargs)
   1195 # Do not call functions when jit is used
   1196 full_backward_hooks, non_full_backward_hooks = [], []

File ~/.local/lib/python3.8/site-packages/transformers/models/time_series_transformer/modeling_time_series_transformer.py:1424, in TimeSeriesTransformerModel.forward(self, past_values, past_time_features, past_observed_mask, static_categorical_features, static_real_features, future_values, future_time_features, decoder_attention_mask, head_mask, decoder_head_mask, cross_attn_head_mask, encoder_outputs, past_key_values, output_hidden_states, output_attentions, use_cache, return_dict)
   1421 use_cache = use_cache if use_cache is not None else self.config.use_cache
   1422 return_dict = return_dict if return_dict is not None else self.config.use_return_dict
-> 1424 transformer_inputs, loc, scale, static_feat = self.create_network_inputs(
   1425     past_values=past_values,
   1426     past_time_features=past_time_features,
   1427     past_observed_mask=past_observed_mask,
   1428     static_categorical_features=static_categorical_features,
   1429     static_real_features=static_real_features,
   1430     future_values=future_values,
   1431     future_time_features=future_time_features,
   1432 )
   1434 if encoder_outputs is None:
   1435     enc_input = transformer_inputs[:, : self.config.context_length, ...]

File ~/.local/lib/python3.8/site-packages/transformers/models/time_series_transformer/modeling_time_series_transformer.py:1331, in TimeSeriesTransformerModel.create_network_inputs(self, past_values, past_time_features, static_categorical_features, static_real_features, past_observed_mask, future_values, future_time_features)
   1329     static_feat = torch.cat((static_real_features, static_feat), dim=1)
   1330 if static_categorical_features is not None:
-> 1331     embedded_cat = self.embedder(static_categorical_features)
   1332     static_feat = torch.cat((embedded_cat, static_feat), dim=1)
   1333 expanded_static_feat = static_feat.unsqueeze(1).expand(-1, time_feat.shape[1], -1)

File ~/.local/lib/python3.8/site-packages/torch/nn/modules/module.py:1194, in Module._call_impl(self, *input, **kwargs)
   1190 # If we don't have any hooks, we want to skip the rest of the logic in
   1191 # this function, and just call forward.
   1192 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1193         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1194     return forward_call(*input, **kwargs)
   1195 # Do not call functions when jit is used
   1196 full_backward_hooks, non_full_backward_hooks = [], []

File ~/.local/lib/python3.8/site-packages/transformers/models/time_series_transformer/modeling_time_series_transformer.py:76, in TimeSeriesFeatureEmbedder.forward(self, features)
     72 else:
     73     cat_feature_slices = [features]
     75 return torch.cat(
---> 76     [
     77         embed(cat_feature_slice.squeeze(-1))
     78         for embed, cat_feature_slice in zip(self.embedders, cat_feature_slices)
     79     ],
     80     dim=-1,
     81 )

File ~/.local/lib/python3.8/site-packages/transformers/models/time_series_transformer/modeling_time_series_transformer.py:77, in <listcomp>(.0)
     72 else:
     73     cat_feature_slices = [features]
     75 return torch.cat(
     76     [
---> 77         embed(cat_feature_slice.squeeze(-1))
     78         for embed, cat_feature_slice in zip(self.embedders, cat_feature_slices)
     79     ],
     80     dim=-1,
     81 )

File ~/.local/lib/python3.8/site-packages/torch/nn/modules/module.py:1194, in Module._call_impl(self, *input, **kwargs)
   1190 # If we don't have any hooks, we want to skip the rest of the logic in
   1191 # this function, and just call forward.
   1192 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1193         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1194     return forward_call(*input, **kwargs)
   1195 # Do not call functions when jit is used
   1196 full_backward_hooks, non_full_backward_hooks = [], []

File ~/.local/lib/python3.8/site-packages/torch/nn/modules/sparse.py:160, in Embedding.forward(self, input)
    159 def forward(self, input: Tensor) -> Tensor:
--> 160     return F.embedding(
    161         input, self.weight, self.padding_idx, self.max_norm,
    162         self.norm_type, self.scale_grad_by_freq, self.sparse)

File ~/.local/lib/python3.8/site-packages/torch/nn/functional.py:2210, in embedding(input, weight, padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse)
   2204     # Note [embedding_renorm set_grad_enabled]
   2205     # XXX: equivalent to
   2206     # with torch.no_grad():
   2207     #   torch.embedding_renorm_
   2208     # remove once script supports set_grad_enabled
   2209     _no_grad_embedding_renorm_(weight, input, max_norm, norm_type)
-> 2210 return torch.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse)

RuntimeError: Expected all tensors to be on the same device, but found at least two devices, cuda:0 and cpu! (when checking argument for argument index in method wrapper__index_select)


hmm.. no I guess now things are on different devices... ok perhaps try to train with the number of categoricals set to 0 (i.e. no categorical features...)


When changing num_static_categorical_features from 1 to 0 in the config code, the model was able to train and test successfully. Have not checked quality of run as it was a brief train for testing purposes.

config = TimeSeriesTransformerConfig(
    prediction_length=prediction_length,
    # context length:
    context_length=prediction_length * 2,
    # lags coming from helper given the freq:
    lags_sequence=lags_sequence,
    # we'll add 2 time features ("month of year" and "age", see further):
    num_time_features=len(time_features) + 1,
    # we have a single static categorical feature, namely time series ID:
    **num_static_categorical_features=0,**
    # it has 366 possible values:
    cardinality=[len(train_dataset)],
    # the model will learn an embedding of size 2 for each of the 366 possible values:
    embedding_dimension=[2],
    
    # transformer params:
    encoder_layers=4,
    decoder_layers=4,
    d_model=32,
)

model = TimeSeriesTransformerForPrediction(config)

ok cool then my suspicion was correct... the categorical ids in the train and test set need to have some cardinality... and it is this cardinality which is in the config...  it seems there is some categorical id which is larger than the cardinality you gave the model when you configured it

Hmm so I guess I am a bit confused on that conclusion. The train and test sets have the same IDs and match 1-1.

`train_df.item_id.unique()`
array(['UPC1', 'UPC10', 'UPC1060910111', 'UPC1061962711', 'UPC1060180111',
       'UPC1090018031', ...

`test_df.item_id.unique()`
array(['UPC1', 'UPC10', 'UPC1060910111', 'UPC1061962711', 'UPC1060180111',
       'UPC1090018031', ...

I ensured this as I have the following code as well
`test_df = test_df[test_df.item_id.isin(train_df.item_id.unique())]`

Is it that they are not able to be strings?

Still after running the following code to reassign the strings to numbers, I still get the original error.
```
train_classes = np.array(train_df["item_id"])
train_classnames, train_indices = np.unique(train_classes, return_inverse=True)

test_classes = np.array(test_df["item_id"])
test_classnames, test_indices = np.unique(test_classes, return_inverse=True)

train_df["item_id"] = train_indices
test_df["item_id"] = test_indices
```

`train_df.item_id.unique()`
array([  0,   1,   2,   3,   4,   5,   6,   7,   8,   9,  10,  11,  12,
        13,  14,  15,  16,  17,  18,  19,  20,  21,  22,  23,  24,  25,
        26,  27,  28,  29,  30,  31,  32,  33,  34,  35,  36,  37,  38,
        39,  40,  41,  42,  43,  44,  45,  46,  47,  48,  49,  50,  51,
        52,  53,  54,  55,  56,  57,  58,  59,  60,  61,  62,  63,  64,
        65,  66,  67,  68,  69,  70,  71,  72,  73,  74,  75,  76,  77,
        78,  79,  80,  81,  82,  83,  84,  85,  86,  87,  88,  89,  90,
        91,  92,  93,  94,  95,  96,  97,  98,  99, 100, 101, 102, 103,
       104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116,
       117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129,
       130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142,
       143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155,
       156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168,
       169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181,
       182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194,
       195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207,
       208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220,
       221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233,
       234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246,
       247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259,
       260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272,
       273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285,
       286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298,
       299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311,
       312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324,
       325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337,
       338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350,
       351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363,
       364, 365])

`test_df.item_id.unique()`
array([  0,   1,   2,   3,   4,   5,   6,   7,   8,   9,  10,  11,  12,
        13,  14,  15,  16,  17,  18,  19,  20,  21,  22,  23,  24,  25,
        26,  27,  28,  29,  30,  31,  32,  33,  34,  35,  36,  37,  38,
        39,  40,  41,  42,  43,  44,  45,  46,  47,  48,  49,  50,  51,
        52,  53,  54,  55,  56,  57,  58,  59,  60,  61,  62,  63,  64,
        65,  66,  67,  68,  69,  70,  71,  72,  73,  74,  75,  76,  77,
        78,  79,  80,  81,  82,  83,  84,  85,  86,  87,  88,  89,  90,
        91,  92,  93,  94,  95,  96,  97,  98,  99, 100, 101, 102, 103,
       104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116,
       117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129,
       130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142,
       143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155,
       156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168,
       169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181,
       182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194,
       195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207,
       208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220,
       221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233,
       234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246,
       247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259,
       260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272,
       273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285,
       286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298,
       299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311,
       312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324,
       325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337,
       338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350,
       351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363,
       364, 365])

ok sorry for the confusion! `item_id` can be a string however the `static_categorical_features` for each of the time series can potentially contain the corresponding integer id, e.g. for the very first one `static_categorical_features = [0]` etc. and thus the `cardinality = [366]` and you can specify the embedding vector dimension... can you confirm that the `cardinality` is correct? 

Ahh you solved it! I made a mistake thinking the item_id was utilized as the static_categorical_features. You are absolultey correct and found the issue. Here was the code causing the issue:
 
```
class ProcessStartField():
    ts_id = 0
    
    def __call__(self, data):
        data["start"] = data["start"].to_timestamp()
        data["feat_static_cat"] = [self.ts_id]
        self.ts_id += 1
        
        return data
```

```
from gluonts.itertools import Map

process_start = ProcessStartField()

list_ds_train = list(Map(process_start, ds_train))
list_ds_test = list(Map(process_start, ds_test))
```

```
from datasets import Dataset, Features, Value, Sequence

features  = Features(
    {    
        "start": Value("timestamp[s]"),
        "target": Sequence(Value("float32")),
        "feat_static_cat": Sequence(Value("uint64")),
        # "feat_static_real":  Sequence(Value("float32")),
        # "feat_dynamic_real": Sequence(Sequence(Value("uint64"))),
        # "feat_dynamic_cat": Sequence(Sequence(Value("uint64"))),
        "item_id": Value("string"),
    }
)
```

```
train_dataset = Dataset.from_list(list_ds_train, features=features)
test_dataset = Dataset.from_list(list_ds_test, features=features)
```

This was causing the train_dataset to get static_categorical_features values from 0-365, and the test_dataset was then getting values from 366-731. I corrected the code to now perform the mapping process individually for each dataset with the function so that it would not begin from the last number. Thanks so much for all your help!