Issue: Calling `cast_column`/`remove_columns` and a sequence of `map` operations ends up making `faiss` fail with `ValueError`

Comment:
Here I'm presenting all the scenarios so that you can further investigate the issue:

- ✅ `cast_column` -> `map` with `remove_columns` -> `map` with `remove_columns` -> `add_faiss_index`

  ```python
  from transformers import DPRContextEncoder, DPRContextEncoderTokenizer
  import torch
  
  torch.set_grad_enabled(False)
  ctx_encoder = DPRContextEncoder.from_pretrained("facebook/dpr-ctx_encoder-single-nq-base")
  ctx_tokenizer = DPRContextEncoderTokenizer.from_pretrained("facebook/dpr-ctx_encoder-single-nq-base")
  
  from datasets import load_dataset, Value
  
  ds = load_dataset("csv", data_files=["sample.csv"], split="train")
  ds = ds.cast_column("id", Value("int32"))
  ds = ds.map(lambda x: {"inputs": f"{ctx_tokenizer.sep_token}".join(["title", "summary"])}, remove_columns=["title", "summary"])
  
  def generate_embeddings(x):
      return {"embeddings": ctx_encoder(**ctx_tokenizer(x["inputs"], return_tensors="pt"))[0][0].numpy()}
  
  ds = ds.map(generate_embeddings, remove_columns=["inputs"])
  ds.add_faiss_index(column="embeddings")
  ```

- ❌ `cast_column` -> `map` -> `remove_columns` -> `map` -> `remove_columns` -> `add_faiss_index`

  ```python
  from transformers import DPRContextEncoder, DPRContextEncoderTokenizer
  import torch
  
  torch.set_grad_enabled(False)
  ctx_encoder = DPRContextEncoder.from_pretrained("facebook/dpr-ctx_encoder-single-nq-base")
  ctx_tokenizer = DPRContextEncoderTokenizer.from_pretrained("facebook/dpr-ctx_encoder-single-nq-base")
  
  from datasets import load_dataset, Value
  
  ds = load_dataset("csv", data_files=["sample.csv"], split="train")
  ds = ds.cast_column("id", Value("int32"))
  ds = ds.map(lambda x: {"inputs": f"{ctx_tokenizer.sep_token}".join(["title", "summary"])})
  ds = ds.remove_columns(["title", "summary"])
  
  def generate_embeddings(x):
      return {"embeddings": ctx_encoder(**ctx_tokenizer(x["inputs"], return_tensors="pt"))[0][0].numpy()}
  
  ds = ds.map(generate_embeddings)
  ds = ds.remove_columns(["inputs"])
  ds.add_faiss_index(column="embeddings")
  ```

- ❌ `cast_column` -> `map` with `remove_columns` -> `map` -> `remove_columns` -> `add_faiss_index`


  ```python
  from transformers import DPRContextEncoder, DPRContextEncoderTokenizer
  import torch
  
  torch.set_grad_enabled(False)
  ctx_encoder = DPRContextEncoder.from_pretrained("facebook/dpr-ctx_encoder-single-nq-base")
  ctx_tokenizer = DPRContextEncoderTokenizer.from_pretrained("facebook/dpr-ctx_encoder-single-nq-base")
  
  from datasets import load_dataset, Value
  
  ds = load_dataset("csv", data_files=["sample.csv"], split="train")
  ds = ds.cast_column("id", Value("int32"))
  ds = ds.map(lambda x: {"inputs": f"{ctx_tokenizer.sep_token}".join(["title", "summary"])}, remove_columns=["title", "summary"])
  
  def generate_embeddings(x):
      return {"embeddings": ctx_encoder(**ctx_tokenizer(x["inputs"], return_tensors="pt"))[0][0].numpy()}
  
  ds = ds.map(generate_embeddings)
  ds = ds.remove_columns(["inputs"])
  ds.add_faiss_index(column="embeddings")
  ```

- ✅ `cast_column` -> `map` -> `remove_columns` -> `map` with `remove_columns` -> `add_faiss_index`


  ```python
  from transformers import DPRContextEncoder, DPRContextEncoderTokenizer
  import torch
  
  torch.set_grad_enabled(False)
  ctx_encoder = DPRContextEncoder.from_pretrained("facebook/dpr-ctx_encoder-single-nq-base")
  ctx_tokenizer = DPRContextEncoderTokenizer.from_pretrained("facebook/dpr-ctx_encoder-single-nq-base")
  
  from datasets import load_dataset, Value
  
  ds = load_dataset("csv", data_files=["sample.csv"], split="train")
  ds = ds.cast_column("id", Value("int32"))
  ds = ds.map(lambda x: {"inputs": f"{ctx_tokenizer.sep_token}".join(["title", "summary"])})
  ds = ds.remove_columns(["title", "summary"])
  
  def generate_embeddings(x):
      return {"embeddings": ctx_encoder(**ctx_tokenizer(x["inputs"], return_tensors="pt"))[0][0].numpy()}
  
  ds = ds.map(generate_embeddings, remove_columns=["inputs"])
  ds.add_faiss_index(column="embeddings")
  ```

- ✅ `map` -> `remove_columns` -> `map` -> `remove_columns` -> `add_faiss_index`


  ```python
  from transformers import DPRContextEncoder, DPRContextEncoderTokenizer
  import torch
  
  torch.set_grad_enabled(False)
  ctx_encoder = DPRContextEncoder.from_pretrained("facebook/dpr-ctx_encoder-single-nq-base")
  ctx_tokenizer = DPRContextEncoderTokenizer.from_pretrained("facebook/dpr-ctx_encoder-single-nq-base")
  
  from datasets import load_dataset, Value
  
  ds = load_dataset("csv", data_files=["sample.csv"], split="train")
  ds = ds.map(lambda x: {"inputs": f"{ctx_tokenizer.sep_token}".join(["title", "summary"])})
  ds = ds.remove_columns(["title", "summary"])
  
  def generate_embeddings(x):
      return {"embeddings": ctx_encoder(**ctx_tokenizer(x["inputs"], return_tensors="pt"))[0][0].numpy()}
  
  ds = ds.map(generate_embeddings)
  ds = ds.remove_columns(["inputs"])
  ds.add_faiss_index(column="embeddings")
  ```