ArrowNotImplementedError when loading json dataset

Question:
Hello community,
When trying to load custom json dataset based on wikipedia dump:
from datasets import load_dataset

wiki_fr_dataset_lp = load_dataset("json", data_files="/media/matthieu/HDD_4T0/Github/semantic-search-through-wikipedia-with-weaviate/step-1/articles.json", split="train")

I got the following error:
---------------------------------------------------------------------------
ArrowNotImplementedError                  Traceback (most recent call last)
<ipython-input-1-f8ba9f3864f3> in <module>
      1 from datasets import load_dataset
      2
----> 3 wiki_fr_dataset_lp = load_dataset("json", data_files="/media/matthieu/HDD_4T0/Github/semantic-search-through-wikipedia-with-weaviate/step-1/articles.json", split="train")
      4 wiki_fr_dataset_lp

~/anaconda3/envs/sts-transformers-gpu-fresh/lib/python3.8/site-packages/datasets/load.py in load_dataset(path, name, data_dir, data_files, split, cache_dir, features, download_config, download_mode, ignore_verifications, keep_in_memory, save_infos, revision, use_auth_token, task, streaming, script_version, **config_kwargs)
   1630
   1631     # Download and prepare data
-> 1632     builder_instance.download_and_prepare(
   1633         download_config=download_config,
   1634         download_mode=download_mode,

~/anaconda3/envs/sts-transformers-gpu-fresh/lib/python3.8/site-packages/datasets/builder.py in download_and_prepare(self, download_config, download_mode, ignore_verifications, try_from_hf_gcs, dl_manager, base_path, use_auth_token, **download_and_prepare_kwargs)
    605                             logger.warning("HF google storage unreachable. Downloading and preparing it from source")
    606                     if not downloaded_from_gcs:
--> 607                         self._download_and_prepare(
    608                             dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
    609                         )

~/anaconda3/envs/sts-transformers-gpu-fresh/lib/python3.8/site-packages/datasets/builder.py in _download_and_prepare(self, dl_manager, verify_infos, **prepare_split_kwargs)
    695             try:
    696                 # Prepare split will record examples associated to the split
--> 697                 self._prepare_split(split_generator, **prepare_split_kwargs)
    698             except OSError as e:
    699                 raise OSError(

~/anaconda3/envs/sts-transformers-gpu-fresh/lib/python3.8/site-packages/datasets/builder.py in _prepare_split(self, split_generator)
   1157                 generator, unit=" tables", leave=False, disable=True  # bool(logging.get_verbosity() == logging.NOTSET)
   1158             ):
-> 1159                 writer.write_table(table)
   1160             num_examples, num_bytes = writer.finalize()
   1161

~/anaconda3/envs/sts-transformers-gpu-fresh/lib/python3.8/site-packages/datasets/arrow_writer.py in write_table(self, pa_table, writer_batch_size)
    440         # reorder the arrays if necessary + cast to self._schema
    441         # we can't simply use .cast here because we may need to change the order of the columns
--> 442         pa_table = pa.Table.from_arrays([pa_table[name] for name in self._schema.names], schema=self._schema)
    443         batches: List[pa.RecordBatch] = pa_table.to_batches(max_chunksize=writer_batch_size)
    444         self._num_bytes += sum(batch.nbytes for batch in batches)

~/anaconda3/envs/sts-transformers-gpu-fresh/lib/python3.8/site-packages/pyarrow/table.pxi in pyarrow.lib.Table.from_arrays()

~/anaconda3/envs/sts-transformers-gpu-fresh/lib/python3.8/site-packages/pyarrow/table.pxi in pyarrow.lib._sanitize_arrays()

~/anaconda3/envs/sts-transformers-gpu-fresh/lib/python3.8/site-packages/pyarrow/array.pxi in pyarrow.lib.asarray()

~/anaconda3/envs/sts-transformers-gpu-fresh/lib/python3.8/site-packages/pyarrow/table.pxi in pyarrow.lib.ChunkedArray.cast()

~/anaconda3/envs/sts-transformers-gpu-fresh/lib/python3.8/site-packages/pyarrow/compute.py in cast(arr, target_type, safe)
    295     else:
    296         options = CastOptions.unsafe(target_type)
--> 297     return call_function("cast", [arr], options)
    298
    299

~/anaconda3/envs/sts-transformers-gpu-fresh/lib/python3.8/site-packages/pyarrow/_compute.pyx in pyarrow._compute.call_function()

~/anaconda3/envs/sts-transformers-gpu-fresh/lib/python3.8/site-packages/pyarrow/_compute.pyx in pyarrow._compute.Function.call()

~/anaconda3/envs/sts-transformers-gpu-fresh/lib/python3.8/site-packages/pyarrow/error.pxi in pyarrow.lib.pyarrow_internal_check_status()

~/anaconda3/envs/sts-transformers-gpu-fresh/lib/python3.8/site-packages/pyarrow/error.pxi in pyarrow.lib.check_status()

ArrowNotImplementedError: Unsupported cast from struct<title: string, content: string, count: int64> to struct using function cast_struct

Anyone would have an advice on this error?
Thanks!

Context:
Hi,
it’s hard to debug these kinds of errors without having access to data. I’d assume that in some examples (JSON lines) some fields, or subfields, are missing based on the error message. Also, you can try to increase the chunksize 2 argument and see if that works.


[Solved] Image dataset seems slow for larger image size

Question:
Hi,
Recently I tried to train an image captioning flax model with TPU, and I found it is very slow → each batch (256 images) requires 26 seconds. I finally found the cause:
My dataset have a column named “image” and it stores image array (being cached).
However, operations like batch = dataset[idx], where idx is a list of length 256 will take 15 seconds, if the image size is (256 , 256). If I store the flattened image array (i.e. 1D array as length 256 * 256 * 3), it will be faster.
The provided script demonstrate this situation, with image/batch size 128.
I am wondering if I am doing something wrong, and if there is a better way to handle image datasets.
import numpy as np
import time

from datasets import load_dataset, Dataset

ds = load_dataset("cifar10")
train_ds = ds["train"]
# Take 512 examples
train_ds = train_ds.select(range(512))

# `_images` has shape (batch_size, 32, 32, 3)
# Let's make large images (batch_size, 32 * FACTOR, 32 * FACTOR, 3)
FACTOR = 4

def preprocess_function(examples):

    inputs = {}
    images = examples["img"]

    # convert to np.float3f2 array
    _images = np.array(images, dtype=np.float32)

    # Make image larger
    batch_size, image_size = _images.shape[0:2]

    image_size = image_size * FACTOR

    # use (H, W, C) format
    _images = np.concatenate([_images] * FACTOR ** 2, axis=0).reshape((batch_size, image_size, image_size, 3))
    # flatten images to 1D -> `data_loader` will be faster
    # _images = _images.reshape((batch_size, image_size * image_size * 3))

    inputs["image"] = [x for x in _images]
    inputs["label"] = examples["label"]

    return inputs

times_1 = []
times_2 = []

def data_loader(rng: np.random.Generator, dataset: Dataset, batch_size: int, shuffle: bool = False):
    """
    Returns batches of size `batch_size` from truncated `dataset`, sharded over all local devices.
    Shuffle batches if `shuffle` is `True`.
    """

    steps_per_epoch = len(dataset) // batch_size

    if shuffle:
        batch_idx = rng.permutation(len(dataset))
    else:
        batch_idx = range(len(dataset))

    batch_idx = batch_idx[: steps_per_epoch * batch_size]  # Skip incomplete batch.
    batch_idx = batch_idx.reshape((steps_per_epoch, batch_size))

    for idx in batch_idx:

        s = time.time()
        # (bs=128, img_size=128)
        #   - (H, W, C) -> takes 6.959 seconds
        #   - flatten   -> takes 2.407 seconds
        batch = dataset[idx]
        e = time.time()
        times_1.append(e-s)

        s = time.time()
        # (bs=128, img_size=128)
        #   - (H, W, C) -> takes 1.661 seconds
        #   - flatten   -> takes 0.673 seconds
        batch = {k: np.array(v) for k, v in batch.items()}
        e = time.time()
        times_2.append(e-s)

        yield batch

# The results are cached
train_ds = train_ds.map(
    preprocess_function,
    batched=True,
    batch_size=16,
    num_proc=2,
)

# Create sampling rng
input_rng = np.random.default_rng(seed=42)
train_loader = data_loader(input_rng, train_ds, batch_size=128, shuffle=True)

start = time.time()
for idx, batch in enumerate(train_loader):

    end = time.time()
    print(f"batch: {idx} | time: {end - start}")
    start = time.time()

print(f"Average times_1 in data_loader: {np.mean(times_1)}")
print(f"Average times_2 in data_loader: {np.mean(times_2)}")

Context:
Hi !
There are two optimizations that you can use:

Use the Array3D feature type. Otherwise it will consider your arrays to be lists of arbitrary sizes, and it takes some time to collate them again into one numpy array.

features = Features({
    **train_ds.features,
    "image": Array3D(dtype="int32", shape=(128, 128, 3))
})
train_ds = train_ds.map(
    preprocess_function,
    batched=True,
    batch_size=16,
    num_proc=2,
    features=features
)

Use the ‘numpy’ format. This way when accessing examples they will already be numpy arrays, and you will also benefit from the end-to-end zero-copy array read from Arrow to numpy.

train_ds = train_ds.with_format("numpy")

It becomes 100 times faster when querying the data loader !
Here are my results without the optimizations:
Average times_1 in data_loader: 7.690094709396362

And then, with these optimizations:
Average times_1 in data_loader: 0.0582084059715271


Question:
I am training ncbi_disease dataset 1 using transformers trainer.
Here are the features of the dataset as follows

DatasetDict({
train: Dataset({
features: [‘id’, ‘tokens’, ‘ner_tags’],
num_rows: 5433
})
validation: Dataset({
features: [‘id’, ‘tokens’, ‘ner_tags’],
num_rows: 924
})
test: Dataset({
features: [‘id’, ‘tokens’, ‘ner_tags’],
num_rows: 941
})
})

This is an output to a sample of training data tuple as follows
{'id': '20',
 'ner_tags': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 2, 0, 0, 0],
 'tokens': ['For',
  'both',
  'sexes',
  'combined',
  ',',
  'the',
  'penetrances',
  'at',
  'age',
  '60',
  'years',
  'for',
  'all',
  'cancers',
  'and',
  'for',
  'colorectal',
  'cancer',
  'were',
  '0',
  '.']}

Here is the function for tokenization and I get this error

ArrowInvalid: Column 1 named id expected length 512 but got length 1000

def tokenize_text(examples):

    return tokenizer(str(examples["tokens"]),truncation=True,max_length=512)

dataset=dataset.map(tokenize_text,batched=True)

Any clue how to solve this problem?

Context:
Hey @ghadeermobasher , This has been explained in chapter 5 of the course (The 🤗 Datasets library - Hugging Face Course 1). Scroll down a bit and you will find a similar error with the explanation.
You need to modify your “tokenize_text” function as such:
def tokenize_text(examples):
    result = tokenizer(str(examples["tokens"]),truncation=True,
                       max_length=512, return_overflowing_tokens=True)

    sample_map = result.pop("overflow_to_sample_mapping")
    for key, values in examples.items():
        result[key] = [values[i] for i in sample_map]
    return result


Fastest way to do inference on a large dataset in huggingface?

Question:
Hello there!
Here is my issue. I have a model trained with huggingface (yay) that works well. Now I need to perform inference and compute predictions on a very large dataset of several millions of observations.
What is the best way to proceed here?  Should I write the prediction loop myself? Which routines in datasets should be useful here? My computer has lots of RAM (100GB+), 20+ cpus and a big GPU card.
Thanks!

Context:
Hi ! If you have enough RAM I guess you can use any tool to load your data.
However if your dataset doesn’t fit in RAM I’d suggest to use the datasets, since it allows to load datasets without filling up your RAM and gives excellent performance.
Then I guess you can write your prediction loop yourself. Just make sure to pass batches of data to your model to make sure that your GPU is fully utilized. You can also use my_dataset.map() with batched=True and set batch_size to a reasonable value


New dataset added_review for improvement

Question:
I have recently created a dataset called “english-quotes”.
The data collection process is web scraping using BeautifulSoup and Requests libraries. I also added the script that I created to scrape data and create the dataset  in the Card Description (under " Who are the source Data producers ?")
It’s just a start. I wanted to validate (for myself) the “Datasets” chapter of the course by a final self-evaluation different from the dataset of the course “github_issues” (I admit, I changed the sections of the data card a bit). If you have any remarks or comments for improvement, please let me know (as I will add more advanced datasets in the future)… and thanks.

the link of the new dataset added “english_quotes”: Abirate/english_quotes · Datasets at Hugging Face 1

Context:
This is awesome ! Thanks for adding this dataset
Do you know if there’s a list somewhere of all the possible tags ? It can be useful to know how many classes there are to train multi class classification models.


How do I add features on a local dataset

Question: Hey, I was reading  Create a dataset loading script — datasets 1.16.1 documentation 2 and I am confused about how to add attributes to my local dataset. Can anyone explain how do I do this on a local dataset?

Context:
Without it I am getting:-
  File "/opt/conda/lib/python3.7/collections/__init__.py", line 1027, in __
getitem__
    raise KeyError(key)
KeyError: 'premise'


Unable to load CommonVoice latest version

Question:
This line of code works well with commonvoice version 6.1.0  but i get error while using for version .70.0
from datasets import load_dataset, load_metric, Audio

common_voice_train = load_dataset("common_voice", "lg", split="train+validation",version="7.0.0")
common_voice_test = load_dataset("common_voice", "lg", split="test",version="7.0.0")

ValueError: Cannot name a custom BuilderConfig the same as an available BuilderConfig. Change the name. Available BuilderConfigs: ['ab', 'ar', 'as', 'br', 'ca', 'cnh', 'cs', 'cv', 'cy', 'de', 'dv', 'el', 'en', 'eo', 'es', 'et', 'eu', 'fa', 'fi', 'fr', 'fy-NL', 'ga-IE', 'hi', 'hsb', 'hu', 'ia', 'id', 'it', 'ja', 'ka', 'kab', 'ky', 'lg', 'lt', 'lv', 'mn', 'mt', 'nl', 'or', 'pa-IN', 'pl', 'pt', 'rm-sursilv', 'rm-vallader', 'ro', 'ru', 'rw', 'sah', 'sl', 'sv-SE', 'ta', 'th', 'tr', 'tt', 'uk', 'vi', 'vot', 'zh-CN', 'zh-HK', 'zh-TW']

what do i need to change. Thanks

Context: cc @anton-l


Can we upload datasets with a total size like the pile?

Question:
Hi!
We want to create a very large text dataset inspired by the pile. To distribute the dataset we would like to upload the dataset on datasets of hugginface. But we were wondering if there any limitations in sizes for uploading datasets?
Thank you for your responses!

Context:
Hi!
That’s so cool! No, we don’t have any limitations in terms of size. As the dataset you are working on is pretty big, would you be interested in collaborating with us? This way it will be easier for us to help you. Also, could you tell us a bit more about your project? Feel free to let me know here or via e-mail (mario@huggingface.co).
cc @thomwolf


Item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()} AttributeError: ‘list’ object has no attribute ‘items’

Question:
I am trying to classify a file using the model obtained from finetuning bertsequenceclassification.
I get this error when loading the new test file (it does not have label), it is unknown file.
# Create torch dataset
class Dataset(torch.utils.data.Dataset):
	def __init__(self, encodings, labels=None):
		self.encodings = encodings
		self.labels = labels

	def __getitem__(self, idx):
		item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
		if self.labels:
			item["labels"] = torch.tensor(self.labels[idx])
			#print(item)
		return item

	def __len__(self):
		print(len(self.encodings["input_ids"]))
		return len(self.encodings["input_ids"])

# prepare dat for classification

tokenizer = XXXTokenizer.from_pretrained(model_name)
	print("Transform xml file to pandas series core...")
	text, file_name = transform_xml_to_pd(file)  # transform xml file to pd

	# Xtest_emb, s = get_xxx_layer(Xtest['sent'], path_to_model_lge)  # index 2 correspond to sentences
	#print(text)

	print("Preprocess text with spacy model...")
	clean_text = make_new_traindata(text['sent'])
	#print(clean_text[1])  # clean text ; 0 = raw text ; and etc...

	X = list(clean_text)
	X_text_tokenized = []

	for x in X:
		#print(type(x))
		x_encoded = tokenizer(str(x), padding="max_length", truncation=True, max_length=512)
		#print(type(x_encoded))
		#print(x_encoded)
		X_text_tokenized.append(x_encoded)

	#print(type(X_text_tokenized))

	X_data = Dataset(X_text_tokenized)

	print(type(X_data))
	print(X_data['input_ids'])

Error

  File "/scriptTraitements/classifying.py", line 153, in __getitem__
    item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
AttributeError: 'list' object has no attribute 'items'

Context:
Hi,
this line is the problem:
X_data = Dataset(X_text_tokenized)

because X_text_tokenized is a list of dictionaries and not a dictionary of lists. You can fix this with the following code:
def list_of_dicts_to_dict_of_lists(d):
    dic = d[0]
    keys = dic.keys()
    values = [dic.values() for dic in d]
    return {k: list(v) for k, v in zip(keys, zip(*values))}

X_data = Dataset(list_of_dicts_to_dict_of_lists(X_text_tokenized))


Question:
Hi there,
I prepared my data into a DatasetDict object that I saved to disk with the save_to_disk method. I’d like to upload the generated folder to the HuggingFace Hub and use it using the usual load_dataset function. Though, I have not yet found a way to do so. Is this possible?
Thanks a lot in advance for your help.
Best,
Pietro

Context:
Hi,
this week’s release of datasets will add support for directly pushing a Dataset/DatasetDict object to the Hub. In the meantime, you can use a to_{format} method, where format is one of ["csv", "json", "txt", "parquet"] on each split of the DatasetDict object and push the generated files to the Hub (follow the docs here 1 for more information). Also note that this requires the master version of the library, which you can install with:
pip install git+https://github.com/huggingface/datasets.git

Without the master version, you’ll have to specify a list of files to load each split separately (docs on that are here 1).


Error when fine-tuning with the Trainer API

Question:
Hi,
I’m fine-tuning a model with the Trainer API and following these instructions: https://huggingface.co/docs/transformers/training#finetuning-in-pytorch-with-the-trainer-api 3
However, after I have defined the compute_metrics function and tried to run the script, it gave me the following error:
Traceback (most recent call last):
  File "/home/le/torch_tutorial/lm_1_perpl.py", line 77, in <module>
    trainer.train()
  File "/home/le/torch_tutorial/venv/lib/python3.9/site-packages/transformers/trainer.py", line 1391, in train
    self._maybe_log_save_evaluate(tr_loss, model, trial, epoch, ignore_keys_for_eval)
  File "/home/le/torch_tutorial/venv/lib/python3.9/site-packages/transformers/trainer.py", line 1491, in _maybe_log_save_evaluate
    metrics = self.evaluate(ignore_keys=ignore_keys_for_eval)
  File "/home/le/torch_tutorial/venv/lib/python3.9/site-packages/transformers/trainer.py", line 2113, in evaluate
    output = eval_loop(
  File "/home/le/torch_tutorial/venv/lib/python3.9/site-packages/transformers/trainer.py", line 2354, in evaluation_loop
    metrics = self.compute_metrics(EvalPrediction(predictions=all_preds, label_ids=all_labels))
  File "/home/le/torch_tutorial/lm_1_perpl.py", line 67, in compute_metrics
    return metric.compute(predictions=predictions, references=labels)
  File "/home/le/torch_tutorial/venv/lib/python3.9/site-packages/datasets/metric.py", line 393, in compute
    self.add_batch(predictions=predictions, references=references)
  File "/home/le/torch_tutorial/venv/lib/python3.9/site-packages/datasets/metric.py", line 434, in add_batch
    batch = self.info.features.encode_batch(batch)
  File "/home/le/torch_tutorial/venv/lib/python3.9/site-packages/datasets/features/features.py", line 1049, in encode_batch
    encoded_batch[key] = [encode_nested_example(self[key], obj) for obj in column]
  File "/home/le/torch_tutorial/venv/lib/python3.9/site-packages/datasets/features/features.py", line 1049, in <listcomp>
    encoded_batch[key] = [encode_nested_example(self[key], obj) for obj in column]
  File "/home/le/torch_tutorial/venv/lib/python3.9/site-packages/datasets/features/features.py", line 853, in encode_nested_example
    return schema.encode_example(obj)
  File "/home/le/torch_tutorial/venv/lib/python3.9/site-packages/datasets/features/features.py", line 297, in encode_example
    return int(value)
TypeError: only size-1 arrays can be converted to Python scalars

Do you have any ideas on what can be causing it? I have not changed anything in my code except for adding the compute_metrics function (like in the tutorial) and adding the compute_metrics argument in Trainer (before this addition everything was working perfectly):
def compute_metrics(eval_pred):
    logits, labels = eval_pred
    predictions = np.argmax(logits, axis=-1)
    return metric.compute(predictions=predictions, references=labels)

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=lm_datasets["train"],
    eval_dataset=lm_datasets["validation"],
    compute_metrics=compute_metrics,
)

Context: What I would do in this case is simply add a print statement within the compute_metrics function to view your logits and labels, their shapes, et.


[Semantic search with FAISS] Can’t manage to format embeddings column to numpy format

Question:
Hello,
I would like to test the semantic search with FAISS according the HF course but on wikipedia dataset.
I use a local docker with sentence-transformer algorithm to compute embeddings for each paragraph, according following function:
def get_embeddings(text_list):
    payload_resp = requests.request("POST", api_sent_embed_url, data=json.dumps(text_list))
    return np.array(json.loads(payload_resp.content.decode("utf-8")), dtype=np.float32)

Using this function I managed to obtain numpy array on output on those test data:
payload1 = ["Navigateur Web : Ce logiciel permet d'accéder à des pages web depuis votre ordinateur. Il en existe plusieurs téléchargeables gratuitement comme Google Chrome ou Mozilla. Certains sont même déjà installés comme Safari sur Mac OS et Edge sur Microsoft."]
payload2 = ["Google Chrome. Mozilla. Safari."]
payload = payload1 + payload2

embedding = get_embeddings(payload)
embedding.shape

(2, 384)
However, when trying to apply this on my embeddings_dataset
Dataset({
    features: ['id', 'revid', 'text', 'title', 'url'],
    num_rows: 100
})

I still obtain for the “embeddings” column list format and not numpy:
embeddings_dataset= embeddings_dataset.map(lambda x: {"embeddings": get_embeddings(x["text"])})
type(embeddings_test[0]["embeddings"])

list
Would anyone have an advice regarding this problem?

Context:
Hi,
set format of the embeddings column to NumPy after the map call to get a NumPy array:
embeddings_dataset.set_format("numpy", columns=["embeddings"], output_all_columns=True)
type(embeddings_test[0]["embeddings"])


Question:
I am currently fine-tuning a language model using a policy-gradient reinforcement learning technique. Instead of a standard loss function, I am using a reward function and the REINFORCE algorithm to teach the model to emulate some desired behaviour.
As part of the reward function, I compute the ROUGE score between a reference sentence and a generated one. To do this I’m currently using a list comprehension that zips together two lists of sentences (ref_batch and pred_batch below) and then calculates the rouge score for each.
The code looks something like this:
from datasets import load_metric
rouge_metric = load_metric("rouge")

def get_rouge_score(ref, pred):
    return rouge_metric.compute(rouge_types=["rougeL"], predictions=[pred], references=[ref])['rougeL'].mid.fmeasure

rouge_scores = torch.tensor([get_rouge_score(ref,pred) for ref,pred in zip(ref_batch, pred_batch)], device=device)

The problem with this is that it is very slow. The list comprehension iterates through examples one by one and uses the CPU to do the operation. By contrast, the rest of the training loop runs using tensor cores on a GPU. Hence this step is a significant bottleneck; profiling on the training step shows that this step alone takes up ~60% of the training time.
So my question: how could I parallelize this step, or even make it faster another way? If there’s a way to calculate scores on the GPU, that would be awesome. If not, is there an easy way I can use multiple CPU cores for this?
I’m also able to change the metric from ROUGE to another one that is more able to be parallelized, if that helps.
Thanks

Context:
Hi ! To speed up the processing you can pass keep_in_memory=True to load_metric to keep each sample in memory (by default it writes them on disk to save memory, but since you’re passing the examples one by one to compute you don’t need this). This should speed up your computation significantly.
Moreover feel free to use python multiprocessing to parallelize this (using multiprocessing.Pool and Pool.imap for example)


How to load only test dataset from `librispeech_asr`?

Question:
Hi, I want to evaluate my model with Librespeech dataset

Train.500
Train.360
Train.100
Valid
Test

clean
-
104014
28539
2703
2620

other
148688
-
-
2864
2939

I don’t need train.500, train.360, train.100 and valid set. Is there any way to load only test set from librispeech_asr?
I can load all data and only keep test set later but it takes too long to load all data :(( So I want to load only test set.
from datasets import load_dataset, DatasetDict

# It takes forever to load everything here...
libre_dataset = load_dataset("librispeech_asr", 'clean')

keep = ["test"]
libre_dataset = DatasetDict({k: dataset for k, dataset in libre_dataset .items() if k in keep})

Thanks for reading!

Context:
Hey @IlllIIII you can specify the split you want to load by passing split="test" to the load_dataset() function (docs). This will still download all the splits in the dataset, so if space is an issue you can stream the elements of the test set one by one:
from datasets import load_dataset

dset = load_dataset("librispeech_asr", 'clean', split="test", streaming=True)
next(iter(dset))


Dataset preview not showing for uploaded DatasetDict

Question:
I created a DatasetDict and pushed it here 6.
I’m getting the message
Server Error
Status code:   400
Exception:     Status400Error
Message:       could not get the config name for this dataset

Am I supposed to create a config file somewhere that I missed so the dataset viewer works?
Thanks!

Context:
Hey @dansbecker as far as I know, the dataset viewer currently supports datasets with a loading script (example 1) or raw data files in common formats like JSON and CSV (example 2).
I agree that being able view the contents of DatasetDict objects would be a nice feature, so I’m tagging @lhoestq and @severo  in case they have any additional insights here


How preprocessing with IterableDataset works?

Question: I want to know how functions like map, the filter works with IterableDataset type?

Context:
Hi,
currently, IterableDataset doesn’t support filter (you can use a simple check to filter unwanted examples), and for map see the docs here 2.


Permanently saving dataset with load_dataset

Question:
Hi all,
How do I permanently download the dataset via load_dataset?
When I do this:
# I load almost all glue datasets
dataset = load_dataset("glue", "mrpc")
dataset = load_dataset("glue", "mnli")
...
dataset = load_dataset("glue", "qnli")

It still redownloads some of the datasets after running the code several times. Because its saved in cache.
Is there a way to download them in a way with load_data that it stays there permanently?

Context:
Hi ! Unless you clear the cache, the datasets are only downloaded once.
What makes you think that the datasets are being redownloaded ?


Question: Anyone know of  RoBERTa pretraining script with support for Dataset streadmin?

Context:
Hi ! I don’t think the community has already shared a script for RoBERTa pretraining using dataset streaming yet. However if you’re interested in looking into this, here are a few pointers:
RoBERTa was trained with BookCorpus, CC news and OpenWebText
BookCorpus and OpenWebText have been replicated and open sourced as BookCorpusOpen and OpenWebText2 (The Pile)
You can load and interleave the datasets with
from datasets import load_dataset, interleave_datasets

def only_keep_text(example):
    return {"text": example["text"]}

bc = load_dataset("bookcorpusopen", split="train", streaming=True)
ccn = load_dataset("cc_news", split="train", streaming=True)
# this one currently has streaming issues - will fix soon
# owt = load_dataset("the_pile_openwebtext2", split="train", streaming=True)

dataset = interleave_datasets([
    bc.map(only_keep_text),
    ccn.map(only_keep_text),
    # owt.map(only_keep_text)
])

Then you can check the documentation to see how to use it in a pytorch training loop: Stream — datasets 1.16.1 documentation 3


Creating and uploading dataset Huggingface Hub vs Dataset creation script

Question:
Hello,
Our team is in the process of creating (manually for now) a multilingual machine translation dataset for low resource languages. Currently, we have text files for each language sourced from different documents. The number of lines in the text files are the same. For example, for each document we have lang1.txt and lang2.txt each with n lines. Each line in lang1.txt maps to each line in lang2.txt. We currently have these text files in a Github repository.
Once we have completed this dataset curation (this will be actively on-going) we would like to upload to the Huggingface Hub. Essentially, we would like for this to be similar to glue with different configurations corresponding to different multi-lingual datasets.
When I was looking over the documentation page, I found several resources. One is the uploading to Huggingface Hub 1, another is creating a dataset loading script 2.
I’m confused about the function provided by these two methods. Also, I don’t if just uploading the raw text files that we have is enough to be able to work with.
Question: What is the easiest and most efficient way to upload the kind of dataset that I have?
Thank you.

Context:
Hi ! Yes you can upload your text files to the Hub. Then in order to define the configurations and how the examples must be read from the text files, you must also upload a dataset script in the same repository as your data.
If you don’t upload a dataset script, then the default dataset builder for .txt file is used (and basically it concatenates all the text data together).
If I understand correctly your dataset is a parallel dataset like flores, so I think to make your dataset script you can get some inspiration from the flores dataset script 7. Also feel free to read the documentation you mentioned about how to create a dataset script.


Question:
Hi all,
I am trying to create a label2idx dictionary, where e.g.
labels2idx = {
    0: #all datasample indexes with label 0
    1: #all datasample indexes with label 1
    2: #all datasample indexes with label 2
    ...
}

I am loading the dataset via the hugginface datasets library (e.g. glue->mrpc using load_dataset). Is there a nice way to do this?
The most naive solution is to first loop over all datasamples by hand and then create the labels2idx by hand. I am wondering if there is a more nice way to do it using the library?

Context:
You can directly access this as follows:
from datasets import load_dataset

dataset = load_dataset("glue", "mrpc")

labels = dataset["train"].features["label"].names

id2label = {idx: label for idx, label in enumerate(labels)}
label2id = {label: idx for idx, label in enumerate(labels)}


Should I shard dataset in distributed training?

Question:
as the title, should I shard dataset set with rank?
for example:
rank_dataset = dataset.shard(num_shards=training_args.world_size, index=training_args.rank)

or Trainer will do that automate?

Context: The Trainer does the sharding for you. Same if you use Accelerate with your own training loop.


Getting list of tensors instead of tensor array after using set_format

Question:
So I’m bit unclear about how set_format converts data into torch tensors. My understanding is that if I use set_format to my dataset that has lists as values, it would conver them to tensors. I do get normal torch tensors for 1d and 2d lists, but when I have 3d lists, somehow it returns list of tensors.
Here is a toy example:
from datasets import Dataset

ex1 = {'a':[[1,1],[1,2]], 'b':[1,1]}
ex2 = {'a':[[[2,1],[2,2]], [[3,1],[3,2]]], 'b':[1,1]}

d1 = Dataset.from_dict(ex1)
d1.set_format('torch', columns=['a','b'])
d2 = Dataset.from_dict(ex2)
d2.set_format('torch', columns=['a','b'])

print(d1[:2])
print(d2[:2])

and the output is:
{'a': tensor([[1, 1],
        [1, 2]]), 'b': tensor([1, 1])}
{'a': [[tensor([2, 1]), tensor([2, 2])], [tensor([3, 1]), tensor([3, 2])]], 'b': tensor([1, 1])}

As you can see, for d2 dataset with 3d lists, I’m getting list of tensors, instead of getting just straight tensor arrays as were the cases for 1d and 2d lists in d1 dataset. Why is it returning list of tensors for 3d lists? Is there a way to get straight tensor arrays for such lists?
Really need a good clarification on it. Thank you.

Context:
Hi,
this inconsistency is due to how PyArrow converts nested sequences to NumPy by default but can be fixed by casting the corresponding column to the ArrayXD type.
E.g. in your example:
dset = Dataset.from_dict(
    {"a": [[[2,1],[2,2]], [[3,1],[3,2]]], "b": [1,1]},
    features=Features({"a": Array2D(shape=(2, 2), dtype="int32"), "b": Value("int32")})
)
dset.set_format('torch', columns=['a','b'])

If you want to cast the existing dataset, use map instead of cast (cast fails on special extension types):
dset = Dataset.from_dict({"a": [[[2,1],[2,2]], [[3,1],[3,2]]], "b": [1,1]})
dset = dset.map(lambda batch: batch, batched=True, features=Features({"a": Array2D(shape=(2, 2), dtype="int32"), "b": Value("int32")}))
dset.set_format('torch', columns=['a','b'])


Datasets map modifying audio array to list?

Question:
I’m trying to use my own data for fine-tuning a Wav2Vec2 model but every time I create my DatasetDict, it converts my audio array to a list? Am I doing something incorrectly? How can I preserve the audio as an array?
def speech_file_to_array_fn(batch):
    speech_array, sampling_rate = sf.read(batch["file"])
    batch["audio"] = speech_array
    return batch

updated_dataset = my_dataset.map(speech_file_to_array_fn)

my_audio = updated_dataset['test'][0]
print(my_audio)
{'file': '/disks/data3/UASPEECH/control/CM04/CM04_B2_UW33_M3.wav', 'text': 'APPROACH', 'audio': [0.0001220703125, -0.00018310546875, 0.000152587890625, -0.00030517578125, 6.103515625e-05, 9.1552734375e-05,...}

print(type(my_audio['audio']))
<class 'list'>

Context:
Hi,
set format to NumPy to get an array as follows:
updated_dataset.set_format("numpy", columns=["audio"], output_all_columns=True)
updated_dataset["test"][0]


Question:
Hi!
My question is not directly related to HF libraries, so it is a bit off-topic, but I hope the moderators will not take too strict a view on that and let me keep it.
When training ResNet on ImageNet dataset, I coded some dataloading functionality by hand, which was extremely useful to me. I am currently transitioning from TF2 to PyTorch and I am very new to PyTorch Dataset and Dataloader classes. I am wondering whether PyTorch Dataset/DataLoader classes make the flow I coded by hand available out of the box. I did read PyTorch tutorials and API docs before posting the question.
Here is the problem:
ImageNet consists of ~ 1.3mln JPEG images, which take about 140Gb disk space. When compiling a batch, one needs to read a batch_size number of image files from the disk, and each of them needs to be pre-processed and this pre-processing is computationally expensive (load an image file of size NxM, randomly choose an integer in the interval [256, 480], re-size the image in the way that the shortest size is equal to this integer, crop randomly to extract a 224x224 square image, apply random color transformation to it, etc…). If this pre-processing could be done once and then used for all the epochs, it wouldn’t be a problem at all, but it needs to be re-done for each file each epoch (that’s how data augmenation is achieved). And training requires a large number of epochs (50-120).
Here is how I solved it:
I borrowed 30Gb from my RAM and made a buffer disk out of it. This space is enough to comfortably accomodate more that 1,000 batches (already pre-processed). I created a text file, monitoring the training progress (with two lines: current epoch number, current batch number), the training process updates this file after each 200 batches (which is equivalent of 1% of epoch with my batch size). Then I
wrote a run-time pre-processing script (both training and pre-processing run at the same time in parallel), which checks:

where the training process currently is
where the buffer currently starts (at which batch number)
where the buffer currently ends (at which batch number)

If the pre-processing script sees that the training process went ahead and it is now safe to delete some batches from the start of the buffer, then it does so to free space. If the pre-processing script sees that the training process has less than 800 batches pre-processed and waiting to be fed to the training process, then it jumps into action, pre-processes more batches and places them at the end of the queue. Then it waits. It checks every 100 seconds whether there is any work for it, does it if there is, and then waits again. Pre-processing takes place 100% on the CPU and I could use multiple threads. This is important. Without it the pre-processing would not be able to work fast enough, and the main GPU training process would have to wait (which is unacceptable).
Can PyTorch Dataset/DataLoader classes provide the above functionality out of the box? If yes, I would appreciate if you could give a me push in the right direction. There is even no need to give me a code example (although that would be nice). Just tell me whether it is possible and where to look if it is.
Thank you!!
Alex

Context: If there is anybody who is only 80-90% sure that there is no out of the box functionality like that, it would also help me a lot.


The  Datasets library - Hugging Face Course

Question:
huggingface.co

The 🤗 Datasets library - Hugging Face Course 6

In the above link, I am not able to understand anything about the parameter: return_overflowing_tokens used in the tokenizer object and also what is this error message and why are we getting:
ArrowInvalid: Column 1 named condition expected length 1463 but got length 1000

Context:
Hi !
I think return_overflowing_tokens  can be used to keep the overflowing tokens in case an example is longer than max_length. For example for max_length=4 and for an example consisting of 10 tokens:

if return_overflowing_tokens=False, then the example is cropped and you get one list of 4 tokens
if return_overflowing_tokens=True, then the example is split to lists of maximum 4 tokens, so you end up with three lists with length 4, 4 and 2.

The error
ArrowInvalid: Column 1 named condition expected length 1463 but got length 1000

means that there is a mismatch between the number of  rows for condition and the other rows created by the tokenize function. Indeed from 1,000 examples, with each example having columns like condition, the tokenize function returned 1,463 tokenized texts. Because some columns have more rows than others, it can’t form a valid dataset table.
But since in the end you don’t care about the columns function at this point, you can just drop it and only keep the 1,463 tokenized texts with remove_columns=drug_dataset["train"].column_names
I hope that helps


Question:
In the dataset I have 5000000 rows, I would like to add a column called ‘embeddings’ to my dataset.
dataset = dataset.add_column('embeddings', embeddings)
The variable embeddings is a numpy memmap array of size (5000000, 512).
But I get this error:

ArrowInvalidTraceback (most recent call last)
 in
----> 1 dataset = dataset.add_column(‘embeddings’, embeddings)
/opt/conda/lib/python3.8/site-packages/datasets/arrow_dataset.py in wrapper(*args, **kwargs)
486         }
487         # apply actual function
 → 488         out: Union[“Dataset”, “DatasetDict”] = func(self, *args, **kwargs)
489         datasets: List[“Dataset”] = list(out.values()) if isinstance(out, dict) else [out]
490         # re-apply format to the output
/opt/conda/lib/python3.8/site-packages/datasets/fingerprint.py in wrapper(*args, **kwargs)
404             # Call actual function
405
 → 406             out = func(self, *args, **kwargs)
407
408             # Update fingerprint of in-place transforms + update in-place history of transforms
/opt/conda/lib/python3.8/site-packages/datasets/arrow_dataset.py in add_column(self, name, column, new_fingerprint)
3346             :class:Dataset
3347         “”"
 → 3348         column_table = InMemoryTable.from_pydict({name: column})
3349         # Concatenate tables horizontally
3350         table = ConcatenationTable.from_tables([self._data, column_table], axis=1)
/opt/conda/lib/python3.8/site-packages/datasets/table.py in from_pydict(cls, *args, **kwargs)
367     @classmethod
368     def from_pydict(cls, *args, **kwargs):
 → 369         return cls(pa.Table.from_pydict(*args, **kwargs))
370
371     @inject_arrow_table_documentation(pa.Table.from_batches)
/opt/conda/lib/python3.8/site-packages/pyarrow/table.pxi in pyarrow.lib.Table.from_pydict()
/opt/conda/lib/python3.8/site-packages/pyarrow/table.pxi in pyarrow.lib._from_pydict()
/opt/conda/lib/python3.8/site-packages/pyarrow/array.pxi in pyarrow.lib.asarray()
/opt/conda/lib/python3.8/site-packages/pyarrow/array.pxi in pyarrow.lib.array()
/opt/conda/lib/python3.8/site-packages/pyarrow/array.pxi in pyarrow.lib._ndarray_to_array()
/opt/conda/lib/python3.8/site-packages/pyarrow/error.pxi in pyarrow.lib.check_status()
ArrowInvalid: only handle 1-dimensional arrays

How can I solve?

Context:
Hi,
it should work if you use concatenate_datasets instead:
import datasets
dset_embed = datasets.Dataset.from_dict({"embeddings": embeddings})
dset_concat = datasets.concatenate_datasets([dset, dset_embed], axis=1)


Question:
Hello,
I’m trying to upload a multilingual low resource West Balkan machine translation dataset called rosetta_balcanica on Hugging Face hub. The data is stored in Github and was manually extracted. This is an on-going project. I’ve created a dataset creation script that should enable one to download and load the dataset based on the configuration specified. I’m also following the documentation page for creating a dataset loading script.
After writing out the script, I use datasets-cli to test the script. First off, its not clear where to run this command even though it says at the root, of the datasets directory. This to me implies the root of the repo, but it seems its one directory above that.
When I run the command datasets-cli test rosetta_balcanica --save_infos --all_configs (the directory rosetta_balcanica has only a README.md and rosetta_balcania.py), I get this error:
Traceback (most recent call last):
  File "/home/sudarshan/anaconda3/envs/rb_hub/bin/datasets-cli", line 8, in <module>
    sys.exit(main())
  File "/home/sudarshan/anaconda3/envs/rb_hub/lib/python3.7/site-packages/datasets/commands/datasets_cli.py", line 33, in main
    service.run()
  File "/home/sudarshan/anaconda3/envs/rb_hub/lib/python3.7/site-packages/datasets/commands/test.py", line 119, in run
    module = dataset_module_factory(path)
  File "/home/sudarshan/anaconda3/envs/rb_hub/lib/python3.7/site-packages/datasets/load.py", line 1083, in dataset_module_factory
    combined_path, download_mode=download_mode, dynamic_modules_path=dynamic_modules_path
  File "/home/sudarshan/anaconda3/envs/rb_hub/lib/python3.7/site-packages/datasets/load.py", line 669, in get_module
    download_config=self.download_config,
  File "/home/sudarshan/anaconda3/envs/rb_hub/lib/python3.7/site-packages/datasets/load.py", line 292, in _download_additional_modules
    f"To be able to use {name}, you need to install the following dependencies"
ImportError: To be able to use rosetta_balcanica, you need to install the following dependencies['datasets,'] using 'pip install datasets,' for instance'

However, I have already installed datasets so I’m not sure whey I’m getting this error. This is my dataset creation script rosetta_balcanica.py:
_DESCRIPTION="""
Rosetta-Balcanica is a set of evaluation datasets for low resource western Balkan languages manually sourced from articles from OSCE website.
"""

_HOMEPAGE='https://github.com/ebegoli/rosetta-balcanica'
_DATA_URL='https://github.com/ebegoli/rosetta-balcanica/raw/main/rosetta_balcanica.tar.gz'
_VERSION=datasets.Version('1.0.0')

class RosettaBalcanicaConfig(datasets.BuilderConfig):
  """BuilderConfig for Rosetta Balcanica for low resource West Balcan languages
  """

  def __init__(self, lang_pair=(None, None), **kwargs):
    assert lang_pair in _VALID_LANGUAGE_PAIRS, (f"Language pair {lang_pair} not supported (yet)")
    name = f'{lang_pair[0]} to {lang_pair[1]}'
    desc = f'Translation dataset from {lang_pair[0]} to {lang_pair[1]}'
    super(RosettaBalcanicaConfig, self).__init__(
      name=name,
      description=desc,
      version=_VERSION,
      **kwargs
    )

    self.lang_pair = lang_pair

class RoesettaBalcancia(datasets.GeneratorBasedBuilder):
  logger.debug("i'm in builder")
  BUILDER_CONFIGS = [
    RosettaBalcanicaConfig(
      lang_pair=lang_pair,
      versino=_VERSION,
    )
    for lang_pair in _VALID_LANGUAGE_PAIRS
  ]

  def _info(self):
    return datasets.DatasetInfo(
      description=_DESCRIPTION,
      features=datasets.Features(
        {'translation': datasets.features.Translation(languages=self.config.lang_pair)}
      ),
      homepage=_HOMEPAGE,
      supervised_keys=self.config.lang_pair,
      citation=_CITATION,
    )

  def _split_generators(self, dl_manager):
    archive = dl_manager.download(_DATA_URL)
    source,target = self.config.lang_pair
    non_en = source if target == 'en' else target
    data_dir = f'en-{non_en}'

    return [
      datasets.SplitGenerator(
      name=datasets.Split.TRAIN,
      gen_kwargs={
        'source_file': f'{data_dir}/train_{source}.txt',
        'target_file': f'{data_dir}/train_{target}.txt',
        'files': dl_manager.iter_archive(archive)
      }
    ),
      datasets.SplitGenerator(
      name=datasets.Split.TEST,
      gen_kwargs={
        'source_file':  f'{data_dir}/test_{source}.txt',
        'target_file': f'{data_dir}/test_{target}.txt',
        'files': dl_manager.iter_archive(archive)
      }
    ),
    ]

  def _generate_examples(self, source_file, target_file, files):
    source_sents, target_sents = None, None
    for path, f in files:
      if path == source_file:
        source_sents = f.read().decode('utf-8').split('\n')
      elif path == target_file:
        target_sents = f.read().decode('utf-8').split('\n')
      if source_sents is not None and target_sents is not None:
        break

    assert len(target_sents) == len(source_sents), (f"Sizes do not match: {len(source_sents) vs len(target_sents)} for {source_file} vs {target_file}")

    source,target = self.config.lang_pair
    for idx, (l1, l2) in enumerate(zip(source_sents, target_sents)):
      result = {
        'translation': {source: l1, target: l2}
      }
      if all(result.values()):
        yield idx, result

Please could I get some help on this?
Thanks!

Context:
Hi ! You did right by passing the path to the directory when contains your dataset script
And thanks for reporting the issue with the documentation, it’s indeed not clear enough, we’ll fix that.
Also regarding the error your get, I think this is because you import your dependencies in one single line, which is not parsed correctly by the dataset loader:
import datasets, logging

As a workaround you can just split the import on multiple lines
import logging
import datasets

I hope that helps !


Conditionally sample example from the dataset

Question: How how to conditionally sample example from the dataset. Eg dataset.sample(‘id’==‘abc’)?

Context: This is also referred to as filtering the dataset, docs can be found here: Main classes — datasets 1.15.1 documentation 1


Why is simply accessing dataset features so slow?

Question:
See this Colab notebook for runnable versions of the code snippets pasted below.

I’ve been playing with the SQuAD dataset and I’ve noticed that simply accessing features of mapped versions of the dataset is extremely slow:
# Let's play with the validation set.
squad = load_dataset("squad")
ds = squad["validation"]

# standard tokeinzer
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")

# tokenizer the dataset with standard hyperparameters.
f = lambda x: tokenizer(x["question"], x["context"], max_length=384, stride=128, truncation="only_second", padding="max_length", return_overflowing_tokens=True, return_offsets_mapping=True)
mapped_ds = ds.map(f, batched=True, remove_columns=val_ds.column_names)

Now, simply executing the statements – just accessing the features –
attention_mask = mapped_ds["attention_mask"]
input_ids = mapped_ds["input_ids"]
offset_mapping = mapped_ds["offset_mapping"]
overflow_to_sample_mapping = mapped_ds["overflow_to_sample_mapping"]
token_type_ids = mapped_ds["token_type_ids"]

takes about 12 seconds. Why does this take so long? My understanding from the docs is that an HF dataset is stored as a collection of feature columns. (Arrow table format?) Were this so, each assignment in the above snippet should just be assigning a pointer. There’s no data copying happening here, right?
Digging a little deeper, assigning attention_mask, input_ids, and token_type_ids each take 1-1.5 seconds. Assigning overflow_to_sample_mapping takes milliseconds, and assigning offet_mapping takes 8-9 seconds. It seems that accessing features with more complex structure is taking longer. I’m very curious what are these assignment statements actually doing, under the hood!
On a related note, looping over mapped_ds takes significantly longer than looping over its features, extracted and zipped:
for x in mapped_ds:
  pass

takes 15 seconds while
for x in zip(attention_mask, input_ids, offset_mapping, overflow_to_sample_mapping, token_type_ids):
  pass

takes 1.5 seconds.
This does have user-facing impact (right?): If you’re going to loop over the evaluation set in a custom evaluation script (overriding a trainer’s evaluate method, say), you’re better off storing the individual features of the evaluation dataset on the trainer instance once and for all and looping over their zip on each evaluation step (of which there can be many!), rather than iterating over the evaluation dataset itself each time. (A custom evaluation script of this sort is used to compute “exact match” and “F1” metrics on the SQuAD data. See the evaluate method of QuestionAnsweringTrainer in this file 1 and the postprocess_qa_predicitons function in this one.)

Context:
Hi,
this behavior is expected.
dataset[column] loads the entire dataset column into memory (this is why this part seems so slow to you), so operating directly on this data will be faster than operating on the dataset created with load_dataset, which is memory-mapped, i. e., stored in an arrow file by default. However, in many cases, a dataset or dataset transforms are too big to fit in RAM, so having them in memory is not an option and we’d rather pay a penalty (in respect to read/write operations) of having it stored in a file.
If you are sure that you have enough RAM, you can load the dataset as follows to have it in memory:
load_dataset("squad", split="validation", keep_in_memory=True)


Question:
Hello!
Is there a recommended/easy way to find the total number of tokens in a huggingface dataset?
I know it’s possible to use FreqDist in nltk, but it would be great if there was an implementation in datasets.
Thanks in advance.

Context:
Hi,
currently, there is no easy way to do that because Dataset.unique doesn’t have an option to flatten sequences, which are returned after tokenization (sequence of tokens).
However, it shouldn’t be too hard to add support for that. Then you will be able to do the following to solve your task:
def tokenize(batch):
    batch["text_col"] = [word_tokenize(text) for text in batch["text_col"]]
    return batch

dset_tokenized = dset.map(tokenize, batched=True)
unique_tokens = dset_tokenized.unique("text_col", flatten=True)  # <- new argument
fdist = FreqDist(unique_tokens)

(a link 3 to the issue on GH where you can track progress)


Rouge implementation of Huggingface Datasets

Question:
Hello, I’m using rouge implemented in Huggingface Datasets to get rouge score for my summarization texts.
The result is like
{'rouge1': AggregateScore(low=Score(precision=0.16620308156871524, recall=0.18219819615984395, fmeasure=0.16226017699359463), mid=Score(precision=0.17274338501705871, recall=0.1890957812369246, fmeasure=0.16823877588620403), high=Score(precision=0.17934569582981455, recall=0.1965626706042028, fmeasure=0.17491509794856058)),
'rouge2': AggregateScore(low=Score(precision=0.12478835737689957, recall=0.1362113231755514, fmeasure=0.12055941950062395), mid=Score(precision=0.1303967602691664, recall=0.1423747229852964, fmeasure=0.1258363976151122), high=Score(precision=0.13654527560789362, recall=0.1488071465116122, fmeasure=0.13184989406704056)),
'rougeL': AggregateScore(low=Score(precision=0.16568068818352072, recall=0.1811919016674486, fmeasure=0.1614784523482225), mid=Score(precision=0.17156684723552357, recall=0.1879777628247058, fmeasure=0.16720699286250762), high=Score(precision=0.17788847350584547, recall=0.1948899838530898, fmeasure=0.17316501523379826))}

What is the low, mid, and high categories mean? If I only want one rouge F1 value, which one should I choose?
Thanks for your attention!

Context:
HuggingFace Datasets implements the ROUGE metric using the rouge_score Python package, as seen here 1.
When taking a look at the source code of this package, it seems that low, mid and high refer to confidence intervals for the scores. As seen here, there are three bounds: low (row 0), mid (row 1) and high (row 2). Mid is always the mean, while low and high bounds are specified by the confidence_interval (which defaults to 0.95 meaning it will return the 2.5th and 97.5th percentiles for a 95% confidence interval on the mean).


Datasets.load_metric(“cer”) does not work

Question:
CER function does not work properly, even when one tries to execute the example in help. Code is below
from datasets import load_metric
cer = load_metric("cer")
predictions = ["this is the prediction", "there is an other sample"]
references = ["this is the reference", "there is another one"]

cer_score = cer.compute(predictions=predictions, references=references)
print(cer_score)

outputs
ValueError                                Traceback (most recent call last)
/tmp/ipykernel_5923/623465361.py in <module>
      4 references = ["this is the reference", "there is another one"]
      5
----> 6 cer_score = cer.compute(predictions=predictions, references=references)
      7 print(cer_score)

/opt/conda/lib/python3.7/site-packages/datasets/metric.py in compute(self, predictions, references, **kwargs)
    402             references = self.data["references"]
    403             with temp_seed(self.seed):
--> 404                 output = self._compute(predictions=predictions, references=references, **kwargs)
    405
    406             if self.buf_writer is not None:

~/.cache/huggingface/modules/datasets_modules/metrics/cer/4e547cc82fc2e597c84fe25f48ed77e3a9acfd354308fe654ccbc6ea9473a61a/cer.py in _compute(self, predictions, references, concatenate_texts)
    132                 prediction,
    133                 truth_transform=cer_transform,
--> 134                 hypothesis_transform=cer_transform,
    135             )
    136             incorrect += measures["substitutions"] + measures["deletions"] + measures["insertions"]

/opt/conda/lib/python3.7/site-packages/jiwer/measures.py in compute_measures(truth, hypothesis, truth_transform, hypothesis_transform, **kwargs)
    208     # Preprocess truth and hypothesis
    209     truth, hypothesis = _preprocess(
--> 210         truth, hypothesis, truth_transform, hypothesis_transform
    211     )
    212

/opt/conda/lib/python3.7/site-packages/jiwer/measures.py in _preprocess(truth, hypothesis, truth_transform, hypothesis_transform)
    327         raise ValueError(
    328             "number of ground truth inputs ({}) and hypothesis inputs ({}) must match.".format(
--> 329                 len(transformed_truth), len(transformed_hypothesis)
    330             )
    331         )

ValueError: number of ground truth inputs (21) and hypothesis inputs (22) must match.

Context:
Hi,
this issue was fixed a few days ago, and it will be included in the next release of datasets.
In the meantime, you can download the updated version of the metric script as follows:
cer_metric = load_metric("cer", revision="master")


Access word piece tokens from BERT tokenized dataset

Question:
I am new to huggingface and I am trying to figure out if it is possible to get the the word piece tokens when mapping a dataset to a BERT tokenizer. This is what I have right now:
from transformers import AutoTokenizer
from datasets import Dataset

bert_tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')

def tokenize_function_testing(examples):
    return bert_tokenizer(examples["sentences"], padding="max_length", max_length=32, truncation=True)

dataset = {"sentence_ids": ["123", "432", "443"],
           "labels": [3.0, 1.5, 1.0],
           "sentences": ["They stopped by infrequently",
                         "They are characteristically late",
                         "They live far away"]}

dataset = Dataset.from_dict(dataset)

tokenized_data = dataset.map(tokenize_function_testing)

And I was hoping I’d have access in tokenized_data to what you get out when you use tokens():
bert_tokenizer("They stopped by infrequently").tokens()
['[CLS]', 'they', 'stopped', 'by', 'in', '##fr', '##e', '##quent', '##ly', '[SEP]']

Is there any way to get this information without mapping the dataset a second time to get bert_tokenizer(examples["sentences"], padding="max_length", max_length=32, truncation=True).tokens() to a new column? I might not be thinking about this correctly so feel free to point out if I missed something obvious! Thanks!

Context:
You can update your function, to add both the encoded text, as well as the tokens:
def tokenize_function_testing(examples):
     encoding = bert_tokenizer(examples["sentences"], padding="max_length", max_length=32, truncation=True)
     # add tokens
     encoding["tokens"] = bert_tokenizer.convert_ids_to_tokens(encoding.input_ids)
     return encoding


Process “Downloading and preparing dataset json/default” doesnt proceed

Question:
Hello,
i’m working on a zero-shot classifaction model/pipeline. I have a problem when loading my data into a datset_dict. It does not happen everytime but quite often.
The moment i run follwing code cell, the cell will stuck or remain in the load_dataset line and will not further proceed.
# Loading data into DatasetDict and preparing
features = Features({"nli_label": ClassLabel(num_classes=3, names=['entailment','neutral','contradiction']), "intent_description": Value("string"), "intent": Value("string"), "text": Value("string"), "zlabel": Value("string"), "intent_dn": Value("string")})
dataset_dict = load_dataset('json', data_files={'train': 'train_data.json', 'test': 'test_data.json'})
dataset_dict = dataset_dict.map(features.encode_example, features=features)
dataset_dict = dataset_dict.rename_column("nli_label", "labels")
dataset_dict = dataset_dict.rename_column("zlabel", "hypothesis")
dataset_dict = dataset_dict.rename_column("text", "premise")

Im working in a google colab env. Im not sure if the jupyter env is the root of the problem. Below you can see the output of the cell. In order to proceed, i have to restart the runtime and hope that it works (what happens sometimes)

Screenshot 2021-11-15 at 11.41.372152×642 126 KB

Context: i’m also curious why the output states “Downloading and preparing dataset […]” but im using (or at least want to use) local files


Load pre-existing in-memory splits into a Dataset

Question:
Hello,
For some experiments I have made, I have already divided my local private data into “train”, “test”, and “valid” and saved them into a single .json file with the following stucture:
{
“train”: [
{
“text”: “this is the first example”
“label”: 2
},
{
“text”: “this is the second example”
“label”: 1
},
],
“test”: …
“valid”: …
}
but I struggle to create a Dataset out of it. In this tutorial (in memory section):
https://huggingface.co/docs/datasets/loading_datasets.html#from-a-python-dictionary 1
It isn’t clear how to proceed. I looked at the source code of arrow_dataset.py for this method from_dict, but I didn’t manage to find out the solution to my problem.
Does anyone have an idea (except splitting the original file into several files and reformating them)?

Context:
Hi,
this is possible but not nearly as clean as the approach which involves converting data to JSON Lines and splitting it to have one file per split:
import datasets
ddict = datasets.DatasetDict()
for split in ["train", "test", "valid"]:
   ddict.update(datasets.load_dataset("json", data_files={split: "path/to/data/file.json"}, field=split)


Question:
I want to load dataset locally. (such as xcopa).
for xcopa, i manually download the datasets from this Link, and set the mode to offline mode.  The code is:
import os
os.environ['HF_DATASETS_OFFLINE'] ='1'
from datasets import load_dataset
xcopa = load_dataset('./datasets/datasets/xcopa/xcopa.py',
                     name='et',
                     date_dir='path/to/zip',
                     cache_dir='path/to/cache')

But it still want to download the zip files from the Link rather than the locally files.
So i compare the difference between the xcopa.py & matinf.py (which should manually download). They do have some differences, so Can xcopa dataset be load locally (manually download the zip files and load it)? (Maybe in the _split_generators method of the xcopa.py)

Context:
Hi,
I’ll assume you are trying to download the dataset that way due to an error thrown by load_dataset("xcopa", "et"). It’s going to be fixed soon!
As you correctly pointed out, there are some differences in the data that are causing the error. In the meantime, you can bypass the error and download the dataset (with the newest data) as follows:
from datasets import load_dataset
dset = load_dataset("xcopa", "et", ignore_verifications=True)


Describe a nullable/optional column in dataset loading script

Question:
Assume a dataset has a float column that isn’t populated in all of its rows.
What is the generally accepted way to signify it in datasets.Features and _generate_examples?
Setting the column value to None does not work.

Context:
Hi,
you can use Value("float32") or Value("float64") as a column feature and set missing values to None. If this doesn’t work, please open an issue on GitHub and provide the code that reproduces it. The None handling in datasets still has some rough edges, and we are currently fixing it (More robust `None` handling by mariosasko · Pull Request #3195 · huggingface/datasets · GitHub 1).


Deploy Model without Amazon SageMaker or Accelerated Inference API

Question:
How I can deploy the model without using Accelerated Inference API or Amazon SageMaker.
I want to download the model then build my own API to call it, is that possible??

Context: Yes that 's possible. You can make a FastAPI app for instance to expose your model as a REST API, and then deploy it on a platform of choice.


Question:
huggingface uses fsspec for the API for datasets, right? are there plans to use IPFS? seems like there is a read-only fsspec 7.
maybe instead of “democratizing machine learning”, it can be “decentralizing machine learning”?

Context:
Hi,
I’m afraid the project you’ve linked is not mature enough, but we can reconsider adding the IPFS filesystem once this is changed.


Question:
i saw huggingface is like the top data loader for a lot of the paperswithcode datasets pages. one major one is missing; imagenet!
are there plans to upload this? i know they have been kinda strict with licensing, but kaggle is now officially allowed to host it 1
huge huge huge opportunity here! especially if you can use the streaming functionality of datasets.

      paperswithcode.com

Papers with Code - ImageNet Dataset

  The ImageNet dataset contains 14,197,122 annotated images according to the WordNet hierarchy. Since 2010 the dataset is used in the ImageNet Large Scale Visual Recognition Challenge (ILSVRC), a benchmark in image classification and object...

Context:
Hi,
currently, we are working on better support for vision datasets (e.g. we will have a dedicated feature for images), so we should have ImageNet in the library soon (a month or so).


Question:
I am currently using the following in order to save checkpoints (every 1000)
continue_from = 49000
save_for_every = 1000
encoded_dataset = datasets.load_from_disk("encoded_rvl_cdip")
for i in range(continue_from, data.shape[0], save_for_every):
    dataset = Dataset.from_pandas(data[i:i+save_for_every])
    encoded_subset = dataset.map(preprocess_data, remove_columns=dataset.column_names, features=features,
                                 batched=True, batch_size=2)
    encoded_dataset = datasets.concatenate_datasets([encoded_dataset, encoded_subset])
    print(f"Saving to encoded_rvl_cdip {i}...")
    encoded_dataset.save_to_disk("encoded_rvl_cdip_1")

print("Done")

However the problem is the file that I am saving is getting incrementally larger each iteration. Is there a way I can add to a saved file instead of having to overwrite the whole thing?

Context:
Hi ! When you save a dataset, it saves the whole thing. Therefore at each iteration it writes both the data from encoded_rvl_cdip but also all the chunks of data you loaded from data until that point.
So this is not exactly what you want since at each iteration it writes a really big dataset, that is getting bigger and bigger.
What you can do instead is simply save the chunks that you want to add in separate datasets, and then load first your original dataset, then load each chunk, and then concatenate them together


Question:
transformers: ‘4.10.3’
datasets: 1.12.1
Im trying to follow the examples from here to make my own custom metric: datasets/super_glue.py at master · huggingface/datasets · GitHub 1
def _info(self):
return datasets.MetricInfo(
description=’_DESCRIPTION’,
citation=’_CITATION’,
inputs_description=’_KWARGS_DESCRIPTION’,
features=datasets.Features({
“predictions”: {
“1”: datasets.Value(“int64”),
“2”: datasets.Value(“int64”),
“3”: datasets.Value(“int64”),
},
‘references’: datasets.Value(“int64”),
}),
codebase_urls=[],
reference_urls=[],
# format=‘numpy’
)
and somewhere in code I call this:
metric.add_batch(predictions=outputs[‘predictions’], references=outputs[‘references’])
If  my predictions is not nested but just  equal to datasets.Value(“int64”) then it works.
What am I doing wrong here?
Stack trace:
metric.add_batch(predictions=outputs[‘predictions’], references=outputs[‘references’])
File “/home/dspokoyn/.cache/pypoetry/virtualenvs/unit-7vxG7edj-py3.7/lib/python3.7/site-packages/datasets/metric.py”, line 431, in add_batch
batch = self.info.features.encode_batch(batch)
File “/home/dspokoyn/.cache/pypoetry/virtualenvs/unit-7vxG7edj-py3.7/lib/python3.7/site-packages/datasets/features.py”, line 1080, in encode_batch
encoded_batch[key] = [encode_nested_example(self[key], obj) for obj in column]
File “/home/dspokoyn/.cache/pypoetry/virtualenvs/unit-7vxG7edj-py3.7/lib/python3.7/site-packages/datasets/features.py”, line 1080, in
encoded_batch[key] = [encode_nested_example(self[key], obj) for obj in column]
File “/home/dspokoyn/.cache/pypoetry/virtualenvs/unit-7vxG7edj-py3.7/lib/python3.7/site-packages/datasets/features.py”, line 886, in encode_nested_example
k: encode_nested_example(sub_schema, sub_obj) for k, (sub_schema, sub_obj) in utils.zip_dict(schema, obj)
File “/home/dspokoyn/.cache/pypoetry/virtualenvs/unit-7vxG7edj-py3.7/lib/python3.7/site-packages/datasets/features.py”, line 885, in
return {
File “/home/dspokoyn/.cache/pypoetry/virtualenvs/unit-7vxG7edj-py3.7/lib/python3.7/site-packages/datasets/utils/py_utils.py”, line 99, in zip_dict
yield key, tuple(d[key] for d in dicts)
File “/home/dspokoyn/.cache/pypoetry/virtualenvs/unit-7vxG7edj-py3.7/lib/python3.7/site-packages/datasets/utils/py_utils.py”, line 99, in
yield key, tuple(d[key] for d in dicts)
TypeError: string indices must be integers

Context:
Hi ! Can you provide an example of predictions and references that you pass to the metric ?
You defined these feature types:
features=datasets.Features({
  “predictions”: {
    “1”: datasets.Value(“int64”),
    “2”: datasets.Value(“int64”),
    “3”: datasets.Value(“int64”),
  },
  ‘references’: datasets.Value(“int64”),
})

So your predictions must look like this:
predictions = [{"1": i, "2": j, "3": k}]

and your references like this:
references = [r]

where i,j,k,r are supposed to be integers.
Can you check that this is indeed the feature types that you want to use, and that your data match this structure ?


How to use several datasets that fit into the RAM?

Question:
Hello,
I have a the following issue. I am trying to train a language model in tensorflow using the nice new TF notebooks notebooks/language_modeling_from_scratch-tf.ipynb at master · huggingface/notebooks · GitHub 1
I know that the starting point of the training is to actually load the data using the datasets package.
from datasets import load_dataset
datasets = load_dataset("wikitext", "wikitext-2-raw-v1")

and then do something like

tokenized_datasets = datasets.map(
    tokenize_function, batched=True, num_proc=4, remove_columns=["text"]
)

My issue is that my text data is stored on my disk as several parquet files with the following constraints:

each file fits into the RAM easily (say in a Pandas dataframe)
there are too many files to load them all at once in the RAM

What is the best way to proceed using datasets here? Concatenating them will not work as my RAM is not sufficient. And training in the notebook seems to only work with one instance of a datasets
What do you think?
Thanks!

Context:
Hi ! You can load all your parquet files this way:
datasets = load_dataset("path/to/the/directory/containing/your/parquet/files")

It does two things:

it converts all the parquet files into Arrow files, and store them in the datasets cache
it opens the Arrow files using memory-mapping - that means that the part of your disk that contains the data is going to be used as virtual memory to load the dataset. Therefore it will load all your data without filling up your RAM.

Then once your dataset is loaded you can tokenize it:
tokenized_datasets = datasets.map(
    tokenize_function, batched=True, num_proc=4, remove_columns=["text"]
)


Explicitly defining schema in a dataset?

Question:
Hello,
We are trying to create a dataset 1 with two main kinds of tabular data. They are both stored in CSV files, and have mostly the same fields, albeit with a few important differences. The Dataset library has trouble reading the dataset in any way. The files are neither concatenated into one with missing fields nulled, nor split into two separate entities in the object - both ways would be perfectly acceptable for our goals. Is there any configuration work we can do to prevent rebuilding large sets like this from scratch?

Context:
Hi,
If I understand you correctly, you are trying to create a dataset from two CSV files in one read, e.g.:
dset = load_dataset("csv", data_files=["file1.csv", "file2.csv"])

which errors out because these files don’t share the same set of fields.
If that’s the case, you have two options:

create a separate dataset from each file, add None values where needed (either with map to have them cached in a file or with add_column to have them in RAM), and concatenate the datasets (make sure the datasets have matching schemas before the concatenate_datasets call - if not, use cast)
add a dataset loading script to the Hub (the preferred way if you want to load the dataset directly from the Hub without additional processing described in Option 1). There you can easily assign None to missing values. Or, to avoid missing data, you can have a separate config for each CSV file. More info on writing a dataset loading script can be found here 1, and an example of the script which reads CSV data here.


Most efficient way to retrieve N rows for a subset of columns

Question:
Hello,
I would like to retrieve rows from a dataset using a sequence of indexes as efficiently as possible. Each row contains many fields, so I would like to query the Arrow table for a subset of columns in order to exploit at best the column format.
My current method is the following:
def retrieve_rows(dataset: Dataset, indexes: Iterable[int], keys:List[str]):
    """Retrieved n rows from the `dataset` for the specific keys."""
    if keys is not None and len(keys) == 1:
        key = keys[0]
        retrieved_rows = map(dataset[key].__getitem__, indexes)
        retrieved_rows = [{key: x} for x in retrieved_docs]
    else:
        retrieved_rows = map(dataset.__getitem__, indexes)
        # filter keys
        retrieved_docs = [{k: v for k, v in row.items() if keys is None or k in keys} for row in retrieved_rows]
   return retrieved_docs

Limitations
However, it comes with two limitations

for len(keys)==1, the whole column is loaded.
for len(keys)>1, all columns are queried.

Dataset comes with a select method, but this create a new Dataset object, which seems quite cumbersome for my use case.

Questions
So my questions are:
a. how to query rows for a subset of columns
b. how to batch queries (or using an iterator of idx)
c. or alternatively, is it possible to return the Arrow table directly, so I can fine-tune the queries?

Context:
Hi Mario,
Thank you very much for the detailed reply. I have tried different things, including using the pyarrow.Table.take method directly. Based on profiling, I have settled on:
class FetchRows:
    def __init__(self, dataset: Dataset, keys: List[str]):
        self.dataset = dataset.remove_columns(set(dataset.column_names) - set(keys))

    def __call__(self, indexes: List[int]) -> Dict[str, Any]:
        return self.dataset.select(indexes, keep_in_memory=True)[None:None]


Question:
Having trouble opening large datasets since dataset viewer introduced. Here’s an example: https://huggingface.co/datasets/tydiqa 9
Maybe can add an optional “View Preview” button for these datasets, so that we can still open page without the preview?

Context: thanks for the report we are looking at it


Does masked language model training script does random shuffle on the dataset?

Question:
I haven’t seen this been explicitly defined in the code, although in the “no trainer” script the shuffling is defined. Internally in datasets the shuffling is done or is it missing in the MLM with trainer?
If it’s missing, I can contribute with a PR

Context:
Shuffle is not enabled in the default dataloaders in the trainer. If you want to add a PR, you need to add an argument to the TrainingArguments and update the Trainer dataloaders.
See comment by @sgugger below.


Datasets map tokenization throws OSError: No space left on device

Question:
I am running the run_mlm.py 2 example script with my custom dataset, but I am getting out of memory error, even using the keep_in_memory=True parameter. My custom dataset is a set of CSV files, but for now, I’m only loading a single file (200 Mb) with 200 million rows.
Before running the script I have about 128 Gb free disk, when I run the script it creates a couple of arrow files with 11Gb sizes until the disk is full. Although I have small disk space available, I do have 128Gb of RAM and during execution, only 30Gb gets used, shouldn’t the RAM usage be greater and had less disk usage?
        with training_args.main_process_first(desc="dataset map tokenization"):
            tokenized_datasets = raw_datasets.map(
                tokenize_function,
                batched=True,
                num_proc=data_args.preprocessing_num_workers,
                remove_columns=column_names,
                desc="Running tokenizer on every text in dataset",
                keep_in_memory=True,
            )

Stacktrace:
Traceback (most recent call last):
  File "/home/emanuel/twitter-br/env/lib/python3.8/site-packages/datasets/table.py", line 54, in _write_table_to_file
    writer.write_batch(batch)
  File "pyarrow/ipc.pxi", line 384, in pyarrow.lib._CRecordBatchWriter.write_batch
OSError: [Errno 28] No space left on device

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "train.py", line 627, in <module>
    main()
  File "train.py", line 477, in main
    tokenized_datasets = raw_datasets.map(
  File "/home/emanuel/twitter-br/env/lib/python3.8/site-packages/datasets/dataset_dict.py", line 484, in map
    {
  File "/home/emanuel/twitter-br/env/lib/python3.8/site-packages/datasets/dataset_dict.py", line 485, in <dictcomp>
    k: dataset.map(
  File "/home/emanuel/twitter-br/env/lib/python3.8/site-packages/datasets/arrow_dataset.py", line 2141, in map
    transformed_shards[index] = async_result.get()
  File "/home/emanuel/twitter-br/env/lib/python3.8/site-packages/multiprocess/pool.py", line 771, in get
    raise self._value
  File "/home/emanuel/twitter-br/env/lib/python3.8/site-packages/multiprocess/pool.py", line 537, in _handle_tasks
    put(task)
  File "/home/emanuel/twitter-br/env/lib/python3.8/site-packages/multiprocess/connection.py", line 209, in send
    self._send_bytes(_ForkingPickler.dumps(obj))
  File "/home/emanuel/twitter-br/env/lib/python3.8/site-packages/multiprocess/reduction.py", line 54, in dumps
    cls(buf, protocol, *args, **kwds).dump(obj)
  File "/home/emanuel/twitter-br/env/lib/python3.8/site-packages/dill/_dill.py", line 498, in dump
    StockPickler.dump(self, obj)
  File "/usr/local/lib/python3.8/pickle.py", line 487, in dump
    self.save(obj)
  File "/usr/local/lib/python3.8/pickle.py", line 560, in save
    f(self, obj)  # Call unbound method with explicit self
  File "/usr/local/lib/python3.8/pickle.py", line 901, in save_tuple
    save(element)
  File "/usr/local/lib/python3.8/pickle.py", line 560, in save
    f(self, obj)  # Call unbound method with explicit self
  File "/home/emanuel/twitter-br/env/lib/python3.8/site-packages/dill/_dill.py", line 990, in save_module_dict
    StockPickler.save_dict(pickler, obj)
  File "/usr/local/lib/python3.8/pickle.py", line 971, in save_dict
    self._batch_setitems(obj.items())
  File "/usr/local/lib/python3.8/pickle.py", line 997, in _batch_setitems
    save(v)
  File "/usr/local/lib/python3.8/pickle.py", line 603, in save
    self.save_reduce(obj=obj, *rv)
  File "/usr/local/lib/python3.8/pickle.py", line 717, in save_reduce
    save(state)
  File "/usr/local/lib/python3.8/pickle.py", line 560, in save
    f(self, obj)  # Call unbound method with explicit self
  File "/home/emanuel/twitter-br/env/lib/python3.8/site-packages/dill/_dill.py", line 990, in save_module_dict
    StockPickler.save_dict(pickler, obj)
  File "/usr/local/lib/python3.8/pickle.py", line 971, in save_dict
    self._batch_setitems(obj.items())
  File "/usr/local/lib/python3.8/pickle.py", line 997, in _batch_setitems
    save(v)
  File "/usr/local/lib/python3.8/pickle.py", line 578, in save
    rv = reduce(self.proto)
  File "/home/emanuel/twitter-br/env/lib/python3.8/site-packages/datasets/table.py", line 176, in __getstate__
    _write_table_to_file(table=table, filename=filename)
  File "/home/emanuel/twitter-br/env/lib/python3.8/site-packages/datasets/table.py", line 56, in _write_table_to_file
    return sum(batch.nbytes for batch in batches)
OSError: [Errno 28] No space left on device
Makefile:11: recipe for target 'train' failed

Context: As a workaround, I used my own dataset class to tokenize samples on the “__getitem__” method. In this way only the RAM is used, although, slower.


Specifying download directory for custom dataset loading script

Question: I’m following this tutorial 3 for making a custom dataset loading script  that is callable through datasets.load_dataset(). In the section about downloading data files and organizing splits, it says that datasets.DatasetBuilder._split_generators() takes a datasets.DownloadManager as input. When loading the dataset, the loader always downloads to /root/.cache/a/randomly/generated/path. Is there a way I can tell the datasets.DownloadManager to download and read from a specific directory through datasets.load_dataset()?

Context:
Hi,
by default, the download directory is set to ~/.cache/huggingface/downloads. To change the location, either set the HF_DATASETS_DOWNLOADED_DATASETS_PATH env variable to a different value or modify the DOWNLOADED_DATASETS_PATH variable:
import datasets
from pathlib import Path
datasets.config.DOWNLOADED_DATASETS_PATH = Path(target_path)
datasets.load_datase(...)

However, what you want to do in most cases is not only to change the location of the download directory but to change the entire cache directory to ignore the old cache. You can control this easily with the cache_dir argument in load_dataset (or with the HF_DATASETS_CACHE env variable /datasets.config.HF_DATASETS_CACHE)


Question:
I have a raw dataset that contains image file paths. The preprocessing function will read these files and prepare features, which are numpy float32 arrays, with shape (Height, Width, 3). A direct use of datasets will cache the results. For 600K images of size (224, 224, 3), it takes > 350 GB. I plan to extend my work to CC3M or even CC12M datasets, and the required disk volume becomes too large for me.
I am wondering what’s a good approach to this. I don’t want to cache the whole results considering the disk usage, but I can’t keep it in memory neither.
My idea is:

On each epoch starts, shuffle the raw dataset.
Split the raw dataset to several small ones. And use map(keep_in_memory=True) on each of them in turn.

For each split, iterate over it in batches.

Is there better way to deal with large image datasets using datasets built-in methods? Or a better way then what I describe above.
Here is an example pseudo code:
# Think the datasets containing image file paths
# Assume we have 100M examples.
ds = load_dataset("...")
train_ds_raw = ds["train"]

EPOCHS = 3
SPLIT_SIZE = 512
SPLITS = len(train_ds_raw) // SPLIT_SIZE
for epoch in range(EPOCHS):

    # shuffle at each epoch start
    train_ds = train_ds_raw.shuffle()

    for idx in range(SPLITS):

        # Use `map()` on smaller portions, so we can keep map()'s results in memory instead of writing to cache
        start_idx = SPLIT_SIZE * idx
        end_idx = SPLIT_SIZE * (idx + 1)
        _train_ds = train_ds_raw.select(range(start_idx, end_idx))

        # Load image files from disk + feature processing: multiprocessing
        _train_ds = _train_ds.map(
            preprocess_function,
            batched=True,
            batch_size=16,
            num_proc=4,
            features=features,
            keep_in_memory=True,
        )
        _train_ds = _train_ds.with_format("numpy")

        # Create sampling rng
        input_rng = np.random.default_rng(seed=42)

        # Some more (fast) processing without multiprocessing
        # No need to shuffle here
        train_loader = data_loader(input_rng, _train_ds, batch_size=128, shuffle=False)

        # training
        for idx, batch in enumerate(train_loader):
            pass

Context:
Hi,
we are currently working on the Image feature, which will make this whole process faster.
In the meantime, you can use Dataset.with_transform and load the images on the fly when you need them. This way, only the current batch will occupy some memory. The only downside of this approach is that you’ll have to implement any sort of additional formatting by yourself in with_transform (you can also use DataLoader’s collate_fn for this) because with_format and with_transform currently cannot be chained.


Dealing with large objects as arguments in datasets.map

Question:
Hi, I’ve recently noticed that fingerprinting takes a lot of time when function arguments are large, e.g.
import datasets
from datasets import load_dataset
# Random dataset
dataset = load_dataset('glue', 'mrpc', split='test')

def do_nothing(examples, important_dict):
    return examples

# A very big dictionary
important_dict = {}
for i in range(10**8):
    important_dict[i] = 'nothing'

dataset = dataset.map(
    lambda examples: do_nothing(examples, important_dict),
    batched=True,
    load_from_cache_file=False,
    num_proc=10, # The larger the slower
    keep_in_memory=True)

Fingerprinting of the do_nothing function takes a lot of time even when only one processor is used, but for many, it is basically N times more (which can be days when the arguments of the do_nothing function are really big).
It would be great if it was possible to disable fingerprinting altogether, or somehow exclude certain function arguments from being fingerprinted (like the imporant_dict), or if not exclude then mark them as immutable so that the fingerprint can be calculated only once. Or am I doing something completely wrong and there is a better way?
System info:

Ubuntu
Python 3.8
datasets==1.14.0

Context:
Hi ! Computing the fingerprint of the mapped dataset is necessary for the caching mechanism to work.
So you can disable this with set_caching_enabled(True), but every time you re-run your code it will recompute the map call.
The fingerprint is computed by hashing the code and the variables in your map function. So it takes time because it hashes your big dictionary. It could be helpful if you could specify the hash of your dictionary in advance maybe. Or if you could specify in advance the fingerprint of the resulting dataset. What do you think ?


Sentence Order Prediction - Dataset Creation

Question:
Hi all,
I noticed that the LineByLineWithSOPTextDataset 3 is slated for deprecation, with the recommendation to use the Datasets library. Scanning the documentation of Datasets, I couldn’t find a drop-in replacement.
Does such a replacement exist? If no replacement exists at the moment, is there a timeline till deprecation and a replacement on the horizon? Thanks so much!
Warm regards,
Sahil

Context:
Hi ! This class is deprecated in favor of using

the Datasets library to load the dataset
the Dataset.map function to apply the tokenization

You can check out the run_mlm.py 3 script to see how it loads a text dataset 1 and then applies tokenization.
It should be possible to use the LineByLineWithSOPTextDataet.create_examples_from_document in Dataset.map to get examples in the SOP format.


Dynamic Padding not working for Custom Dataset

Question:
Thank you for building a great platform!
I am trying to use custom torch.utils.data.Dataset instead of datasets.Dataset and I am getting following error:
ValueError: Caught ValueError in DataLoader worker process 0.
Original Traceback (most recent call last):
  File "/usr/local/lib/python3.7/dist-packages/transformers/feature_extraction_utils.py", line 158, in convert_to_tensors
    tensor = as_tensor(value)
ValueError: expected sequence of length 105600 at dim 2 (got 92160)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/lib/python3.7/dist-packages/torch/utils/data/_utils/worker.py", line 202, in _worker_loop
    data = fetcher.fetch(index)
  File "/usr/local/lib/python3.7/dist-packages/torch/utils/data/_utils/fetch.py", line 47, in fetch
    return self.collate_fn(data)
  File "<ipython-input-8-1e61eb8cbac4>", line 94, in __call__
    return_tensors="pt",
  File "/usr/local/lib/python3.7/dist-packages/transformers/models/wav2vec2/processing_wav2vec2.py", line 127, in pad
    return self.current_processor.pad(*args, **kwargs)
  File "/usr/local/lib/python3.7/dist-packages/transformers/feature_extraction_sequence_utils.py", line 225, in pad
    return BatchFeature(batch_outputs, tensor_type=return_tensors)
  File "/usr/local/lib/python3.7/dist-packages/transformers/feature_extraction_utils.py", line 73, in __init__
    self.convert_to_tensors(tensor_type=tensor_type)
  File "/usr/local/lib/python3.7/dist-packages/transformers/feature_extraction_utils.py", line 165, in convert_to_tensors
    "Unable to create tensor, you should probably activate padding "
ValueError: Unable to create tensor, you should probably activate padding with 'padding=True' to have batched tensors with the same length.

I passed data_collactor which, in my understanding, responsible for dynamic padding. Here is code
# -------------- Custom Dataset --------------
class Dataset(torch.utils.data.Dataset):
    def __init__(self, df):
        self.df = df

    def __len__(self):
        return self.df.shape[0]

    def __getitem__(self, idx):
        row = self.df.iloc[idx]
        speech_array, sampling_rate = sf.read(row['file'])

        input_values = processor(speech_array, sampling_rate=16_000).input_values
        with processor.as_target_processor():
            labels = processor(row["sentence"]).input_ids

        return {
            "input_values": input_values,
            "labels": labels
        }

# --------------- Trainer Part --------------
from transformers import Trainer
data_collator = DataCollatorCTCWithPadding(processor=processor, padding=True)

trainer = Trainer(
    model=model,
    data_collator=data_collator,
    args=training_args,
    compute_metrics=compute_metrics,
    train_dataset=all_train,
    eval_dataset=cv_test,
    tokenizer=processor.feature_extractor,
)

Has anybody faced same issue?

Context:
This error solved!
Actually, this was not error but I got confused by how input_values method of Wav2Vec2Processor works.
speech_array, sampling_rate = sf.read(row.file)
# speech_array.shape = (8900,)

input_values = processor(speech_array, sampling_rate=16_000).input_values
# input_values.shape = (1, 8900)

However, when you use it with huggingface datasets, it will return 1D array
def prepare_dataset(batch):
    batch["input_values"] = processor(batch["speech"], sampling_rate=16_000).input_values

    with processor.as_target_processor():
        batch["labels"] = processor(batch["target_text"]).input_ids
    return batch

dataset = dataset.map(prepare_dataset, remove_columns=common_voice_train.column_names)
# np.array(dataset[0]["input_values"]).shape = (8900,)


What’s the best way to change (convert) column type in Dataset

Question: I’m struggling to convert the integer column to float one. I didn’t find anything related to HF datasets documentation.

Context: Looks like the best way to deal with this is to use cast 8  method.


Can’t use datasets offline, even if I have uploaded the datasets to .cache dir

Question:
I want to use sst dataset on my school server,
my dataset loding code is: raw_dataset = datasets.load_dataset('glue', 'sst2')
I have uploaded my local downloaded dataset to the \.cache\huggingface\datasets dir.
I also use os.environ['HF_DATASETS_OFFLINE ']= "1" to force the program don’t try to search the internet.
But I still got:
ConnectionError: Couldn't reach https://raw.githubusercontent.com/huggingface/datasets/1.13.0/datasets/glue/glue.py
Could anyone help me to figure it out?


Question:
I usually use padding in batches before I get into the datasets library.
I found that dataset.map support batched and batch_size. But it seems that only padding all examples (in dataset.map) to fixed length or max_length make sense with subsequent batch_size in creating DataLoader.
Otherwise, if I use map function like lambda x: tokenizer(x["sentence"], padding=True, truncation=True) I get errors like RuntimeError: stack expects each tensor to be equal size, but got [56] at entry 0 and [53] at entry 8 when iterating the dataloader since I could not find a way to iterating the same batches in datasets.map in dataloader.
Am I right?

Context: That’s because padding=True makes the tokenization pad to the longest sequence in the batch. Therefore two batches may have different length


Question:
Hi.
I have an h5 file which consists of two datasets. One is for metadata (labels and etc) and one is for the actual data which is a 2d array for each element. From my experience working on datasets, HF’s dataset has a very good caching mechanism and is usually way faster than plain vanilla dataset I can code via pure python. I wanted to know how can I create a HF dataset from this hdf5 file I have, and are there any examples for custom datasets?
The dataset loading script template has parts about downloading the data and split generators, which I don’t think are related to my current task.
Additionally. I also have to split the data based on conditions of the label of the data, and not pure randomly. How can I achieve that?

Context:
Hi,
For the first dataset, try to load the data in chunks with Dataset.from_pandas if you can somehow convert the data to be pandas-compatible, or with Dataset.from_dict if you can load the data into a dict. Then, store each chunk to disk (Dataset.save_to_disk) and reload the chunk (datasets.load_from_disk). Next, concatenate the loaded chunks with datasets.concatenate_datasets (set the axis param to 0). Feel free to follow my snippet from the above because the idea is the same.
For the second dataset, load the JSON strings with Dataset.from_json. This dataset will already be on disk, so you don’t have to worry about RAM usage.
Finally, concatenate these two datasets as follows:
datasets.concatenate_dataset([windows_dataset, metadata_dataset], axis=1)


HuggingFace dataset: each element in list of batch should be of equal size

Question:
I’m trying to use HuggingFace’s tokenizers and datasets with a PyTorch dataloader, like so:
            dataset = load_dataset(
                'wikitext',
                'wikitext-2-raw-v1',
                split='train[:5%]',  # take only first 5% of the dataset
                cache_dir=cache_dir)

            tokenized_dataset = dataset.map(
                lambda e: self.tokenizer(e['text'],
                                         padding=True,
                                         max_length=512,
                                         # padding='max_length',
                                         truncation=True),
                batched=True)

with a dataloader:
        dataloader = torch.utils.data.DataLoader(
            dataset=tokenized_dataset,
            batch_size=batch_size,
            shuffle=True)

But the dataloader throws the following error:
  File "/home/rschaef/CoCoSci-Language-Distillation/distillation_v2/ratchet_learning/train.py", line 139, in run_epoch
    for batch_idx, batch in enumerate(task.dataloader):
  File "/home/rschaef/CoCoSci-Language-Distillation/cocosci/lib/python3.6/site-packages/torch/utils/data/dataloader.py", line 435, in __next__
    data = self._next_data()
  File "/home/rschaef/CoCoSci-Language-Distillation/cocosci/lib/python3.6/site-packages/torch/utils/data/dataloader.py", line 475, in _next_data
    data = self._dataset_fetcher.fetch(index)  # may raise StopIteration
  File "/home/rschaef/CoCoSci-Language-Distillation/cocosci/lib/python3.6/site-packages/torch/utils/data/_utils/fetch.py", line 47, in fetch
    return self.collate_fn(data)
  File "/home/rschaef/CoCoSci-Language-Distillation/cocosci/lib/python3.6/site-packages/torch/utils/data/_utils/collate.py", line 73, in default_collate
    return {key: default_collate([d[key] for d in batch]) for key in elem}
  File "/home/rschaef/CoCoSci-Language-Distillation/cocosci/lib/python3.6/site-packages/torch/utils/data/_utils/collate.py", line 73, in <dictcomp>
    return {key: default_collate([d[key] for d in batch]) for key in elem}
  File "/home/rschaef/CoCoSci-Language-Distillation/cocosci/lib/python3.6/site-packages/torch/utils/data/_utils/collate.py", line 81, in default_collate
    raise RuntimeError('each element in list of batch should be of equal size')
RuntimeError: each element in list of batch should be of equal size

Why is this happening and how do I prevent it from happening?

Context:
An example collate function may look like this in your case:
def collate_tokenize(data):
  text_batch = [element["text"] for element in data]
  tokenized = tokenizer(text_batch, padding='longest', truncation=True, return_tensors='pt')
  return tokenized

Then you pass to the dataloader:
dataloader = torch.utils.data.DataLoader(
    dataset=dataset,
    batch_size=batch_size,
    shuffle=True,
    collate_fn=collate_tokenize
    )

Also, here 9’s a somewhat outdated article that has an example of collate function.


Problem loading datasets library from Kaggle

Question:
After installing datasets and import it in Kaggle (with TPU support, no acceleration environment is ok), I received this warning message:
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-8-1127ec19bd45> in <module>
     15 from operator import itemgetter
     16 from typing import Iterator, List, Optional
---> 17 import datasets
     18 from datasets import load_dataset, load_metric
     19

/opt/conda/lib/python3.7/site-packages/datasets/__init__.py in <module>
     31     )
     32
---> 33 from .arrow_dataset import Dataset, concatenate_datasets
     34 from .arrow_reader import ArrowReader, ReadInstruction
     35 from .arrow_writer import ArrowWriter

/opt/conda/lib/python3.7/site-packages/datasets/arrow_dataset.py in <module>
     36 import pandas as pd
     37 import pyarrow as pa
---> 38 import pyarrow.compute as pc
     39 from multiprocess import Pool, RLock
     40 from tqdm.auto import tqdm

/opt/conda/lib/python3.7/site-packages/pyarrow/compute.py in <module>
     16 # under the License.
     17
---> 18 from pyarrow._compute import (  # noqa
     19     Function,
     20     FunctionOptions,

/opt/conda/lib/python3.7/site-packages/pyarrow/_compute.pyx in init pyarrow._compute()

ValueError: pyarrow.lib.Codec size changed, may indicate binary incompatibility. Expected 48 from C header, got 40 from PyObject

Seems to be updating pyarrow-5.0.0 from pyarrow-4.0.0>?
I tried to uninstall and install again with no help. any ideas? Thanks

Context:
Hi,
make sure to restart the kernel after installing datasets and then it should work.