Package: rapidnlp-datasets
rapidnlp-datasetsData pipelines for bothTensorFlowandPyTorch!If you want to load public datasets, try:tensorflow/datasetshuggingface/datasetsIf you want to load local, personal dataset with minimized boilerplate, userapidnlp-datasets!installationpipinstall-Urapidnlp-datasetsUsageHere are few examples to show you how to use this library.QuickStart: Sequence Classification TaskQuickStart: Question Answering TaskQuickStart: Token Classification TaskQuickStart: Masked Language Model TaskQuickStart: SimCSE(Sentence Embedding)sequence-classification-quickstartimporttorchfromtokenizersimportBertWordPieceTokenizerfromrapidnlp_datasetsimportDatasetForSequenceClassificationfromrapidnlp_datasets.tfimportTFDatasetForSequenceClassifiation# build datasettokenizer=BertWordPieceTokenizer.from_file("testdata/vocab.txt")dataset=DatasetForSequenceClassification(tokenizer)dataset.add_jsonl_files(input_files=["testdata/sequence_classification.jsonl"])# convert to tf.data.Datasettf_dataset=dataset.to_tf_dataset(batch_size=32)foridx,batchinenumerate(iter(tf_dataset)):print("No.{}batch:\n{}".format(idx,batch))# save tfrecorddataset.save_tfrecord("testdata/sequence_classification.tfrecord")# build dataset from tfrecord filesdataset=TFDatasetForSequenceClassifiation.from_tfrecord_files("testdata/sequence_classification.tfrecord")foridx,batchinenumerate(iter(dataset)):print("No.{}batch:\n{}".format(idx,batch))# convert to torch.utils.data.Datasetpt_dataset=dataset.to_pt_dataset()dataloader=torch.utils.data.DataLoader(pt_dataset,num_workers=2,shuffle=True,batch_size=32,collate_fn=pt_dataset.batch_padding_collator)foridx,batchinenumerate(dataloader):print("No.{}batch:\n{}".format(idx,batch))question-answering-quickstartimporttorchfromtokenizersimportBertWordPieceTokenizerfromrapidnlp_datasetsimportDatasetForQuestionAnsweringfromrapidnlp_datasets.tfimportTFDatasetForQuestionAnswering# build datasettokenizer=BertWordPieceTokenizer.from_file("testdata/vocab.txt")dataset=DatasetForQuestionAnswering(tokenizer)dataset.add_jsonl_files(input_files="testdata/qa.jsonl")# convert to tf.data.Datasettf_dataset=dataset.to_tf_dataset()foridx,batchinenumerate(iter(tf_dataset)):print("NO.{}batch:\n{}".format(idx,batch))# save to tfrecorddataset.save_tfrecord("testdata/qa.tfrecord")# build dataset from tfrecord filestf_dataset=TFDatasetForQuestionAnswering.from_tfrecord_files("testdata/qa.tfrecord",batch_size=32,padding="batch")foridx,batchinenumerate(iter(tf_dataset)):print()print("No.{}batch:\n{}".format(idx,batch))# convert to torch.utils.data.Datasetpt_dataset=dataset.to_pt_dataset()dataloader=torch.utils.data.DataLoader(pt_dataset,batch_size=32,collate_fn=pt_dataset.batch_padding_collator,)foridx,batchinenumerate(dataloader):print("No.{}batch:\n{}".format(idx,batch))token-classification-quickstartimporttorchfromtokenizersimportBertWordPieceTokenizerfromrapidnlp_datasetsimportDatasetForTokenClassificationfromrapidnlp_datasets.tfimportTFDatasetForTokenClassification# build datasettokenizer=BertCharLevelTokenizer.from_file("testdata/vocab.txt")dataset=DatasetForTokenClassification(tokenizer)dataset.add_jsonl_files("testdata/token_classification.jsonl",label2id=_label_to_id)# conver to tf.data.Datasettf_dataset=dataset.to_tf_dataset()foridx,batchinenumerate(iter(tf_dataset)):print("No.{}batch:\n{}".format(idx,batch))# save dataset to tfrecorddataset.save_tfrecord("testdata/token_classification.tfrecord")# build dataset from tfrecord filestf_dataset=TFDatasetForTokenClassification.from_tfrecord_files(input_files="testdata/token_classification.tfrecord",batch_size=4,)foridx,batchinenumerate(iter(tf_dataset)):print("No.{}batch:\n{}".format(idx,batch))# convert to torch.utils.data.Datasetpt_dataset=dataset.to_pt_dataset()dataloader=torch.utils.data.DataLoader(pt_dataset,num_workers=1,batch_size=4,collate_fn=pt_dataset.batch_padding_collator)foridx,batchinenumerate(dataloader):print("No.{}batch:\n{}".format(idx,batch))masked-language-models-quickstartimporttorchfromtokenizersimportBertWordPieceTokenizerfromrapidnlp_datasetsimportDatasetForMaskedLanguageModelfromrapidnlp_datasets.tfimportTFDatasetForMaksedLanguageModel# build datasettokenizer=BertWordPieceTokenizer.from_file("testdata/vocab.txt")dataset=DatasetForMaskedLanguageModel(tokenizer)dataset.add_jsonl_files(input_files=["testdata/mlm.jsonl"])dataset.add_text_files(input_files=["/path/to/text/files"])# convert to tf.data.Datasettf_dataset=dataset.to_tf_dataset(batch_size=4)foridx,batchinenumerate(iter(tf_dataset)):print("No.{}batch:\n{}".format(idx,batch))# save dataset as tfrecorddataset.save_tfrecord("testdata/mlm.tfrecord")# load tf.data.Dataset from tfrecord filesdataset=TFDatasetForMaksedLanguageModel.from_tfrecord_files(input_files="testdata/mlm.tfrecord",batch_size=4)foridx,batchinenumerate(iter(dataset)):print("No.{}batch:\n{}".format(idx,batch))# convert to torch.utils.data.Datasetpt_dataset=dataset.to_pt_dataset()# build dataloaderdataloader=torch.utils.data.DataLoader(pt_dataset,batch_size=4,num_workers=1,collate_fn=pt_dataset.batch_padding_collator)foridx,batchinenumerate(dataloader):print("No.{}batch:\n{}".format(idx,batch))simcse-quickstartimporttorchfromtokenizersimportBertWordPieceTokenizerfromrapidnlp_datasetsimportDatasetForSimCSEfromrapidnlp_datasets.tfimportTFDatasetForSimCSE# build datasetdataset=DatasetForSimCSE(tokenizer=BertWordPieceTokenizer.from_file("testdata/vocab.txt"),with_positive_sequence=False,with_negative_sequence=False,)dataset.add_jsonl_files("testdata/simcse.jsonl")# convert to tf.data.Datasettf_dataset=dataset.to_tf_dataset()foridx,batchinenumerate(iter(tf_dataset)):print()print("No.{}batch:\n{}".format(idx,batch))# save to tfrecorddataset.save_tfrecord("testdata/simcse.tfrecord")# build dataset from tfrecord filestf_dataset=TFDatasetForSimCSE.from_tfrecord_files("testdata/simcse.tfrecord",with_positive_sequence=False,with_negative_sequence=False,)foridx,batchinenumerate(iter(tf_dataset)):print("No.{}batch:\n{}".format(idx,batch))# convert to torch.utils.data.Datasetpt_dataset=dataset.to_pt_dataset()dataloader=torch.utils.data.DataLoader(pt_dataset,num_workers=2,shuffle=True,batch_size=32,collate_fn=pt_dataset.batch_padding_collator)foridx,batchinenumerate(dataloader):print("No.{}batch:\n{}".format(idx,batch))