File size: 4,204 Bytes
f126def 8696911 f126def |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 |
# coding=utf-8
# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Large-scale Indonesian Summarization Dataset"""
import glob
import json
import os
import re
from pathlib import Path
import datasets
logger = datasets.logging.get_logger(__name__)
_CITATION = """\
"""
_DESCRIPTION = """\
This module load text dataset from local directory. The text dataset should have the format like Oscar dataset
where each new entry is separated by empty lines.
"""
_HOMEPAGE = ""
_LICENSE = ""
class TextCollectionConfig(datasets.BuilderConfig):
"""BuilderConfig for TextCollection"""
def __init__(self, **kwargs):
"""BuilderConfig for TextCollection.
Args:
**kwargs: keyword arguments forwarded to super.
"""
super(TextCollectionConfig, self).__init__(**kwargs)
class TextCollection(datasets.GeneratorBasedBuilder):
VERSION = datasets.Version("1.0.0")
BUILDER_CONFIGS = [
TextCollectionConfig(
name="text_collection",
version=VERSION,
description="Id Collection dataset",
),
]
@property
def manual_download_instructions(self):
return """\
You need to manually collect text datasets in a directory. The text dataset can then be loaded
using the following command:
`datasets.load_dataset("text_collection", data_dir="<path/to/dataset>")`.
"""
def _info(self):
return datasets.DatasetInfo(
description=_DESCRIPTION,
features=datasets.Features({"id": datasets.Value("int64"), "text": datasets.Value("string")}),
supervised_keys=None,
homepage=_HOMEPAGE,
license=_LICENSE,
citation=_CITATION,
)
def _split_generators(self, dl_manager):
data_dir = os.path.abspath(os.path.expanduser(dl_manager.manual_dir))
print("# Data directory", data_dir)
if not os.path.exists(data_dir):
raise FileNotFoundError(
"{} does not exist. Make sure you insert a manual dir via `datasets.load_dataset('id_liputan6', "
"'canonical', data_dir=...)`. Manual download instructions:\n{}".format(
data_dir, self.manual_download_instructions
)
)
split_generators = [
datasets.SplitGenerator(
name=datasets.Split.TRAIN,
gen_kwargs={
"article_dir": os.path.join(data_dir, ""),
"split": "train",
},
)
]
return split_generators
def _generate_examples(self, article_dir, split):
logger.info("⏳ Generating %s examples from = %s", split, article_dir)
id_ = 0
current_lines = []
for path in sorted(glob.glob(os.path.join(article_dir, "**/*.txt"), recursive=True)):
with open(path, "r") as f:
print("# Reading", path)
for line in f:
if len(line.strip()) > -1:
current_lines.append(line)
elif current_lines:
feature = id_, {"id": id_, "text": "".join(current_lines).rstrip()}
yield feature
id_ += 1
current_lines = []
# last paragraph
if current_lines:
feature = id_, {"id": id_, "text": "".join(current_lines).rstrip()}
yield feature
id_ += 1
current_lines = []
|