Tokenize multiple files

James_Sanchez: <p>Hi All,</p>
<p>I am trying to run the following for multiple files to tokenize.</p>
<pre><code class="lang-auto">for file in all_files:
    data = TextLoader(file=file, tokenizer=tokenizer)
    train_dataloader = DataLoader(data, batch_size=120, shuffle=False) # Shuffle should be set to False
</code></pre>
<p>I have defined TextLoader and DataLoader and it doesn’t seem to pick up.</p>
<p>I get the following message.</p>
<p>''ValueError                                Traceback (most recent call last)<br>
 in &lt;cell line: 27&gt;()<br>
26<br>
27 for file in all_files:<br>
—&gt; 28     data = TextLoader(file=file,tokenizer=tokenizer)<br>
29     train_dataloader = DataLoader(data, batch_size=120, shuffle=False) # Shuffle should be set to False<br>
30</p>
<p>5 frames<br>
/usr/local/lib/python3.10/dist-packages/pandas/io/json/_json.py in _parse(self)<br>
1318         if orient == “columns”:<br>
1319             self.obj = DataFrame(<br>
 → 1320                 loads(json, precise_float=self.precise_float), dtype=None<br>
1321             )<br>
1322         elif orient == “split”:</p>
<p>Regards,</p>
<p>James</p>

ptrblck: <p>I don’t see the actual error in the stacktrace, so did you forget to paste it?</p>

James_Sanchez: <p>Hi ptrblck</p>
<p>Thank you for feedback.</p>
<p>Well I get this</p>
<pre><code>    if orient == "columns":
        self.obj = DataFrame(
            loads(json, precise_float=self.precise_float), dtype=None
        )
    elif orient == "split":
        decoded = {
            str(k): v
            for k, v in loads(json, precise_float=self.precise_float).items()
        }
        self.check_keys_split(decoded)
        orig_names = [
            (tuple(col) if isinstance(col, list) else col)
            for col in decoded["columns"]
        ]
        decoded["columns"] = dedup_names(
            orig_names,class FrameParser(Parser):

            is_potential_multi_index(orig_names, None),
        )
        self.obj = DataFrame(dtype=None, **decoded)
    elif orient == "index":
        self.obj = DataFrame.from_dict(
            loads(json, precise_float=self.precise_float),
            dtype=None,
            orient="index",
        )
    elif orient == "table":
        self.obj = parse_table_schema(json, precise_float=self.precise_float)
    else:
        self.obj = DataFrame(
            loads(json, precise_float=self.precise_float), dtype=None
        )

def _process_converter(self, f, filt=None) -&gt; None:
    """
    Take a conversion function and possibly recreate the frame.
    """
    if filt is None:
        filt = lambda col, c: True

    obj = self.obj
    assert obj is not None  # for mypy

    needs_new_obj = False
    new_obj = {}
    for i, (col, c) in enumerate(obj.items()):
        if filt(col, c):
            new_data, result = f(col, c)
            if result:
                c = new_data
                needs_new_obj = True
        new_obj[i] = c

    if needs_new_obj:
        # possibly handle dup columns
        new_frame = DataFrame(new_obj, index=obj.index)
        new_frame.columns = obj.columns
        self.obj = new_frame

def _try_convert_types(self) -&gt; None:
    if self.obj is None:
        return
    if self.convert_dates:
        self._try_convert_dates()

    self._process_converter(
        lambda col, c: self._try_convert_data(col, c, convert_dates=False)
    )

def _try_convert_dates(self) -&gt; None:
    if self.obj is None:
        return

    # our columns to parse
    convert_dates_list_bool = self.convert_dates
    if isinstance(convert_dates_list_bool, bool):
        convert_dates_list_bool = []
    convert_dates = set(convert_dates_list_bool)

    def is_ok(col) -&gt; bool:
        """
        Return if this col is ok to try for a date parse.
        """
        if not isinstance(col, str):
            return False

        col_lower = col.lower()
        if (
            col_lower.endswith("_at")
            or col_lower.endswith("_time")
            or col_lower == "modified"
            or col_lower == "date"
            or col_lower == "datetime"
            or col_lower.startswith("timestamp")
        ):
            return True
        return False

    self._process_converter(
        lambda col, c: self._try_convert_to_date(c),
        lambda col, c: (
            (self.keep_default_dates and is_ok(col)) or col in convert_dates
        ),
    )
</code></pre>
<p>Regards,</p>
<p>James.</p>