Repository: cogent3/cogent3
Pull request #1864 (2024-05-14 02:53:02)

Mixture of patches


Patch:
diff --git a/src/cogent3/core/annotation_db.py b/src/cogent3/core/annotation_db.py
index d8cd86554..54442a59e 100644
--- a/src/cogent3/core/annotation_db.py
+++ b/src/cogent3/core/annotation_db.py
@@ -1287,6 +1287,7 @@ class GffAnnotationDb(SqliteAnnotationDbMixin):
             rows.append(tuple(record.get(c) for c in col_order))
 
         self.db.executemany(sql, rows)
+
         self.db.commit()
         del reduced
 
diff --git a/src/cogent3/util/io.py b/src/cogent3/util/io.py
index 16d3f117f..8e4d20bf7 100644
--- a/src/cogent3/util/io.py
+++ b/src/cogent3/util/io.py
@@ -1,17 +1,15 @@
 import contextlib
-import os
 import shutil
 import uuid
 
 from bz2 import open as bzip_open
 from gzip import open as gzip_open
 from io import TextIOWrapper
-from os import path as os_path
-from os import remove
-from pathlib import Path
+from os import PathLike, remove
+from pathlib import Path, PurePath
 from re import compile
 from tempfile import mkdtemp
-from typing import IO, Callable, Optional, Tuple, Union
+from typing import IO, Callable, Iterator, Optional, Tuple, Union
 from urllib.parse import ParseResult, urlparse
 from urllib.request import urlopen
 from zipfile import ZipFile
@@ -21,12 +19,13 @@ from chardet import detect
 from cogent3.util.misc import _wout_period
 
 
+PathType = Union[str, PathLike, PurePath]
 # support prefixes for urls
 _urls = compile("^(http[s]*|file)")
 
 
 def _get_compression_open(
-    path: Optional[os.PathLike] = None, compression: Optional[str] = None
+    path: Optional[PathType] = None, compression: Optional[str] = None
 ) -> Optional[Callable]:
     """returns function for opening compression formats
 
@@ -47,7 +46,7 @@ def _get_compression_open(
     return {"gz": gzip_open, "bz2": bzip_open, "zip": open_zip}.get(compression, None)
 
 
-def open_zip(filename: os.PathLike, mode: str = "r", **kwargs) -> IO:
+def open_zip(filename: PathType, mode: str = "r", **kwargs) -> IO:
     """open a single member zip-compressed file
 
     Note
@@ -81,7 +80,7 @@ def open_zip(filename: os.PathLike, mode: str = "r", **kwargs) -> IO:
         return TextIOWrapper(opened, encoding=encoding)
 
 
-def open_(filename: os.PathLike, mode="rt", **kwargs) -> IO:
+def open_(filename: PathType, mode="rt", **kwargs) -> IO:
     """open that handles different compression
 
     Parameters
@@ -186,7 +185,7 @@ class atomic_write:
     """performs atomic write operations, cleans up if fails"""
 
     def __init__(
-        self, path: os.PathLike, tmpdir=None, in_zip=None, mode="w", encoding=None
+        self, path: PathType, tmpdir=None, in_zip=None, mode="w", encoding=None
     ):
         """
 
@@ -308,7 +307,7 @@ class atomic_write:
 T = Optional[str]
 
 
-def get_format_suffixes(filename: os.PathLike) -> Tuple[T, T]:
+def get_format_suffixes(filename: PathType) -> Tuple[T, T]:
     """returns file, compression suffixes"""
     filename = Path(filename)
     if not filename.suffix:
@@ -343,8 +342,81 @@ def remove_files(list_of_filepaths, error_on_missing=True):
         raise OSError("Some filepaths were not accessible: %s" % "\t".join(missing))
 
 
-def path_exists(path: os.PathLike) -> bool:
+def path_exists(path: PathType) -> bool:
     """whether path is a valid path and it exists"""
-    with contextlib.suppress(ValueError, TypeError):
-        return os_path.exists(str(path))
+    with contextlib.suppress(Exception):
+        return Path(path).exists()
     return False
+
+
+def iter_splitlines(
+    path: PathType, chunk_size: Optional[int] = 1_000_000
+) -> Iterator[str]:
+    """yields line from file
+
+    Parameters
+    ----------
+    path
+        data file
+    chunk_size
+        number of bytes to load in one go from path
+
+    Notes
+    -----
+    Loads chunks of data from the file, yields one line at a time
+    """
+    path = Path(path)
+    if chunk_size and path.stat().st_size < chunk_size:
+        # file is smaller than provided chunk_size, just
+        # load it all
+        chunk_size = None
+
+    with open_(path) as infile:
+        last = ""
+        while True:
+            data = infile.read(chunk_size)
+            if not data:  # end of file
+                break
+
+            data = last + data
+            end_is_newline = data.endswith("\n")
+            lines = data.splitlines()
+            last = lines.pop(-1)
+            if end_is_newline:
+                # even if text is from Windows and uses "\r\n", pythons
+                # string splitlines() will respect \n
+                last += "\n"
+
+            if not len(lines):
+                # we have not seen a newline
+                continue
+
+            yield from lines
+
+        if last:
+            yield from last.splitlines()
+
+
+def iter_line_blocks(
+    path: PathType, num_lines: int = 1000, chunk_size: Optional[int] = 1_000_000
+) -> Iterator[list[str]]:
+    """yields list with num_lines str from path
+
+    Parameters
+    ----------
+    path
+        data file
+    num_lines
+        number of lines per block
+    chunk_size
+        number of bytes to load in one go from path
+    """
+    lines = []
+    for line in iter_splitlines(path, chunk_size=chunk_size):
+        lines.append(line)
+        if len(lines) == num_lines:
+            yield lines
+            lines = []
+
+    if lines:
+        yield lines


Test patch:
diff --git a/tests/test_util/test_io.py b/tests/test_util/test_io.py
index 8cbc1e664..f4739f668 100644
--- a/tests/test_util/test_io.py
+++ b/tests/test_util/test_io.py
@@ -8,10 +8,13 @@ from urllib.parse import urlparse
 
 import pytest
 
+from cogent3.app.composable import NotCompleted
 from cogent3.util.io import (
     _path_relative_to_zip_parent,
     atomic_write,
     get_format_suffixes,
+    iter_line_blocks,
+    iter_splitlines,
     open_,
     open_url,
     path_exists,
@@ -125,6 +128,12 @@ def test_open_handles_bom(tmp_dir):
             assert got == text, f"failed reading {path}"
 
 
+@pytest.mark.parametrize("non", (None, ""))
+def test_open_empty_raises(non):
+    with pytest.raises(ValueError):
+        open_(non)
+
+
 def test_aw_zip_from_path(tmp_dir):
     """supports inferring zip archive name from path"""
     path = tmp_dir / "foo.txt"
@@ -252,6 +261,7 @@ def test_get_format_suffixes_pathlib(name, expect):
         ("(a:0.1,b:0.1,(c:0.1,d:0.1):0.1)", False),
         (__file__, True),
         (pathlib.Path(__file__), True),
+        (NotCompleted("FAIL", "test", message="none", source="unknown"), False),
     ),
 )
 def test_path_exists(val, expect):
@@ -378,3 +388,108 @@ def test_open_url_exceptions():
         _ = open_url(
             "ftp://raw.githubusercontent.com/cogent3/cogent3/develop/tests/data/gff2_test.gff",
         )
+
+
+def test_iter_splitlines_one(tmp_path):
+    # file has a single line
+    path = tmp_path / "one-line.txt"
+    value = "We have text on one line."
+    path.write_text(value)
+    got = list(iter_splitlines(path))
+    assert got == [value]
+
+
+@pytest.mark.parametrize("newline", ("\n", "\r\n"))
+def test_iter_splitlines_line_diff_newline(tmp_path, newline):
+    path = tmp_path / "multi-line.txt"
+    value = ["We have some", "text on different lines", "which load"]
+    with open(path, mode="w", newline=newline) as out:
+        out.write("\n".join(value))
+    # we use a very small chunk size
+    got = list(iter_splitlines(path, chunk_size=5))
+    assert got == value
+
+
+@pytest.mark.parametrize("newline", ("\n", "\r\n"))
+def test_iter_splitlines_file_endswith_newline(tmp_path, newline):
+    path = tmp_path / "multi-line.txt"
+    value = ["We have some", "text on different lines", "which load"]
+    with open(path, mode="w", newline=newline) as out:
+        out.write("\n".join(value))
+    # we use a very small chunk size
+    got = list(iter_splitlines(path, chunk_size=5))
+    assert got == value
+
+
+def test_iter_splitlines_chunk_size_exceeds_file_size(tmp_path):
+    path = tmp_path / "multi-line.txt"
+    value = ["We have some", "text on different lines", "which load"]
+    path.write_text("\n".join(value))
+    # we use a massive chunk size
+    got = list(iter_splitlines(path, chunk_size=5_000_000))
+    assert got == value
+
+
+@pytest.mark.parametrize(
+    "value",
+    (
+        # creates a one line block ending on newline
+        "With text\nending on a\nended in newline.",
+        # creates a two line block ending on newline
+        "With text\nending\non a\nended in newline.",
+    ),
+)
+def test_iter_splitlines_chunk_endswith_newline(tmp_path, value):
+    path = tmp_path / "multi-line.txt"
+    # character 22 is a newline
+    value = value.splitlines()
+    path.write_text("\n".join(value))
+    # we use a chunk size that ends with a newline
+    got = list(iter_splitlines(path, chunk_size=11))
+    assert got == value
+
+
+def test_iter_splitlines_chunk_empty_file(tmp_path):
+    path = tmp_path / "zero.txt"
+    path.write_text("")
+    got = list(iter_splitlines(path))
+    assert not got
+
+
+def test_iter_line_blocks_correct_size(tmp_path):
+    # correctly break up
+    path = tmp_path / "multi-line.txt"
+    value = ["We have some", "text on different lines", "which load"]
+    path.write_text("\n".join(value))
+    # we use a massive chunk size
+    got = list(iter_line_blocks(path, num_lines=2, chunk_size=5))
+    expect = [value[:2], value[-1:]]
+    assert got == expect
+
+
+def test_iter_line_blocks_empty(tmp_path):
+    path = tmp_path / "zero.txt"
+    path.write_text("")
+    # we use a massive chunk size
+    got = list(iter_line_blocks(path, num_lines=2))
+    assert not got
+
+
+def test_iter_line_blocks_one(tmp_path):
+    # file has a single line
+    path = tmp_path / "one-line.txt"
+    value = "We have text on one line."
+    path.write_text(value)
+    got = list(iter_line_blocks(path, num_lines=2))
+    assert got == [[value]]
+
+
+def test_iter_line_blocks_none_num_lines(tmp_path):
+    # correctly break up
+    path = tmp_path / "multi-line.txt"
+    value = ["We have some", "text on different lines", "which load"]
+    path.write_text("\n".join(value))
+    # we use a massive chunk size
+    got = list(iter_line_blocks(path, num_lines=None))
+    expect = [value]
+    assert got == expect