Repository: cogent3/cogent3
Pull request #1226 (2022-12-13 01:05:35)

ENH: implement updated loader and writer apps


Patch:
diff --git a/src/cogent3/app/data_store_new.py b/src/cogent3/app/data_store_new.py
index 66747a9d6..14b616e5d 100644
--- a/src/cogent3/app/data_store_new.py
+++ b/src/cogent3/app/data_store_new.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import inspect
+import json
 import re
 import reprlib
 
@@ -77,7 +78,10 @@ class DataMemberABC(ABC):
     def __eq__(self, other):
         """to check equality of members and check existence of a
         member in a list of members"""
-        return (self.data_store, self.unique_id) == (other.data_store, other.unique_id)
+        return type(self) == type(other) and (self.data_store, self.unique_id) == (
+            other.data_store,
+            other.unique_id,
+        )
 
     @property
     def md5(self):
@@ -591,3 +595,29 @@ def convert_tinydb_to_sqlite(source: Path, dest: Optional[Path] = None) -> DataS
         dstore.db.execute(cmnd, (lock_id,))
 
     return dstore
+
+
+def make_record_for_json(identifier, data, completed):
+    """returns a dict for storage as json"""
+    try:
+        data = data.to_rich_dict()
+    except AttributeError:
+        pass
+
+    data = json.dumps(data)
+    return dict(identifier=identifier, data=data, completed=completed)
+
+
+def load_record_from_json(data):
+    """returns identifier, data, completed status from json string"""
+    if type(data) == str:
+        data = json.loads(data)
+
+    value = data["data"]
+    if isinstance(value, str):
+        try:
+            value = json.loads(value)
+        except json.JSONDecodeError:
+            pass
+
+    return data["identifier"], value, data["completed"]
diff --git a/src/cogent3/app/io_new.py b/src/cogent3/app/io_new.py
index 57390bbb9..53ebe4bd5 100644
--- a/src/cogent3/app/io_new.py
+++ b/src/cogent3/app/io_new.py
@@ -1,7 +1,38 @@
+import json
+
+from enum import Enum
+from typing import Optional, Union
+
+import numpy
+
+from cogent3.core.alignment import ArrayAlignment, SequenceCollection
+from cogent3.core.moltype import get_moltype
+from cogent3.core.profile import (
+    make_motif_counts_from_tabular,
+    make_motif_freqs_from_tabular,
+    make_pssm_from_tabular,
+)
+from cogent3.evolve.fast_distance import DistanceMatrix
 from cogent3.format.alignment import FORMATTERS
+from cogent3.parse.sequence import PARSERS
+from cogent3.util.deserialise import deserialise_object
+from cogent3.util.table import Table
 
-from .composable import WRITER, NotCompleted, define_app
-from .typing import IdentifierType, SeqsCollectionType
+from .composable import LOADER, WRITER, NotCompleted, define_app
+from .data_store_new import (
+    DataStoreABC,
+    get_data_source,
+    load_record_from_json,
+    make_record_for_json,
+)
+from .typing import (
+    AlignedSeqsType,
+    IdentifierType,
+    SeqsCollectionType,
+    SerialisableType,
+    TabularType,
+    UnalignedSeqsType,
+)
 
 
 __author__ = "Gavin Huttley"
@@ -14,28 +45,263 @@ __email__ = "Gavin.Huttley@anu.edu.au"
 __status__ = "Alpha"
 
 
+class tabular(Enum):
+    table = "table"
+    distances = "distances"
+    motif_counts = "motif_counts"
+    motif_freqs = "motif_freqs"
+    pssm = "pssm"
+
+
+def _read_it(path):
+    try:
+        data = path.read()
+    except AttributeError:
+        try:
+            data = path.read_text()
+        except AttributeError:
+            raise IOError(f"unexpected type {type(path)}")
+    return data
+
+
+def _load_seqs(path, klass, parser, moltype):
+    data = _read_it(path)
+    data = data.splitlines()
+    data = dict(iter(parser(data)))
+    seqs = klass(data=data, moltype=moltype)
+    seqs.info.source = str(path)
+    return seqs
+
+
+@define_app(app_type=LOADER)
+class load_aligned:
+    """Loads aligned sequences. Returns an Alignment object."""
+
+    def __init__(self, moltype=None, format="fasta"):
+        """
+        Parameters
+        ----------
+        moltype
+            molecular type, string or instance
+        format : str
+            sequence file format
+        """
+        self.moltype = moltype if moltype is None else get_moltype(moltype)
+        self._parser = PARSERS[format.lower()]
+
+    T = Union[SerialisableType, AlignedSeqsType]
+
+    def main(self, path: IdentifierType) -> T:
+        """returns alignment"""
+        return _load_seqs(path, ArrayAlignment, self._parser, self.moltype)
+
+
+@define_app(app_type=LOADER)
+class load_unaligned:
+    """Loads unaligned sequences. Returns a SequenceCollection."""
+
+    def __init__(self, *, moltype=None, format="fasta"):
+        """
+        Parameters
+        ----------
+        moltype
+            molecular type, string or instance
+        format : str
+            sequence file format
+        """
+        self.moltype = moltype if moltype is None else get_moltype(moltype)
+        self._parser = PARSERS[format.lower()]
+
+    T = Union[SerialisableType, UnalignedSeqsType]
+
+    def main(self, path: IdentifierType) -> T:
+        """returns sequence collection"""
+        seqs = _load_seqs(path, SequenceCollection, self._parser, self.moltype)
+        return seqs.degap()
+
+
+@define_app(app_type=LOADER)
+class load_tabular:
+    """Loads delimited data. Returns a Table."""
+
+    def __init__(
+        self,
+        with_title=False,
+        with_header=True,
+        limit=None,
+        sep="\t",
+        strict=True,
+        as_type: tabular = "table",
+    ):
+        """
+        Parameters
+        ----------
+        with_title
+            files have a title
+        with_header
+            files have a header
+        limit
+            number of records to read
+        sep
+            field delimiter
+        as_type
+            cogent3 type of tabular data
+        strict
+            all rows MUST have the same number of records
+        """
+        self._sep = sep
+        self._with_title = with_title
+        self._with_header = with_header
+        self._limit = limit
+        self.strict = strict
+        self.as_type = tabular(as_type)
+
+    def _parse(self, data):
+        """returns header, records, title"""
+        title = header = None
+        sep = self._sep
+        strict = self.strict
+        lines = _read_it(data).splitlines()
+        if self._limit:
+            lines = lines[: self._limit]
+        if self._with_title:
+            title = lines.pop(0).strip()
+        if self._with_header:
+            header = [e.strip() for e in lines.pop(0).strip().split(sep)]
+
+        num_records = None if header is None else len(header)
+        rows = []
+        for line in lines:
+            line = line.strip()
+            line = [e.strip() for e in line.split(sep)]
+            if num_records is None:
+                num_records = len(line)
+            if strict and len(line) != num_records:
+                raise AssertionError(
+                    f"Inconsistent number of fields: {len(line)} != {num_records}"
+                )
+            rows.append(line)
+
+        records = []
+        for record in zip(*rows):
+            record = numpy.array(record, dtype="O")
+            try:
+                record = record.astype(int)
+            except ValueError:
+                try:
+                    record = record.astype(float)
+                except ValueError:
+                    pass
+            records.append(record)
+        records = numpy.array(records, dtype="O").T
+        return header, records, title
+
+    def main(self, path: IdentifierType) -> TabularType:
+        try:
+            header, data, title = self._parse(path)
+        except Exception as err:
+            return NotCompleted("ERROR", self, err.args[0], source=str(path))
+
+        if self.as_type is tabular.table:
+            return Table(header=header, data=data, title=title)
+
+        assert data.shape[1] == 3, "Invalid tabular data"
+
+        if self.as_type is tabular.distances:
+            # records is of the form [ [dim-1, dim-2, value] for entries in DistanceMatrix ]
+            return DistanceMatrix({(e[0], e[1]): e[2] for e in data})
+
+        func = {
+            tabular.motif_counts: make_motif_counts_from_tabular,
+            tabular.motif_freqs: make_motif_freqs_from_tabular,
+            tabular.pssm: make_pssm_from_tabular,
+        }
+
+        return func[self.as_type](data)
+
+
+@define_app(app_type=LOADER)
+class load_json:
+    """Loads json serialised cogent3 objects from a json file.
+    Returns whatever object type was stored."""
+
+    def main(self, path: IdentifierType) -> SerialisableType:
+        """returns object deserialised from json at path"""
+        data = _read_it(path)
+        identifier, data, completed = load_record_from_json(data)
+
+        result = deserialise_object(data)
+        if hasattr(result, "info"):
+            result.info["source"] = result.info.get("source", identifier)
+        else:
+            try:
+                identifier = getattr(result, "source", identifier)
+                setattr(result, "source", identifier)
+            except AttributeError:
+                pass
+        return result
+
+
 @define_app(app_type=WRITER)
-class write_seqs:
+class write_json:
     def __init__(
         self,
-        data_store,
-        format="fasta",
+        data_store: DataStoreABC,
     ):
         self.data_store = data_store
-        self._formatter = FORMATTERS[format]
+        self._format = "json"
 
-    def _make_outname(self, obj) -> str:
-        try:
-            identifier = obj.info.source
-        except AttributeError:
-            identifier = obj.source
+    def main(
+        self, data: SerialisableType, identifier: Optional[str] = None
+    ) -> IdentifierType:
+        identifier = identifier or get_data_source(data)
+        if isinstance(data, NotCompleted):
+            return self.data_store.write_not_completed(
+                unique_id=f"{identifier}.json", data=data.to_json()
+            )
+
+        out = make_record_for_json(identifier, data, True)
+        data = json.dumps(out)
+        return self.data_store.write(unique_id=identifier, data=data)
 
-        return identifier
 
-    def main(self, data: SeqsCollectionType, identifier=None) -> IdentifierType:
-        identifier = identifier or self._make_outname(data)
+@define_app(app_type=WRITER)
+class write_seqs:  # todo docstring
+    def __init__(
+        self,
+        data_store: DataStoreABC,
+        format="fasta",
+    ):
+        self.data_store = data_store
+        self._formatter = FORMATTERS[format]
+
+    def main(
+        self, data: SeqsCollectionType, identifier: Optional[str] = None
+    ) -> IdentifierType:
+        identifier = identifier or get_data_source(data)
         if isinstance(data, NotCompleted):
-            return self.data_store.write_not_completed(data.source, data.to_json())
+            return self.data_store.write_not_completed(
+                unique_id=f"{identifier}.json", data=data.to_json()
+            )
 
         data = self._formatter(data.to_dict())
         return self.data_store.write(unique_id=identifier, data=data)
+
+
+@define_app(app_type=WRITER)
+class write_tabular:  # todo doctsring
+    def __init__(self, data_store: DataStoreABC, format="tsv"):
+        self.data_store = data_store
+        self._format = format
+
+    def main(
+        self, data: TabularType, identifier: Optional[str] = None
+    ) -> IdentifierType:
+        identifier = identifier or get_data_source(data)
+        if isinstance(data, NotCompleted):
+            return self.data_store.write_not_completed(
+                unique_id=f"{identifier}.json", data=data.to_json()
+            )
+
+        output = data.to_string(format=self._format)
+        return self.data_store.write(unique_id=identifier, data=output)


Test patch:
diff --git a/tests/test_app/test_io_new.py b/tests/test_app/test_io_new.py
index e9c1c3e0d..a599cc78e 100644
--- a/tests/test_app/test_io_new.py
+++ b/tests/test_app/test_io_new.py
@@ -1,9 +1,16 @@
 from pathlib import Path
+
+import numpy
 import pytest
-from cogent3.app import io_new as io_app_new
-from cogent3.app.composable import source_proxy
+
+from cogent3 import DNA
+from cogent3.app import io_new as io_app
+from cogent3.app.composable import NotCompleted, source_proxy
 from cogent3.app.data_store_new import DataMember, DataStoreDirectory, Mode
-from cogent3.core.alignment import ArrayAlignment
+from cogent3.core.alignment import ArrayAlignment, SequenceCollection
+from cogent3.core.profile import PSSM, MotifCountsArray, MotifFreqsArray
+from cogent3.evolve.fast_distance import DistanceMatrix
+from cogent3.maths.util import safe_log
 from cogent3.parse.sequence import PARSERS
 
 
@@ -16,12 +23,22 @@ __maintainer__ = "Gavin Huttley"
 __email__ = "Gavin.Huttley@anu.edu.au"
 __status__ = "Alpha"
 
+from numpy.testing import assert_allclose
+
+from cogent3.util.table import Table
+
+
 DATA_DIR = Path(__file__).parent.parent / "data"
 
 
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="function")
 def tmp_dir(tmpdir_factory):
-    return tmpdir_factory.mktemp("datastore")
+    return Path(tmpdir_factory.mktemp("io"))
+
+
+@pytest.fixture(scope="function")
+def w_dir_dstore(tmp_dir):
+    return DataStoreDirectory(tmp_dir, mode="w")
 
 
 @pytest.fixture(autouse=True)
@@ -31,7 +48,7 @@ def workingdir(tmp_dir, monkeypatch):
     monkeypatch.chdir(tmp_dir)
 
 
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="function")
 def fasta_dir(tmp_dir):
     tmp_dir = Path(tmp_dir)
     filenames = DATA_DIR.glob("*.fasta")
@@ -43,6 +60,15 @@ def fasta_dir(tmp_dir):
     return fasta_dir
 
 
+def _get_generic_result(source):
+    """creates a generic result with a DNA moltype as the single value"""
+    from cogent3.app.result import generic_result
+
+    gr = generic_result(source=source)
+    gr["dna"] = DNA
+    return gr
+
+
 def test_write_seqs(fasta_dir, tmp_dir):
     """correctly writes sequences out"""
     datastore = DataStoreDirectory(fasta_dir, suffix="fasta")
@@ -54,7 +80,7 @@ def test_write_seqs(fasta_dir, tmp_dir):
     out_data_store = DataStoreDirectory(
         tmp_dir / "test_write_seqs", mode=Mode.w, suffix="fasta"
     )
-    writer = io_app_new.write_seqs(out_data_store, format="fasta")
+    writer = io_app.write_seqs(out_data_store, format="fasta")
     wrote = writer(seqs[0], datamember.unique_id)
     assert isinstance(wrote, DataMember)
 
@@ -83,3 +109,263 @@ def test_source_proxy_simple(fasta_dir):
     # also return source_proxy
     got = list(reader.as_completed([path]))
     assert isinstance(got[0], source_proxy)
+
+
+@pytest.mark.parametrize("suffix", ("nex", "paml", "fasta"))
+def test_load_aligned(suffix):
+    """should handle nexus too"""
+    nexus_paths = DataStoreDirectory(DATA_DIR, suffix=suffix, limit=2)
+    loader = io_app.load_aligned(format=suffix)
+    results = [loader(m) for m in nexus_paths]
+    for result in results:
+        assert isinstance(result, ArrayAlignment)
+
+
+def test_load_unaligned():
+    """load_unaligned returns degapped sequence collections"""
+    fasta_paths = DataStoreDirectory(DATA_DIR, suffix=".fasta", limit=2)
+    fasta_loader = io_app.load_unaligned(format="fasta")
+    for i, seqs in enumerate(map(fasta_loader, fasta_paths)):
+        assert isinstance(seqs, SequenceCollection)
+        assert "-" not in "".join(seqs.to_dict().values())
+        assert seqs.info.source == fasta_paths[i].unique_id
+
+
+@pytest.mark.parametrize(
+    "loader", (io_app.load_aligned, io_app.load_unaligned, io_app.load_tabular)
+)
+def test_load_nonpath(loader):
+    # returns NotCompleted when it's given an alignment/sequence
+    # collection
+    got = loader()({})
+    assert isinstance(got, NotCompleted)
+
+
+def test_load_json(tmp_dir):
+    """correctly loads an object from json"""
+    import json
+
+    from cogent3.app.data_store import make_record_for_json
+
+    data = make_record_for_json("delme", DNA, True)
+    data = json.dumps(data)
+    outpath = tmp_dir / "delme.json"
+    outpath.write_text(data)
+    # straight directory
+    reader = io_app.load_json()
+    got = reader(outpath)
+    assert isinstance(got, DNA.__class__)
+    assert got == DNA
+
+
+def test_load_tabular(tmp_dir):
+    """correctly loads tabular data"""
+    rows = [[1, 2], [3, 4], [5, 6.5]]
+    table = Table(["A", "B"], data=rows)
+    load_table = io_app.load_tabular(sep="\t", with_header=True)
+    outpath = tmp_dir / "delme.tsv"
+    table.write(outpath)
+    new = load_table(outpath)
+    assert new.title == ""
+    assert type(new[0, "B"]) == type(table[0, "B"])
+    assert type(new[0, "A"]) == type(table[0, "A"])
+    outpath = tmp_dir / "delme2.tsv"
+    with open(outpath, "w") as out:
+        out.write("\t".join(table.header[:1]) + "\n")
+        for row in table.tolist():
+            row = "\t".join(map(str, row))
+            out.write(row + "\n")
+    result = load_table(outpath)
+    assert isinstance(result, NotCompleted)
+
+
+def test_load_tabular_motif_counts_array(w_dir_dstore):
+    """correctly loads tabular data for MotifCountsArray"""
+    w_dir_dstore.suffix = "tsv"
+    data = [[2, 4], [3, 5], [4, 8]]
+    mca = MotifCountsArray(data, "AB")
+    loader = io_app.load_tabular(sep="\t", as_type="motif_counts")
+    writer = io_app.write_tabular(data_store=w_dir_dstore, format="tsv")
+    outpath = "delme.tsv"
+    m = writer.main(data=mca, identifier="delme")
+    new = loader(m)
+    assert mca.to_dict() == new.to_dict()
+
+
+def test_load_tabular_motif_freqs_array(w_dir_dstore):
+    """correctly loads tabular data for MotifFreqsArray"""
+    w_dir_dstore.suffix = "tsv"
+    data = [[0.3333, 0.6667], [0.3750, 0.625], [0.3333, 0.6667]]
+    mfa = MotifFreqsArray(data, "AB")
+    loader = io_app.load_tabular(sep="\t", as_type="motif_freqs")
+    writer = io_app.write_tabular(data_store=w_dir_dstore, format="tsv")
+    outpath = "delme"
+    m = writer.main(mfa, identifier="delme")
+    new = loader(m)
+    assert mfa.to_dict() == new.to_dict()
+
+
+def test_load_tabular_pssm(w_dir_dstore):
+    """correctly loads tabular data for PSSM"""
+    w_dir_dstore.suffix = "tsv"
+    # data from test_profile
+    data = [
+        [0.1, 0.3, 0.5, 0.1],
+        [0.25, 0.25, 0.25, 0.25],
+        [0.05, 0.8, 0.05, 0.1],
+        [0.7, 0.1, 0.1, 0.1],
+        [0.6, 0.15, 0.05, 0.2],
+    ]
+    pssm = PSSM(data, "ACTG")
+    loader = io_app.load_tabular(sep="\t", as_type="pssm")
+    writer = io_app.write_tabular(data_store=w_dir_dstore, format="tsv")
+    m = writer.main(pssm, identifier="delme")
+    new = loader(m)
+    assert_allclose(pssm.array, new.array, atol=0.0001)
+
+
+def test_load_tabular_distance_matrix(w_dir_dstore):
+    """correctly loads tabular data for DistanceMatrix"""
+    w_dir_dstore.suffix = "tsv"
+    data = {(0, 0): 0, (0, 1): 4, (1, 0): 4, (1, 1): 0}
+    matrix = DistanceMatrix(data)
+    loader = io_app.load_tabular(sep="\t", as_type="distances")
+    writer = io_app.write_tabular(data_store=w_dir_dstore, format="tsv")
+    m = writer.main(matrix, identifier="delme")
+    new = loader(m)
+    assert matrix.to_dict() == new.to_dict()
+
+
+def test_load_tabular_table(w_dir_dstore):
+    """correctly loads tabular data"""
+    w_dir_dstore.suffix = "tsv"
+    rows = [[1, 2], [3, 4], [5, 6.5]]
+    table = Table(["A", "B"], data=rows)
+    loader = io_app.load_tabular(sep="\t", as_type="table")
+    writer = io_app.write_tabular(data_store=w_dir_dstore, format="tsv")
+    m = writer.main(table, identifier="delme")
+    new = loader(m)
+    assert table.to_dict() == new.to_dict()
+
+
+def test_write_tabular_motif_counts_array(w_dir_dstore):
+    """correctly writes tabular data for MotifCountsArray"""
+    w_dir_dstore.suffix = "tsv"
+    data = [[2, 4], [3, 5], [4, 8]]
+    mca = MotifCountsArray(data, "AB")
+    loader = io_app.load_tabular(sep="\t")
+    writer = io_app.write_tabular(data_store=w_dir_dstore, format="tsv")
+    m = writer.main(mca, identifier="delme")
+    assert isinstance(m, DataMember)
+    new = loader(m)
+    # when written to file in tabular form
+    # the loaded table will have dim-1 dim-2 as column labels
+    # and the key-values pairs listed below; in dict form...
+    expected = {
+        0: {"dim-1": 0, "dim-2": "A", "value": 2},
+        1: {"dim-1": 0, "dim-2": "B", "value": 4},
+        2: {"dim-1": 1, "dim-2": "A", "value": 3},
+        3: {"dim-1": 1, "dim-2": "B", "value": 5},
+        4: {"dim-1": 2, "dim-2": "A", "value": 4},
+        5: {"dim-1": 2, "dim-2": "B", "value": 8},
+    }
+    assert expected == new.to_dict()
+
+
+def test_write_tabular_motif_freqs_array(w_dir_dstore):
+    """correctly writes tabular data for MotifFreqsArray"""
+    w_dir_dstore.suffix = "tsv"
+    data = [[0.3333, 0.6667], [0.3750, 0.625], [0.3333, 0.6667]]
+    mfa = MotifFreqsArray(data, "AB")
+    loader = io_app.load_tabular(sep="\t")
+
+    writer = io_app.write_tabular(data_store=w_dir_dstore, format="tsv")
+    m = writer.main(mfa, identifier="delme")
+    new = loader(m)
+    # when written to file in tabular form
+    # the loaded table will have dim-1 dim-2 as column labels
+    # and the key-values pairs listed below; in dict form...
+    expected = {
+        0: {"dim-1": 0, "dim-2": "A", "value": 0.3333},
+        1: {"dim-1": 0, "dim-2": "B", "value": 0.6667},
+        2: {"dim-1": 1, "dim-2": "A", "value": 0.3750},
+        3: {"dim-1": 1, "dim-2": "B", "value": 0.6250},
+        4: {"dim-1": 2, "dim-2": "A", "value": 0.3333},
+        5: {"dim-1": 2, "dim-2": "B", "value": 0.6667},
+    }
+    assert expected == new.to_dict()
+
+
+def test_write_tabular_pssm(w_dir_dstore):
+    """correctly writes tabular data for PSSM"""
+    w_dir_dstore.suffix = "tsv"
+    # data from test_profile
+    data = numpy.array(
+        [
+            [0.1, 0.3, 0.5, 0.1],
+            [0.25, 0.25, 0.25, 0.25],
+            [0.05, 0.8, 0.05, 0.1],
+            [0.7, 0.1, 0.1, 0.1],
+            [0.6, 0.15, 0.05, 0.2],
+        ]
+    )
+    pssm = PSSM(data, "ACTG")
+    loader = io_app.load_tabular(sep="\t")
+    writer = io_app.write_tabular(data_store=w_dir_dstore, format="tsv")
+    m = writer.main(pssm, identifier="delme")
+    new = loader(m)
+    expected = safe_log(data) - safe_log(numpy.array([0.25, 0.25, 0.25, 0.25]))
+    for i in range(len(expected)):
+        j = i // 4
+        assert numpy.isclose(new.array[i][2], expected[j][i - j], atol=0.0001)
+
+
+def test_write_tabular_distance_matrix(w_dir_dstore):
+    """correctly writes tabular data for DistanceMatrix"""
+    w_dir_dstore.suffix = "tsv"
+    data = {(0, 0): 0, (0, 1): 4, (1, 0): 4, (1, 1): 0}
+    matrix = DistanceMatrix(data)
+    loader = io_app.load_tabular(sep="\t")
+    writer = io_app.write_tabular(data_store=w_dir_dstore, format="tsv")
+    m = writer.main(matrix, identifier="delme")
+    new = loader(m)
+    # when written to file in tabular form
+    # the loaded table will have dim-1 dim-2 as column labels
+    # and the key-values pairs listed below; in dict form...
+    expected = {
+        0: {"dim-1": 0, "dim-2": 1, "value": 4},
+        1: {"dim-1": 1, "dim-2": 0, "value": 4},
+    }
+    assert expected == new.to_dict()
+
+
+def test_write_tabular_table(w_dir_dstore):
+    """correctly writes tabular data"""
+    w_dir_dstore.suffix = "tsv"
+    rows = [[1, 2], [3, 4], [5, 6.5]]
+    table = Table(["A", "B"], data=rows)
+    loader = io_app.load_tabular(sep="\t")
+    writer = io_app.write_tabular(data_store=w_dir_dstore, format="tsv")
+    m = writer.main(table, identifier="delme")
+    new = loader(m)
+    assert table.to_dict() == new.to_dict()
+
+
+def test_write_json_with_info(w_dir_dstore):
+    """correctly writes an object with source attribute from json"""
+    w_dir_dstore.suffix = "json"
+    # create a mock object that pretends like it's been derived from
+    # something
+
+    obj = _get_generic_result("delme.json")
+    writer = io_app.write_json(data_store=w_dir_dstore)
+    m = writer(obj)
+    assert isinstance(m, DataMember)
+    reader = io_app.load_json()
+    got = reader(m)
+    got.deserialised_values()
+    assert got["dna"] == DNA
+
+
+# todo test objects where there is no unique_id provided, or inferrable,
+# should return NotCompletedError