Repository: ethanrowe/python-merky

Issue:
An in-memory store provides a simple token/structure storage interface
We want a simple interface for storing token/structure pairs, so we can build some higher-level abstractions around them without coupling ourselves to any particular storage mechanism (noting that the most obvious "storage" is an in-memory dict).

Do the in-memory case first, to work out the basics.  Probably should have:
* get by key
* set by key; return value should indicate whether key was actually new or not.
* serialize (to JSON, probably)
* deserialize (from JSON, probably)
* close (to flush and release any resources related to the thing)
* `__enter__`/`__exit__` for context manager behavior; close on exit.

Discussion:

Patch:
diff --git a/README.md b/README.md
index faeee9a..b504439 100644
--- a/README.md
+++ b/README.md
@@ -197,6 +197,44 @@ Some example uses of this:
 
 See `merky.cases.tokendict.TokenDict` for more.
 
+# Storage options
+
+Complex storage options are outside the scope of `merky`.  However, for convenience, `merky
+does provide some minimal storage options that can serve for some cases.
+
+An arbitrary transformed structure can be stored/restored using various classes found in the
+`merky.store.structure` module.  For instance:
+
+```python
+# Suppose this gives me some `merky.cases.attrgraph.AttributeGraph` instance.
+g = get_my_graph()
+
+# Now I'm going to store it a file.
+transformed = merky.AnnotationTransformer().transform(g)
+
+# Get a JSON file writer for the path 'my-graph.json'
+store = merky.store.structure.JSONFileWriteStructure('my-graph.json')
+
+# Populate the writer with the transformed data.
+store.populate(transformed)
+
+# Write the file.
+store.close()
+```
+
+Now later you want to restore it.  Like so:
+
+```python
+# Get the reader.
+store = merky.store.structure.JSONFileReadStructure('my-graph.json')
+
+# And away we go.
+g = merky.cases.attrgraph.AttributeGraph.from_token(store.head, store.get)
+```
+
+For a more complex example, see `merky.test.misc.graph_storage_test`, which
+assembles a `TokenDict` of `AttributeGraphs`, writes to JSON, then restores from JSON.
+
 # Don't wear a tie.
 
 It's an anachronistic absurdity that needs to be abolished.  Direct your respect elsewhere.
diff --git a/merky/cases/tokendict.py b/merky/cases/tokendict.py
index 3c6ebbc..aba991d 100644
--- a/merky/cases/tokendict.py
+++ b/merky/cases/tokendict.py
@@ -82,13 +82,14 @@ class TokenDict(object):
                        to be converted to a `TokenDict`.
             `reader`:  a function that, given a "token", returns the
                        corresponding structure.
-            `builder`: (Optional) a function to apply to each value in the
-                       `TokenDict` when assembling; use this to, for instance,
+            `builder`: (Optional) a function to apply to each token in the
+                       `TokenDict` when assembling; it will be called with the
+                       token seen and the `reader` above.  Ue this to, for instance,
                        convert each value into a more specific object.
 
         Returns a new `TokenDict` instance.
         """
         dict_ = reader(token)
-        handler = reader if builder is None else lambda val: builder(reader(val))
+        handler = reader if builder is None else lambda val: builder(val, reader)
         return cls(dict((k, handler(v)) for k, v in six.iteritems(dict_)))
 
diff --git a/merky/serialization.py b/merky/serialization.py
new file mode 100644
index 0000000..0e654ef
--- /dev/null
+++ b/merky/serialization.py
@@ -0,0 +1,14 @@
+import json
+
+def json_serializer(sort=True):
+    """
+    Returns a function that can serialize to JSON, with unicode enabled,
+    nans disallowed, keys sorted, and whitespace-free separators.
+
+    You can override the sorted keys via `sort=False`.
+    """
+    return json.JSONEncoder(ensure_ascii=False,
+                            allow_nan=False,
+                            sort_keys=sort,
+                            separators=(",",":")).encode
+
diff --git a/merky/store/__init__.py b/merky/store/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/merky/store/structure.py b/merky/store/structure.py
new file mode 100644
index 0000000..dabc581
--- /dev/null
+++ b/merky/store/structure.py
@@ -0,0 +1,152 @@
+import codecs
+import collections
+import json
+from .. import serialization
+
+class Structure(object):
+    """
+    Basic interface for a serializ(ed/able) "structure".
+
+    Without providing specifics for read/write operations, gives
+    the basic interface that these things should have in common.
+
+    The purpose of the structure is to combine:
+    * The map of token/structure pairs one gets from transformation
+    * A "head" token indicating which structure is the top-level
+      structure.
+
+    The combination of the `head` token and the `get` method would
+    allow for load of specific cases like the
+    `merky.cases.attrgraph.AttributeGraph` and similar (the `from_token`
+    classmethod).
+    """
+    def populate(self, token_structure_pairs):
+        raise NotImplementedError("populate() not implemented.")
+
+    def get(self, key):
+        raise NotImplementedError("get() not implemented.")
+
+    def __getitem__(self, key):
+        raise NotImplementedError("__getitem()__ not implemented.")
+
+
+
+class TokenMapStructure(Structure):
+    """
+    Maintains an internal in-memory map of token/structure pairs.
+
+    While the `head` can be explicitly set, the `head` defaults to the
+    last token seen via `populate()`.
+    """
+    @staticmethod
+    def default_tokenmap():
+        return collections.OrderedDict()
+
+    def populate(self, token_structure_pairs):
+        """
+        Ingests the `token_structure_pairs` into the internal token map.
+
+        Sets `head` to the last token seen, regardless of the current state of `head`.
+        """
+        self.tokenmap.update(token_structure_pairs)
+        self.head = next(iter(reversed(self.tokenmap))) if len(self.tokenmap) > 0 else None
+
+    def get(self, key):
+        """
+        Returns the corresponding structure from the internal map given the token `key`.
+        """
+        return self.tokenmap.get(key)
+
+    
+    def __getitem__(self, key):
+        """
+        Returns the corresponding structure from the internal map given the token 'key'.
+        """
+        return self.tokenmap[key]
+
+
+class InMemoryStructure(TokenMapStructure):
+    """
+    An in-memory "store" that has both the read and write interfaces.
+    """
+    def __init__(self, tokenmap=None, head=None):
+        self.tokenmap = self.default_tokenmap() if tokenmap is None else tokenmap
+        self.head = head
+
+    def close(self):
+        pass
+
+
+class JSONStreamReadStructure(TokenMapStructure):
+    """
+    Reads merkified structure from a utf-8 JSON stream.
+    """
+    def __init__(self, stream):
+        self.stream = stream
+        self.tokenmap, self.head = self.deserialize_from_stream(stream)
+
+    @classmethod
+    def deserialize_from_stream(cls, stream):
+        return json.load(stream, encoding='utf-8')
+
+
+class JSONStreamWriteStructure(TokenMapStructure):
+    """
+    Writes merkified structure to a utf-8 JSON stream.
+
+    The instance accumulates state internally and only serializes to stream
+    at close().
+    """
+    serializer = serialization.json_serializer(sort=False)
+
+    def __init__(self, stream):
+        self.tokenmap = self.default_tokenmap()
+        self.stream = stream
+
+    def serialize_to_stream(self, stream):
+        stream.write(self.serializer([self.tokenmap, self.head]))
+
+    def close(self):
+        self.serialize_to_stream(self.stream)
+        self.stream.flush()
+
+
+class JSONFileReadStructure(JSONStreamReadStructure):
+    """
+    Reads merkified structure from utf-8 JSON file.
+    """
+    def __init__(self, path):
+        self.path = path
+        self.tokenmap, self.head = self.deserialize_from_file(self.path)
+
+    @classmethod
+    def deserialize_from_file(cls, path):
+        with codecs.open(path, encoding="utf-8", mode="rb") as f:
+            return cls.deserialize_from_stream(f)
+
+
+class JSONFileWriteStructure(JSONStreamWriteStructure):
+    """
+    Writes merkified structure to utf-8 JSON file.
+
+    The instance accumulates state internally and only serializes to the file
+    at close().
+    """
+    def __init__(self, path):
+        self.tokenmap = self.default_tokenmap()
+        self.path = path
+
+    def serialize_to_file(self, path):
+        """
+        Writes the current state to the `path` specified.
+        """
+        with codecs.open(path, encoding="utf-8", mode="wb") as f:
+            self.serialize_to_stream(f)
+
+    def close(self):
+        """
+        Writes the state out to the file at `self.path`.
+        """
+        self.serialize_to_file(self.path)
+
+
diff --git a/merky/transformer.py b/merky/transformer.py
index b6be506..a4a370f 100644
--- a/merky/transformer.py
+++ b/merky/transformer.py
@@ -1,6 +1,5 @@
-import json
-
 from . import digest
+from . import serialization
 from . import tree
 
 class Transformer(object):
@@ -75,10 +74,8 @@ class Transformer(object):
         Subclasses could override this to use an alternate serialization approach; the result must
         support `encode()`.
         """
-        return json.JSONEncoder(ensure_ascii=False,
-                                allow_nan=False,
-                                sort_keys=True,
-                                separators=(",",":")).encode
+        return serialization.json_serializer()
+
 
     def get_dispatcher(self):
         """
diff --git a/setup.py b/setup.py
index a0eed15..a16dfee 100644
--- a/setup.py
+++ b/setup.py
@@ -11,7 +11,10 @@ setup(
     url = "https://github.com/ethanrowe/python-merky",
     packages = ["merky",
                 "merky.cases",
+                "merky.store",
                 "merky.test",
+                "merky.test.misc",
+                "merky.test.store",
                 "merky.test.usecases",
     ],
     long_description = """


Test patch:
diff --git a/merky/test/misc/__init__.py b/merky/test/misc/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/merky/test/misc/graph_storage_test.py b/merky/test/misc/graph_storage_test.py
new file mode 100644
index 0000000..f1a6984
--- /dev/null
+++ b/merky/test/misc/graph_storage_test.py
@@ -0,0 +1,96 @@
+import os
+import tempfile
+import shutil
+
+from nose import tools
+
+import merky
+from merky.cases import attrgraph
+from merky.cases import tokendict
+from merky.store import structure
+
+def tempdir(func):
+    def wrapper(*p, **kw):
+        dir_ = tempfile.mkdtemp()
+        try:
+            return func(*(p + (dir_,)), **kw)
+        finally:
+            shutil.rmtree(dir_)
+    wrapper.__name__ = func.__name__
+    return wrapper
+
+
+def dag(attrs, members):
+    return attrgraph.AttributeGraph(attrs=attrs, members=members)
+
+def getpath(reader, *keys):
+    current = reader.get(reader.head)
+    for key in keys:
+        current = reader.get(current[key])
+    return current
+
+@tempdir
+def test_versioned_graph_save_and_restore(workdir):
+    path = os.path.join(workdir, 'my-graph.json')
+    static = dag({"unchanging": "eternal"}, {})
+
+    graph_s0 = dag({"name": "graph", "version": 0}, {
+                    "static": static,
+                    "changing": dag({"a": "A"}, {}),
+                    "removed": dag({"bleep": "blorp"}, {}),
+                })
+
+    graph_s1 = dag({"name": "graph", "version": 1}, {
+                    "static": static,
+                    "changing": dag({"b": "B"}, {}),
+                })
+    
+    version_map = tokendict.TokenDict({"v0": graph_s0,
+                                       "v1": graph_s0,
+                                       "v2": graph_s1,
+                                       "v3": graph_s1})
+
+    transformer = merky.AnnotationTransformer()
+    writer = structure.JSONFileWriteStructure(path)
+    writer.populate(transformer.transform(version_map))
+    writer.close()
+
+    reader = structure.JSONFileReadStructure(path)
+    tokens = tokendict.TokenDict.from_token(reader.head, reader.get)
+
+    tools.assert_equal(["v0", "v1", "v2", "v3"], list(tokens.keys()))
+
+    # Same states get same tokens.
+    tools.assert_equal(tokens.dict_["v0"], tokens.dict_["v1"])
+    tools.assert_equal(tokens.dict_["v2"], tokens.dict_["v3"])
+    tools.assert_equal(getpath(reader, "v0", 1, "static"),
+                       getpath(reader, "v2", 1, "static"))
+
+    # Different states, different tokens.
+    tools.assert_not_equal(tokens.dict_["v1"], tokens.dict_["v2"])
+    tools.assert_not_equal(getpath(reader, "v0", 1, "changing"),
+                           getpath(reader, "v2", 1, "changing"))
+
+
+    restored = tokendict.TokenDict.from_token(
+        reader.head,
+        reader.get,
+        attrgraph.AttributeGraph.from_token
+    )
+
+    tools.assert_equal(["v0", "v1", "v2", "v3"],
+                        list(restored.keys()))
+
+    tools.assert_equal(dict(graph_s0.attrs), dict(restored["v0"].attrs))
+    tools.assert_equal(dict(graph_s1.attrs), dict(restored["v2"].attrs))
+    tools.assert_equal(["changing", "removed", "static"],
+                        list(sorted(restored["v0"].members.keys())))
+    tools.assert_equal(["changing", "static"],
+                        list(sorted(restored["v2"].members.keys())))
+    tools.assert_equal(dict(graph_s0.members["changing"].attrs),
+                        dict(restored["v1"].members["changing"].attrs))
+    tools.assert_equal(dict(graph_s1.members["changing"].attrs),
+                        dict(restored["v3"].members["changing"].attrs))
+    tools.assert_equal(dict(restored["v0"].members["static"].attrs),
+                        dict(restored["v3"].members["static"].attrs))
+
diff --git a/merky/test/store/__init__.py b/merky/test/store/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/merky/test/store/structure_test.py b/merky/test/store/structure_test.py
new file mode 100644
index 0000000..8a2f071
--- /dev/null
+++ b/merky/test/store/structure_test.py
@@ -0,0 +1,166 @@
+import codecs
+import collections
+import os
+import tempfile
+import shutil
+from nose import tools
+from six.moves import StringIO
+from .. import words
+from merky.store import structure
+
+def od(*p, **kw):
+    return collections.OrderedDict(*p, **kw)
+
+class TestInMemoryStructure(object):
+
+    HEAD_TOKEN = 'sometoptoken'
+    LIST_TOKEN = 'somelisttoken'
+    DICT_TOKEN = 'somedicttoken'
+    HEAD = od((('a', 'somelisttoken'), ('b', 'somedicttoken')))
+    DICT = od(((words.EUROS, words.SHEKELS), (words.EPEES, words.ANGSTROM)))
+    LIST = ["some", "list", words.SHEKELS]
+
+    PAIRS = (
+            (LIST_TOKEN, LIST),
+            (DICT_TOKEN, DICT),
+            (HEAD_TOKEN, HEAD),
+        )
+
+
+    def get_read_store(self):
+        return structure.InMemoryStructure(od(self.PAIRS), self.HEAD_TOKEN)
+
+    
+    def get_write_store(self):
+        return structure.InMemoryStructure()
+
+
+    def verify_dict(self, received, expect):
+        tools.assert_equal(expect, received)
+
+
+    def verify_list(self, received, expect):
+        tools.assert_equal(expect, received)
+
+
+    def verify_head(self, store, head):
+        tools.assert_equal(head, store.head)
+
+
+    def verify_write(self, store, head):
+        # In-memory guy has same state before and after close() call.
+        tools.assert_equal(od(self.PAIRS), store.tokenmap)
+        store.close()
+        tools.assert_equal(head, store.head)
+        tools.assert_equal(od(self.PAIRS), store.tokenmap)
+
+
+    def test_head_from_last_by_default(self):
+        store = self.get_write_store()
+        store.populate(iter(self.PAIRS))
+        self.verify_head(store, self.HEAD_TOKEN)
+        self.verify_write(store, self.HEAD_TOKEN)
+
+
+    def test_overriddable_head(self):
+        store = self.get_write_store()
+        store.populate(iter(self.PAIRS))
+        store.head = self.DICT_TOKEN
+        self.verify_head(store, self.DICT_TOKEN)
+        self.verify_write(store, self.DICT_TOKEN)
+
+
+    def test_read_operations(self):
+        store = self.get_read_store()
+        self.verify_head(store, self.HEAD_TOKEN)
+        self.verify_dict(store.get(self.HEAD_TOKEN), self.HEAD)
+        self.verify_dict(store[self.HEAD_TOKEN], self.HEAD)
+        self.verify_list(store.get(self.LIST_TOKEN), self.LIST)
+        self.verify_list(store[self.LIST_TOKEN], self.LIST)
+        self.verify_dict(store.get(self.DICT_TOKEN), self.DICT)
+        self.verify_dict(store[self.DICT_TOKEN], self.DICT)
+        tools.assert_equal(None, store.get('no no no'))
+        tools.assert_raises(KeyError, lambda: store['no no no'])
+
+
+q = lambda word: words.unify('"', word, '"')
+def commify(*w):
+    terms = []
+    for word in w[:-1]:
+        terms.append(word)
+        terms.append(',')
+    if len(w) > 0:
+        terms.append(w[-1])
+    return words.unify(*terms)
+
+jl = lambda *w: words.unify('[', commify(*w), ']')
+def jd(*pairs):
+    return words.unify('{',
+                            commify(*(words.unify(k, ':', w)
+                                      for k, w in pairs)),
+                        '}')
+
+
+class TestStreamStructure(TestInMemoryStructure):
+    def json(self, head):
+        return jl(
+            jd(
+                (q(self.LIST_TOKEN), jl(*(q(w) for w in self.LIST))),
+                (q(self.DICT_TOKEN), jd(*((q(k), q(v)) for k,v in self.DICT.items()))),
+                (q(self.HEAD_TOKEN), jd(*((q(k), q(v)) for k,v in self.HEAD.items()))),
+            ),
+            q(head),
+        )
+    
+    def get_read_store(self):
+        self.stream = StringIO(self.json(self.HEAD_TOKEN))
+        return structure.JSONStreamReadStructure(self.stream)
+
+    def get_write_store(self):
+        self.stream = StringIO()
+        return structure.JSONStreamWriteStructure(self.stream)
+
+    def verify_dict(self, received, expect):
+        tools.assert_equal(dict(expect), received)
+
+    def verify_list(self, received, expect):
+        tools.assert_equal(list(expect), received)
+
+    def verify_write(self, store, head):
+        # Nothing written to stream until close().
+        tools.assert_equal(0, self.stream.tell())
+        tools.assert_equal('', self.stream.read())
+        store.close()
+        self.stream.seek(0)
+        tools.assert_equal(self.json(head), self.stream.read())
+
+    def test_no_close(self):
+        store = self.get_read_store()
+        tools.assert_raises(AttributeError, getattr, store, 'close')
+
+
+class TestFileStructure(TestStreamStructure):
+    def setup(self):
+        self.workdir = tempfile.mkdtemp()
+        self.path = os.path.join(self.workdir, "some-file.json")
+    
+    def teardown(self):
+        shutil.rmtree(self.workdir)
+
+    def get_read_store(self):
+        with codecs.open(self.path, mode="wb", encoding="utf-8") as f:
+            f.write(self.json(self.HEAD_TOKEN))
+        return structure.JSONFileReadStructure(self.path)
+            
+    def get_write_store(self):
+        return structure.JSONFileWriteStructure(self.path)
+
+    def verify_write(self, store, head):
+        # File doesn't even exist before close.
+        tools.assert_false(os.path.exists(self.path))
+        # But it will after close()
+        store.close()
+        tools.assert_true(os.path.exists(self.path))
+        with codecs.open(self.path, mode='rb', encoding='utf-8') as f:
+            tools.assert_equal(self.json(head), f.read())
+
diff --git a/merky/test/usecases/tokendict_test.py b/merky/test/usecases/tokendict_test.py
index 57c109f..48d6341 100644
--- a/merky/test/usecases/tokendict_test.py
+++ b/merky/test/usecases/tokendict_test.py
@@ -67,9 +67,9 @@ def test_restoration():
 
 
 def test_restoration_with_builder():
-    def wrapper(value):
+    def wrapper(token, reader):
         def wrapped():
-            return ('called me!', value)
+            return ('called me!', reader(token))
         return wrapped
 
     g = tokendict.TokenDict.from_token(TOK.token, CACHE.get, wrapper)