Repository: mahmoud/glom

Issue:
KeyError in _t_child from weakref
I'm running into a weird issue using glom in PySpark on Databricks.

This expression:

`glom(ping, (T[stub]["values"].values(), sum), default=0)`

(where `stub` is `"a11y_time"`)

is consistently throwing this exception when I run it on my real data:

```
/databricks/spark/python/lib/py4j-0.10.6-src.zip/py4j/protocol.py in get_return_value(answer, gateway_client, target_id, name)
     318 raise Py4JJavaError(
     319 "An error occurred while calling {0}{1}{2}.\n". 
 --> 320                     format(target_id, ".", name), value)
     321 else:
     322 raise Py4JError( Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.runJob. : org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 22.0 failed 4 times, most recent failure: Lost task 0.3 in stage 22.0 (TID 243, 10.166.248.213, executor 2): org.apache.spark.api.python.PythonException: 

Traceback (most recent call last):
File "/databricks/spark/python/pyspark/worker.py", line 229, in main process()
File "/databricks/spark/python/pyspark/worker.py", line 224, in process serializer.dump_stream(func(split_index, iterator), outfile)
File "/databricks/spark/python/pyspark/serializers.py", line 372, in dump_stream vs = list(itertools.islice(iterator, batch))
File "/databricks/spark/python/pyspark/rdd.py", line 1354, in takeUpToNumLeft yield next(iterator) 
File "<command-26292>", line 10, in to_row
File "<command-26292>", line 5, in histogram_measures
File "/databricks/python/local/lib/python2.7/site-packages/glom/core.py", line 753, in __getitem__ return _t_child(self, '[', item)
File "/databricks/python/local/lib/python2.7/site-packages/glom/core.py", line 791, in _t_child _T_PATHS[t] = _T_PATHS[parent] + (operation, arg)
File "/usr/lib/python2.7/weakref.py", line 330, in __getitem__ return self.data[ref(key)]
KeyError: <weakref at 0x7f84c7d2f6d8; to '_TType' at 0x7f84c8933f30>
```

The object that's crashing it is, itself, totally unremarkable:

`{'submission_date': u'20180718', 'a11y_count': None, 'a11y_node_inspected_count': None, 'a11y_service_time': None, 'toolbox_time': None, 'toolbox_count': None, 'a11y_time': None, 'branch': u'Treatment', 'client_id': u'some-random-uuid', 'a11y_picker_time': None, 'a11y_select_accessible_for_node': None} `

The Python that Databricks is running looks like `2.7.12 (default, Dec 4 2017, 14:50:18) [GCC 5.4.0 20160609]`.

I can't reproduce it on my Mac in 2.7.14 or 2.7.12.

Discussion:

Patch:
diff --git a/glom/core.py b/glom/core.py
index 69ae4dc..9cbd27b 100644
--- a/glom/core.py
+++ b/glom/core.py
@@ -721,6 +721,12 @@ class _TType(object):
     def __repr__(self):
         return _format_t(_T_PATHS[self][1:])
 
+    def __getstate__(self):
+        return tuple(_T_PATHS[self])
+
+    def __setstate__(self, state):
+        _T_PATHS[self] = state
+
 
 def _format_t(path):
     def kwarg_fmt(kw):


Test patch:
diff --git a/glom/test/test_path_and_t.py b/glom/test/test_path_and_t.py
index 394525f..614899f 100644
--- a/glom/test/test_path_and_t.py
+++ b/glom/test/test_path_and_t.py
@@ -69,3 +69,19 @@ def test_path_access_error_message():
     assert ("PathAccessError: could not access 'b', part 1 of Path('a', T.b), got error: AttributeError"
             in exc_info.exconly())
     assert repr(exc_info.value) == """PathAccessError(AttributeError("\'dict\' object has no attribute \'b\'",), Path(\'a\', T.b), 1)"""
+
+
+def test_t_picklability():
+    import pickle
+
+    class TargetType(object):
+        def __init__(self):
+            self.attribute = lambda: None
+            self.attribute.method = lambda: {'key': lambda x: x * 2}
+
+    spec = T.attribute.method()['key'](x=5)
+
+    rt_spec = pickle.loads(pickle.dumps(spec))
+    assert repr(spec) == repr(rt_spec)
+
+    assert glom(TargetType(), spec) == 10