Repository: cython/cython

Issue:
Deduplicate constant tuples and slices
Cython currently generates compile-time constant slices and tuples (also some call argument tuples) as global constants and creates them once at module initialisation time. However, these constants tend to be widely the same (e.g. a slice like `::-1` or 1-argument tuples), so deduplicating these constants and creating them only once would reduce the memory footprint, the module file size, and the initialisation time.

This is not entirely trivial because it is not enough to test for equality. `(1, 1.0) == (1.0, 1)` in Python, as well as `u'abc' == b'abc'` in Py2.x, so these cases need to be taken into account. A `repr()` comparison might work, though, as should a more sophisticated comparison that involves the object types. For example, going recursively through tuples (of tuples) and collecting `(type(item), item)` pairs for each item in the tuple structure should generate a safe key.

Also note that constant tuples can contain tuples themselves.

See #2102 regarding the module size problem.


Discussion:

Resolving patch:
diff --git a/Cython/Compiler/Code.py b/Cython/Compiler/Code.py
--- a/Cython/Compiler/Code.py
+++ b/Cython/Compiler/Code.py
@@ -1134,6 +1134,7 @@ def __init__(self, writer, module_node, code_config, common_utility_include_dir=
 
         self.const_cnames_used = {}
         self.string_const_index = {}
+        self.dedup_const_index = {}
         self.pyunicode_ptr_const_index = {}
         self.num_const_index = {}
         self.py_constants = []
@@ -1265,13 +1266,19 @@ def get_float_const(self, str_value, value_code):
             c = self.new_num_const(str_value, 'float', value_code)
         return c
 
-    def get_py_const(self, type, prefix='', cleanup_level=None):
+    def get_py_const(self, type, prefix='', cleanup_level=None, dedup_key=None):
+        if dedup_key is not None:
+            const = self.dedup_const_index.get(dedup_key)
+            if const is not None:
+                return const
         # create a new Python object constant
         const = self.new_py_const(type, prefix)
         if cleanup_level is not None \
                 and cleanup_level <= Options.generate_cleanup_code:
             cleanup_writer = self.parts['cleanup_globals']
             cleanup_writer.putln('Py_CLEAR(%s);' % const.cname)
+        if dedup_key is not None:
+            self.dedup_const_index[dedup_key] = const
         return const
 
     def get_string_const(self, text, py_version=None):
@@ -1792,8 +1799,8 @@ def get_py_int(self, str_value, longness):
     def get_py_float(self, str_value, value_code):
         return self.globalstate.get_float_const(str_value, value_code).cname
 
-    def get_py_const(self, type, prefix='', cleanup_level=None):
-        return self.globalstate.get_py_const(type, prefix, cleanup_level).cname
+    def get_py_const(self, type, prefix='', cleanup_level=None, dedup_key=None):
+        return self.globalstate.get_py_const(type, prefix, cleanup_level, dedup_key).cname
 
     def get_string_const(self, text):
         return self.globalstate.get_string_const(text).cname
diff --git a/Cython/Compiler/ExprNodes.py b/Cython/Compiler/ExprNodes.py
--- a/Cython/Compiler/ExprNodes.py
+++ b/Cython/Compiler/ExprNodes.py
@@ -187,6 +187,29 @@ def infer_sequence_item_type(env, seq_node, index_node=None, seq_type=None):
             return item_types.pop()
     return None
 
+
+def make_dedup_key(outer_type, item_nodes):
+    """
+    Recursively generate a deduplication key from a sequence of values.
+    Includes Cython node types to work around the fact that (1, 2.0) == (1.0, 2), for example.
+
+    @param outer_type: The type of the outer container.
+    @param item_nodes: A sequence of constant nodes that will be traversed recursively.
+    @return: A tuple that can be used as a dict key for deduplication.
+    """
+    item_keys = [
+        (py_object_type, None) if node is None
+        else make_dedup_key(node.type, node.args) if node.is_sequence_constructor
+        else make_dedup_key(node.type, (node.start, node.stop, node.step)) if node.is_slice
+        else (node.type, node.constant_result) if node.has_constant_result()
+        else None
+        for node in item_nodes
+    ]
+    if None in item_keys:
+        return None
+    return outer_type, tuple(item_keys)
+
+
 # Returns a block of code to translate the exception,
 # plus a boolean indicating whether to check for Python exceptions.
 def get_exception_handler(exception_value):
@@ -5228,7 +5251,8 @@ def calculate_result_code(self):
 
     def generate_result_code(self, code):
         if self.is_literal:
-            self.result_code = code.get_py_const(py_object_type, 'slice', cleanup_level=2)
+            dedup_key = make_dedup_key(self.type, (self,))
+            self.result_code = code.get_py_const(py_object_type, 'slice', cleanup_level=2, dedup_key=dedup_key)
             code = code.get_cached_constants_writer()
             code.mark_pos(self.pos)
 
@@ -7961,7 +7985,10 @@ def generate_operation_code(self, code):
             return
 
         if self.is_literal or self.is_partly_literal:
-            tuple_target = code.get_py_const(py_object_type, 'tuple', cleanup_level=2)
+            dedup_key = None
+            if self.is_literal:
+                dedup_key = make_dedup_key(self.type, self.args)
+            tuple_target = code.get_py_const(py_object_type, 'tuple', cleanup_level=2, dedup_key=dedup_key)
             const_code = code.get_cached_constants_writer()
             const_code.mark_pos(self.pos)
             self.generate_sequence_packing_code(const_code, tuple_target, plain=not self.is_literal)


Test patch: