Repository: rwth-i6/returnn

Issue:
Obj_diff_str crashing
Network construction is crashing with the following issue.
```python
EXCEPTION
Traceback (most recent call last):
  File "/u/hilmes/experiments/tts_new_sis/work/i6_core/tools/git/CloneGitRepositoryJob.rYBuJf2ULa45/output/repository/rnn.py", line 11, in <module>
    line: main()
    locals:
      main = <local> <function main at 0x147f9a8fce50>
  File "/work/asr3/rossenbach/hilmes/sisyphus_experiments/new_tts/i6_core/tools/git/CloneGitRepositoryJob.rYBuJf2ULa45/output/repository/returnn/__main__.py", line 657, in main
    line: execute_main_task()
    locals:
      execute_main_task = <global> <function execute_main_task at 0x147f9a8fcd30>
  File "/work/asr3/rossenbach/hilmes/sisyphus_experiments/new_tts/i6_core/tools/git/CloneGitRepositoryJob.rYBuJf2ULa45/output/repository/returnn/__main__.py", line 456, in execute_main_task
    line: engine.train()
    locals:
      engine = <global> <returnn.tf.engine.Engine object at 0x147f660f8d90>
      engine.train = <global> <bound method Engine.train of <returnn.tf.engine.Engine object at 0x147f660f8d90>>
  File "/work/asr3/rossenbach/hilmes/sisyphus_experiments/new_tts/i6_core/tools/git/CloneGitRepositoryJob.rYBuJf2ULa45/output/repository/returnn/tf/engine.py", line 1449, in Engine.train
    line: self.init_train_epoch()
    locals:
      self = <local> <returnn.tf.engine.Engine object at 0x147f660f8d90>
      self.init_train_epoch = <local> <bound method Engine.init_train_epoch of <returnn.tf.engine.Engine object at 0x147f660f8d90>>
  File "/work/asr3/rossenbach/hilmes/sisyphus_experiments/new_tts/i6_core/tools/git/CloneGitRepositoryJob.rYBuJf2ULa45/output/repository/returnn/tf/engine.py", line 1474, in Engine.init_train_epoch
    line: self.init_new_network(new_network_desc)
    locals:
      self = <local> <returnn.tf.engine.Engine object at 0x147f660f8d90>
      self.init_new_network = <local> <bound method Engine.init_new_network of <returnn.tf.engine.Engine object at 0x147f660f8d90>>
      new_network_desc = <local> {'ctc_model_mean_absolute_difference_reduce': {'class': 'copy', 'from': 'mean_absolute_difference', 'loss': 'as_is', 'out_shape': {Dim{'phonemes_time'[B(-1)]}, Dim{B}}}, 'ctc_model_mean_absolute_difference_0_reduce': {'class': 'copy', 'from': 'mean_absolute_difference_0', 'loss': 'as_is', 'out_sh..., len = 23
  File "/work/asr3/rossenbach/hilmes/sisyphus_experiments/new_tts/i6_core/tools/git/CloneGitRepositoryJob.rYBuJf2ULa45/output/repository/returnn/tf/engine.py", line 1382, in Engine.init_new_network
    line: print("reinit because network description differs. Diff:",
                dict_diff_str(self.network.layers_desc, net_desc), file=log.v3)
    locals:
      print = <builtin> <built-in function print>
      dict_diff_str = <local> <function dict_diff_str at 0x147f9a9615e0>
      self = <local> <returnn.tf.engine.Engine object at 0x147f660f8d90>
      self.network = <local> <TFNetwork '' train=<tf.Tensor 'globals/train_flag:0' shape=() dtype=bool>>
      self.network.layers_desc = <local> {'ctc_model_mean_absolute_difference_reduce': {'class': 'copy', 'from': 'mean_absolute_difference', 'loss': 'as_is', 'out_shape': {Dim{'phonemes_time'[B]}, Dim{B}}}, 'ctc_model_mean_absolute_difference_0_reduce': {'class': 'copy', 'from': 'mean_absolute_difference_0', 'loss': 'as_is', 'out_shape'..., len = 23
      net_desc = <local> {'ctc_model_mean_absolute_difference_reduce': {'class': 'copy', 'from': 'mean_absolute_difference', 'loss': 'as_is', 'out_shape': {Dim{'phonemes_time'[B(-1)]}, Dim{B}}}, 'ctc_model_mean_absolute_difference_0_reduce': {'class': 'copy', 'from': 'mean_absolute_difference_0', 'loss': 'as_is', 'out_sh..., len = 23
      file = <not found>
      log = <global> <returnn.log.Log object at 0x147f9a9955e0>
      log.v3 = <global> <returnn.log.Stream object at 0x147f661968e0>
  File "/work/asr3/rossenbach/hilmes/sisyphus_experiments/new_tts/i6_core/tools/git/CloneGitRepositoryJob.rYBuJf2ULa45/output/repository/returnn/util/basic.py", line 1111, in dict_diff_str
    line: return obj_diff_str(DictAsObj(self), DictAsObj(other))
    locals:
      obj_diff_str = <global> <function obj_diff_str at 0x147f9a961550>
      DictAsObj = <global> <class 'returnn.util.basic.DictAsObj'>
      self = <local> {'ctc_model_mean_absolute_difference_reduce': {'class': 'copy', 'from': 'mean_absolute_difference', 'loss': 'as_is', 'out_shape': {Dim{'phonemes_time'[B]}, Dim{B}}}, 'ctc_model_mean_absolute_difference_0_reduce': {'class': 'copy', 'from': 'mean_absolute_difference_0', 'loss': 'as_is', 'out_shape'..., len = 23
      other = <local> {'ctc_model_mean_absolute_difference_reduce': {'class': 'copy', 'from': 'mean_absolute_difference', 'loss': 'as_is', 'out_shape': {Dim{'phonemes_time'[B(-1)]}, Dim{B}}}, 'ctc_model_mean_absolute_difference_0_reduce': {'class': 'copy', 'from': 'mean_absolute_difference_0', 'loss': 'as_is', 'out_sh..., len = 23
  File "/work/asr3/rossenbach/hilmes/sisyphus_experiments/new_tts/i6_core/tools/git/CloneGitRepositoryJob.rYBuJf2ULa45/output/repository/returnn/util/basic.py", line 1094, in obj_diff_str
    line: s += ["  " + line for line in dict_diff_str(value_self, value_other).splitlines()]
    locals:
      s = <local> ["attrib 'concat' dict differs:", "  attrib 'from' differ. self: (('enc_lstm_fw', Dim{F'enc_lstm_dim'(256)}), ('enc_lstm_bw', Dim{F'enc_lstm_dim'(256)})), other: (('enc_lstm_fw', Dim{F'enc_lstm_dim'(256)}), ('enc_lstm_bw', Dim{F'enc_lstm_dim'(256)}))", "  attrib 'out_shape' differ. self: {Dim{'ph..., len = 530, _[0]: {len = 29}
      line = <not found>
      dict_diff_str = <global> <function dict_diff_str at 0x147f9a9615e0>
      value_self = <local> {'class': 'reinterpret_data', 'set_dim_tags': {Dim{F'audio_features_feature'(80)}: Dim{F'dec_linear'(80)}}, 'from': 'data:audio_features', 'out_shape': {Dim{'audio_features_time'[B(-1)]}, Dim{B}, Dim{F'dec_linear'(80)}}}
      value_other = <local> {'class': 'reinterpret_data', 'set_dim_tags': {Dim{F'audio_features_feature'(80)}: Dim{F'dec_linear'(80)}}, 'from': 'data:audio_features', 'out_shape': {Dim{'audio_features_time'[B(-1)]}, Dim{B}, Dim{F'dec_linear'(80)}}}
      splitlines = <not found>
  File "/work/asr3/rossenbach/hilmes/sisyphus_experiments/new_tts/i6_core/tools/git/CloneGitRepositoryJob.rYBuJf2ULa45/output/repository/returnn/util/basic.py", line 1111, in dict_diff_str
    line: return obj_diff_str(DictAsObj(self), DictAsObj(other))
    locals:
      obj_diff_str = <global> <function obj_diff_str at 0x147f9a961550>
      DictAsObj = <global> <class 'returnn.util.basic.DictAsObj'>
      self = <local> {'class': 'reinterpret_data', 'set_dim_tags': {Dim{F'audio_features_feature'(80)}: Dim{F'dec_linear'(80)}}, 'from': 'data:audio_features', 'out_shape': {Dim{'audio_features_time'[B(-1)]}, Dim{B}, Dim{F'dec_linear'(80)}}}
      other = <local> {'class': 'reinterpret_data', 'set_dim_tags': {Dim{F'audio_features_feature'(80)}: Dim{F'dec_linear'(80)}}, 'from': 'data:audio_features', 'out_shape': {Dim{'audio_features_time'[B(-1)]}, Dim{B}, Dim{F'dec_linear'(80)}}}
  File "/work/asr3/rossenbach/hilmes/sisyphus_experiments/new_tts/i6_core/tools/git/CloneGitRepositoryJob.rYBuJf2ULa45/output/repository/returnn/util/basic.py", line 1094, in obj_diff_str
    line: s += ["  " + line for line in dict_diff_str(value_self, value_other).splitlines()]
    locals:
      s = <local> ["attrib 'out_shape' differ. self: {Dim{'audio_features_time'[B(-1)]}, Dim{B}, Dim{F'dec_linear'(80)}}, other: {Dim{'audio_features_time'[B(-1)]}, Dim{B}, Dim{F'dec_linear'(80)}}", "attrib 'set_dim_tags' dict differs:"], _[0]: {len = 176}
      line = <not found>
      dict_diff_str = <global> <function dict_diff_str at 0x147f9a9615e0>
      value_self = <local> {Dim{F'audio_features_feature'(80)}: Dim{F'dec_linear'(80)}}
      value_other = <local> {Dim{F'audio_features_feature'(80)}: Dim{F'dec_linear'(80)}}
      splitlines = <not found>
  File "/work/asr3/rossenbach/hilmes/sisyphus_experiments/new_tts/i6_core/tools/git/CloneGitRepositoryJob.rYBuJf2ULa45/output/repository/returnn/util/basic.py", line 1111, in dict_diff_str
    line: return obj_diff_str(DictAsObj(self), DictAsObj(other))
    locals:
      obj_diff_str = <global> <function obj_diff_str at 0x147f9a961550>
      DictAsObj = <global> <class 'returnn.util.basic.DictAsObj'>
      self = <local> {Dim{F'audio_features_feature'(80)}: Dim{F'dec_linear'(80)}}
      other = <local> {Dim{F'audio_features_feature'(80)}: Dim{F'dec_linear'(80)}}
  File "/work/asr3/rossenbach/hilmes/sisyphus_experiments/new_tts/i6_core/tools/git/CloneGitRepositoryJob.rYBuJf2ULa45/output/repository/returnn/util/basic.py", line 1078, in obj_diff_str
    line: value_self = getattr(self, attrib, not_specified)
    locals:
      value_self = <not found>
      getattr = <builtin> <built-in function getattr>
      self = <local> <returnn.util.basic.DictAsObj object at 0x147ea5244df0>
      attrib = <local> Dim{F'audio_features_feature'(80)}
      not_specified = <local> <not-specified>
TypeError: getattr(): attribute name must be string
```


Discussion:

Resolving patch:
diff --git a/returnn/util/basic.py b/returnn/util/basic.py
--- a/returnn/util/basic.py
+++ b/returnn/util/basic.py
@@ -1038,7 +1038,7 @@ def dict_joined(*ds):
   return res
 
 
-def obj_diff_str(self, other):
+def obj_diff_str(self, other, **kwargs):
   """
   :param object self:
   :param object other:
@@ -1053,6 +1053,9 @@ def obj_diff_str(self, other):
     return "other is None and self is %r" % self
   if self == other:
     return "No diff."
+  name = kwargs.pop("attr_type_name", "attrib")  # or "item"
+  if kwargs:
+    raise TypeError("obj_diff_str: invalid kwargs %r" % kwargs)
   s = []
 
   def _obj_attribs(obj):
@@ -1075,26 +1078,26 @@ def __repr__(self):
   if self_attribs is None or other_attribs is None:
     return "self: %r, other: %r" % (self, other)
   for attrib in sorted(set(self_attribs).union(other_attribs)):
-    value_self = getattr(self, attrib, not_specified)
-    value_other = getattr(other, attrib, not_specified)
+    value_self = self.__dict__.get(attrib, not_specified)
+    value_other = other.__dict__.get(attrib, not_specified)
     if isinstance(value_self, list):
       if not isinstance(value_other, list):
-        s += ["attrib %r self is list but other is %r" % (attrib, type(value_other))]
+        s += ["%s %r self is list but other is %r" % (name, attrib, type(value_other))]
       elif len(value_self) != len(value_other):
-        s += ["attrib %r list differ. len self: %i, len other: %i" % (attrib, len(value_self), len(value_other))]
+        s += ["%s %r list differ. len self: %i, len other: %i" % (name, attrib, len(value_self), len(value_other))]
       else:
         for i, (a, b) in enumerate(zip(value_self, value_other)):
           if a != b:
-            s += ["attrib %r[%i] differ. self: %r, other: %r" % (attrib, i, a, b)]
+            s += ["%s %r[%i] differ. self: %r, other: %r" % (name, attrib, i, a, b)]
     elif isinstance(value_self, dict):
       if not isinstance(value_other, dict):
-        s += ["attrib %r self is dict but other is %r" % (attrib, type(value_other))]
+        s += ["%s %r self is dict but other is %r" % (name, attrib, type(value_other))]
       elif value_self != value_other:
-        s += ["attrib %r dict differs:" % attrib]
+        s += ["%s %r dict differs:" % (name, attrib)]
         s += ["  " + line for line in dict_diff_str(value_self, value_other).splitlines()]
     else:
       if value_self != value_other:
-        s += ["attrib %r differ. self: %r, other: %r" % (attrib, value_self, value_other)]
+        s += ["%s %r differ. self: %r, other: %r" % (name, attrib, value_self, value_other)]
   if s:
     return "\n".join(s)
   else:
@@ -1108,7 +1111,7 @@ def dict_diff_str(self, other):
   :return: the difference described
   :rtype: str
   """
-  return obj_diff_str(DictAsObj(self), DictAsObj(other))
+  return obj_diff_str(DictAsObj(self), DictAsObj(other), attr_type_name="item")
 
 
 def find_ranges(ls):


Test patch:
diff --git a/tests/test_Util.py b/tests/test_Util.py
--- a/tests/test_Util.py
+++ b/tests/test_Util.py
@@ -264,6 +264,14 @@ def __init__(self, a, b=13):
   assert_equal(x_repr, "X(a=42, b=13)")
 
 
+def test_dict_diff_str():
+  assert_equal(dict_diff_str({"a": 1, "b": 2}, {"a": 1, "b": 3}), "item 'b' differ. self: 2, other: 3")
+
+
+def test_dict_diff_str_non_str_key():
+  assert_equal(dict_diff_str({1: 1, 2: 2}, {1: 1, 2: 3}), "item 2 differ. self: 2, other: 3")
+
+
 @unittest.skipIf(PY3, "only for Python 2")
 def test_py2_utf8_str_to_unicode():
   assert_equal(py2_utf8_str_to_unicode("a"), "a")