Repository: lisa-lab/pylearn2

Issue:
Add input validation to yaml config parser
See pylearn-dev e-mail "Delitem Error"

Add input validation to yaml config parser
See pylearn-dev e-mail "Delitem Error"



Discussion:
Completed, just need to add unit test.

Completed, just need to add unit test.


Resolving patch:
diff --git a/pylearn2/config/yaml_parse.py b/pylearn2/config/yaml_parse.py
--- a/pylearn2/config/yaml_parse.py
+++ b/pylearn2/config/yaml_parse.py
@@ -6,11 +6,13 @@
 from pylearn2.utils.string_utils import preprocess
 from pylearn2.utils.string_utils import match
 import warnings
+import collections
 
 
 is_initialized = False
 additional_environ = None
 
+
 def load(stream, overrides=None, environ=None, **kwargs):
     """
     Loads a YAML configuration from a string or file-like object.
@@ -325,7 +327,7 @@ def initialize():
     is_initialized = True
 
 
-################################################################################3
+###############################################################################
 # Callbacks used by PyYAML
 
 def multi_constructor_obj(loader, tag_suffix, node):
@@ -335,7 +337,22 @@ def multi_constructor_obj(loader, tag_suffix, node):
     See PyYAML documentation for details on the call signature.
     """
     yaml_src = yaml.serialize(node)
+    construct_mapping(node)
     mapping = loader.construct_mapping(node)
+
+    assert hasattr(mapping, 'keys')
+    assert hasattr(mapping, 'values')
+    for key, val in mapping.iteritems():
+        if val is None:
+            message = "received None as the value for the key %s" % str(key)
+            raise TypeError(message)
+
+    for key in mapping.keys():
+        if not isinstance(key, basestring):
+            message = "Received non string object (%s) as" \
+                      "key in mapping." % str(key)
+            raise TypeError(message)
+
     if '.' not in tag_suffix:
         classname = tag_suffix
         rval = ObjectProxy(classname, mapping, yaml_src)
@@ -345,14 +362,16 @@ def multi_constructor_obj(loader, tag_suffix, node):
 
     return rval
 
+
 def multi_constructor_pkl(loader, tag_suffix, node):
     """
     Callback used by PyYAML when a "!pkl:" tag is encountered.
     """
     global additional_environ
     if tag_suffix != "" and tag_suffix != u"":
-        raise AssertionError('Expected tag_suffix to be "" but it is "' + tag_suffix +
-                    '": Put space between !pkl: and the filename.')
+        raise AssertionError('Expected tag_suffix to be "" but it is "'
+                             + tag_suffix +
+                             '": Put space between !pkl: and the filename.')
 
     mapping = loader.construct_yaml_str(node)
     rval = ObjectProxy(None, {}, yaml.serialize(node))
@@ -360,6 +379,7 @@ def multi_constructor_pkl(loader, tag_suffix, node):
 
     return rval
 
+
 def multi_constructor_import(loader, tag_suffix, node):
     """
     Callback used by PyYAML when a "!import:" tag is encountered.
@@ -368,6 +388,7 @@ def multi_constructor_import(loader, tag_suffix, node):
         raise yaml.YAMLError("!import: tag suffix contains no '.'")
     return try_to_import(tag_suffix)
 
+
 def constructor_import(loader, node):
     """
     Callback used by PyYAML when a "!import <str>" tag is encountered.
@@ -378,6 +399,7 @@ def constructor_import(loader, node):
         raise yaml.YAMLError("import tag suffix contains no '.'")
     return try_to_import(value)
 
+
 def constructor_float(loader, node):
     """
     Callback used by PyYAML when a "!float <str>" tag is encountered.
@@ -386,6 +408,37 @@ def constructor_float(loader, node):
     value = loader.construct_scalar(node)
     return float(value)
 
+
+def construct_mapping(node, deep=False):
+    # This is a modified version of yaml.BaseConstructor.construct_mapping
+    # in which a repeated key raises a ConstructorError
+    if not isinstance(node, yaml.nodes.MappingNode):
+        const = yaml.constructor
+        message = "expected a mapping node, but found"
+        raise const.ConstructorError(None, None,
+                                     "%s %s " % (message, node.id),
+                                     node.start_mark)
+    mapping = {}
+    constructor = yaml.constructor.BaseConstructor()
+    for key_node, value_node in node.value:
+        key = constructor.construct_object(key_node, deep=False)
+        try:
+            hash(key)
+        except TypeError, exc:
+            const = yaml.constructor
+            raise const.ConstructorError("while constructing a mapping",
+                                         node.start_mark,
+                                         "found unacceptable key (%s)" % exc,
+                                         key_node.start_mark)
+        if key in mapping:
+            const = yaml.constructor
+            raise const.ConstructorError("while constructing a mapping",
+                                         node.start_mark,
+                                         "found duplicate key (%s)" % key)
+        value = constructor.construct_object(value_node, deep=False)
+        mapping[key] = value
+    return mapping
+
 if __name__ == "__main__":
     initialize()
     # Demonstration of how to specify objects, reference them
@@ -394,7 +447,7 @@ def constructor_float(loader, node):
         "corruptor" : !obj:pylearn2.corruption.GaussianCorruptor &corr {
             "corruption_level" : 0.9
         },
-        "dae" : !obj:pylearn2.autoencoder.DenoisingAutoencoder {
+        "dae" : !obj:pylearn2.models.autoencoder.DenoisingAutoencoder {
             "nhid" : 20,
             "nvis" : 30,
             "act_enc" : null,


Test patch:
diff --git a/pylearn2/config/tests/test_yaml_parse.py b/pylearn2/config/tests/test_yaml_parse.py
--- a/pylearn2/config/tests/test_yaml_parse.py
+++ b/pylearn2/config/tests/test_yaml_parse.py
@@ -1,11 +1,23 @@
+"""
+Unit tests for ./yaml_parse.py
+"""
+
 import os
 import numpy as np
 import cPickle
 import tempfile
 from numpy.testing import assert_
-from pylearn2.config.yaml_parse import load, load_path
+from pylearn2.config.yaml_parse import load, load_path, initialize
 from os import environ
 from decimal import Decimal
+import yaml
+from pylearn2.models.mlp import MLP, Sigmoid
+
+from pylearn2.models.rbm import GaussianBinaryRBM
+from pylearn2.space import Conv2DSpace
+from pylearn2.linear.conv2d import make_random_conv2D
+from pylearn2.energy_functions.rbm_energy import grbm_type_1
+
 
 def test_load_path():
     fd, fname = tempfile.mkstemp()
@@ -15,29 +27,35 @@ def test_load_path():
     assert_(loaded['a'] == 23)
     os.remove(fname)
 
+
 def test_obj():
     loaded = load("a: !obj:decimal.Decimal { value : '1.23' }")
     assert_(isinstance(loaded['a'], Decimal))
 
+
 def test_floats():
     loaded = load("a: { a: -1.23, b: 1.23e-1 }")
     assert_(isinstance(loaded['a']['a'], float))
     assert_(isinstance(loaded['a']['b'], float))
-    assert_( (loaded['a']['a'] +1.23) < 1e-3 )
-    assert_( (loaded['a']['b'] -1.23e-1) < 1e-3)
+    assert_((loaded['a']['a'] + 1.23) < 1e-3)
+    assert_((loaded['a']['b'] - 1.23e-1) < 1e-3)
+
 
 def test_import():
     loaded = load("a: !import 'decimal.Decimal'")
     assert_(loaded['a'] == Decimal)
 
+
 def test_import_string():
     loaded = load("a: !import decimal.Decimal")
     assert_(loaded['a'] == Decimal)
 
+
 def test_import_colon():
     loaded = load("a: !import:decimal.Decimal")
     assert_(loaded['a'] == Decimal)
 
+
 def test_preproc_rhs():
     environ['TEST_VAR'] = '10'
     loaded = load('a: "${TEST_VAR}"')
@@ -45,6 +63,7 @@ def test_preproc_rhs():
     assert_(loaded['a'] == "10")
     del environ['TEST_VAR']
 
+
 def test_preproc_pkl():
     fd, fname = tempfile.mkstemp()
     with os.fdopen(fd, 'wb') as f:
@@ -55,16 +74,20 @@ def test_preproc_pkl():
     assert_(loaded['a'] == d)
     del environ['TEST_VAR']
 
+
 def test_late_preproc_pkl():
     fd, fname = tempfile.mkstemp()
     with os.fdopen(fd, 'wb') as f:
         array = np.arange(10)
         np.save(f, array)
     environ['TEST_VAR'] = fname
-    loaded = load('a: !obj:pylearn2.datasets.npy_npz.NpyDataset { file: "${TEST_VAR}" }\n')
-    assert_( loaded['a'].yaml_src.find("${TEST_VAR}") != -1 )   # Assert the unsubstituted TEST_VAR is in yaml_src
+    loaded = load('a: !obj:pylearn2.datasets.npy_npz.NpyDataset '
+                  '{ file: "${TEST_VAR}"}\n')
+    # Assert the unsubstituted TEST_VAR is in yaml_src
+    assert_(loaded['a'].yaml_src.find("${TEST_VAR}") != -1)
     del environ['TEST_VAR']
 
+
 def test_unpickle():
     fd, fname = tempfile.mkstemp()
     with os.fdopen(fd, 'wb') as f:
@@ -74,6 +97,7 @@ def test_unpickle():
     assert_(loaded['a'] == d)
     os.remove(fname)
 
+
 def test_unpickle_key():
     fd, fname = tempfile.mkstemp()
     with os.fdopen(fd, 'wb') as f:
@@ -84,3 +108,90 @@ def test_unpickle_key():
     assert_(loaded.values()[0] == 50)
     os.remove(fname)
 
+
+def test_multi_constructor_obj():
+    """
+    Tests whether multi_constructor_obj throws an exception when
+    the keys in mapping are None.
+    """
+    try:
+        loaded = load("a: !obj:decimal.Decimal { 1 }")
+    except TypeError as e:
+        assert str(e) == "received None as the value for the key 1"
+        pass
+    except Exception, e:
+        error_msg = "Got the unexpected error: %s" % (e)
+        raise ValueError(error_msg)
+
+
+def test_duplicate_keywords():
+    """
+    Tests whether there are doublicate keywords in the yaml
+    """
+    initialize()
+    yamlfile = """{
+            "model": !obj:pylearn2.models.mlp.MLP {
+            "layers": [
+                     !obj:pylearn2.models.mlp.Sigmoid {
+                         "layer_name": 'h0',
+                         "dim": 20,
+                         "sparse_init": 15,
+                     }],
+            "nvis": 784,
+            "nvis": 384,
+        }
+    }"""
+
+    try:
+        loaded = load(yamlfile)
+    except yaml.constructor.ConstructorError, e:
+        message = str(e)
+        assert message.endswith("found duplicate key (nvis)")
+        pass
+    except Exception, e:
+        error_msg = "Got the unexpected error: %s" % (e)
+        raise TypeError(error_msg)
+
+
+def test_duplicate_keywords_2():
+    """
+    Tests whether duplicate keywords as independent parameters works fine.
+    """
+    initialize()
+    yamlfile = """{
+             "model": !obj:pylearn2.models.rbm.GaussianBinaryRBM {
+
+                 "vis_space" : &vis_space !obj:pylearn2.space.Conv2DSpace {
+                    "shape" : [32,32],
+                    "num_channels" : 3
+                },
+                "hid_space" : &hid_space !obj:pylearn2.space.Conv2DSpace {
+                    "shape" : [27,27],
+                    "num_channels" : 10
+                },
+                "transformer" :
+                        !obj:pylearn2.linear.conv2d.make_random_conv2D {
+                    "irange" : .05,
+                    "input_space" : *vis_space,
+                    "output_space" : *hid_space,
+                    "kernel_shape" : [6,6],
+                    "batch_size" : &batch_size 5
+                },
+                "energy_function_class" :
+                     !obj:pylearn2.energy_functions.rbm_energy.grbm_type_1 {},
+                "learn_sigma" : True,
+                "init_sigma" : .3333,
+                "init_bias_hid" : -2.,
+                "mean_vis" : False,
+                "sigma_lr_scale" : 1e-3
+
+             }
+    }"""
+
+    loaded = load(yamlfile)
+
+if __name__ == "__main__":
+    test_multi_constructor_obj()
+    test_duplicate_keywords()
+    test_duplicate_keywords_2()
+    test_unpickle_key()