Repository: socialpoint-labs/unity-yaml-parser

Issue:
"-0" is not preserved in original file
This is probably because it is parsed as int by unityparser, while unity probably parses it as float, where the "-0.0" can be preserved.

Happy to open a PR, but not sure where to start.

Discussion:
@sp-ricard-valverde: I'm getting to the conclusion that we need to treat every `float` and `int` value as `str` to preserve all uniqueness in Unity formatting the documents as it seems impossible to track every use case.

When the library was released we tried to be clever about parsing some numeric values so they were available to manipulate in Python but I think the safest approach would be to let the user's Python code convert a given str value to a Python type before manipulation if required. If numeric values are not manipulated in any way they will be dumped back in the same exact str format the were serialized.@rafzi: Not sure if the following is related, so i create a new issue: https://github.com/socialpoint-labs/unity-yaml-parser/issues/67

Solution patch:
diff --git a/unityparser/dumper.py b/unityparser/dumper.py
index d67158a..c37b47f 100644
--- a/unityparser/dumper.py
+++ b/unityparser/dumper.py
@@ -19,9 +19,6 @@ class UnityDumper(Emitter, Serializer, Representer, Resolver):
                  allow_unicode=None, line_break=None,
                  encoding=None, explicit_start=None, explicit_end=None,
                  version=None, tags=None, sort_keys=True, register=None):
-        tags = tags or {}
-        tags.update(UNITY_TAG)
-        version = version or VERSION
         Emitter.__init__(self, stream, canonical=canonical,
                          indent=indent, width=width,
                          allow_unicode=allow_unicode, line_break=line_break)
diff --git a/unityparser/loader.py b/unityparser/loader.py
index b8664dd..5d3b8d2 100644
--- a/unityparser/loader.py
+++ b/unityparser/loader.py
@@ -5,7 +5,7 @@ from .composer import Composer
 from .constants import UNITY_TAG_URI, OrderedFlowDict, UnityClassIdMap
 from .constructor import Constructor
 from .parser import Parser
-from .resolver import Resolver
+from .resolver import Resolver, SmartResolver
 
 
 class UpgradeVersionError(Exception):
@@ -38,6 +38,17 @@ class UnityLoader(Reader, Scanner, Parser, Composer, Constructor, Resolver):
         Resolver.__init__(self)
 
 
+class SmartUnityLoader(Reader, Scanner, Parser, Composer, Constructor, SmartResolver):
+
+    def __init__(self, stream, register=None):
+        Reader.__init__(self, stream)
+        Scanner.__init__(self)
+        Parser.__init__(self)
+        Composer.__init__(self)
+        Constructor.__init__(self, register)
+        SmartResolver.__init__(self)
+
+
 def construct_unity_class(loader, tag_suffix, node):
     try:
         classid = tag_suffix
@@ -66,3 +77,5 @@ def construct_yaml_map(loader, node):
 
 UnityLoader.add_constructor('tag:yaml.org,2002:map', construct_yaml_map)
 UnityLoader.add_multi_constructor(UNITY_TAG_URI, construct_unity_class)
+SmartUnityLoader.add_constructor('tag:yaml.org,2002:map', construct_yaml_map)
+SmartUnityLoader.add_multi_constructor(UNITY_TAG_URI, construct_unity_class)
diff --git a/unityparser/parser.py b/unityparser/parser.py
index 6b64cee..d76be4a 100644
--- a/unityparser/parser.py
+++ b/unityparser/parser.py
@@ -8,6 +8,7 @@ class Parser(YamlParser):
     def __init__(self):
         super(Parser, self).__init__()
         self.parsing_inverted_scalar = False
+        self.non_default_tags = None
 
     def parse_document_start(self):
 
@@ -22,6 +23,9 @@ class Parser(YamlParser):
             # UNITY: only process directives(version and tags) on the first document
             if self.check_prev_token(StreamStartToken):
                 version, tags = self.process_directives()
+                # UNITY: keep track of tags explicitly defined in the document
+                if self.non_default_tags is None:
+                    self.non_default_tags = tags
             else:
                 version, tags = self.yaml_version, self.tag_handles.copy() if self.tag_handles else None
             if not self.check_token(DocumentStartToken):
diff --git a/unityparser/resolver.py b/unityparser/resolver.py
index 4167bf6..1fc22a4 100644
--- a/unityparser/resolver.py
+++ b/unityparser/resolver.py
@@ -18,17 +18,6 @@ class Resolver(BaseResolver):
 #                     |on|On|ON|off|Off|OFF)$''', re.X),
 #     list('yYnNtTfFoO'))
 
-Resolver.add_implicit_resolver(
-    'tag:yaml.org,2002:float',
-    re.compile(r'''^(?:[-+]?(?:0|[1-9][0-9]*)\.(?:[1-9]|[0-9][0-9]*[1-9])?)$''', re.X),
-    list('-+0123456789.'))
-
-Resolver.add_implicit_resolver(
-    'tag:yaml.org,2002:int',
-    # UNITY: Restrict to simple integer values
-    re.compile(r'''^(?:[-+]?(?:0|[1-9][0-9]*))$''', re.X),
-    list('-+0123456789'))
-
 # Resolver.add_implicit_resolver(
 #     'tag:yaml.org,2002:merge',
 #     re.compile(r'^(?:<<)$'),
@@ -60,3 +49,24 @@ Resolver.add_implicit_resolver(
     'tag:yaml.org,2002:yaml',
     re.compile(r'^(?:!|&|\*)$'),
     list('!&*'))
+
+
+class SmartResolver(Resolver):
+    """
+    A Resolver that tries to be smart about how Unity serializes int and float values, but it's known that does not
+    cover all cases.
+    """
+    pass
+
+
+SmartResolver.add_implicit_resolver(
+    'tag:yaml.org,2002:float',
+    re.compile(r'''^(?:[-+]?(?:0|[1-9][0-9]*)\.(?:[1-9]|[0-9][0-9]*[1-9])?)$''', re.X),
+    list('-+0123456789.'))
+
+SmartResolver.add_implicit_resolver(
+    'tag:yaml.org,2002:int',
+    # UNITY: Restrict to simple integer values
+    re.compile(r'''^(?:[-+]?(?:0|[1-9][0-9]*))$''', re.X),
+    list('-+0123456789'))
+
diff --git a/unityparser/utils.py b/unityparser/utils.py
index 034a009..45bfac7 100644
--- a/unityparser/utils.py
+++ b/unityparser/utils.py
@@ -1,8 +1,9 @@
 import io
+from copy import copy
 
 from .dumper import UnityDumper
 from .errors import UnityDocumentError
-from .loader import UnityLoader
+from .loader import UnityLoader, SmartUnityLoader
 from .register import UnityScalarRegister
 
 UNIX_LINE_ENDINGS = '\n'
@@ -10,11 +11,13 @@ UNIX_LINE_ENDINGS = '\n'
 
 class UnityDocument:
 
-    def __init__(self, data, newline=None, file_path=None, register=None):
+    def __init__(self, data, newline=None, file_path=None, register=None, version=None, tags=None):
         self.newline = newline
         self.data = data
         self.file_path = file_path
         self.register = register or UnityScalarRegister()
+        self.version = version
+        self.tags = tags
 
     @property
     def entry(self):
@@ -34,17 +37,35 @@ class UnityDocument:
         """
         file_path = file_path or self.file_path
         assert_or_raise(file_path is not None, UnityDocumentError("file_path parameter must be passed"))
+        kwargs_copy = copy(kwargs)
+        if 'version' not in kwargs:
+            kwargs_copy['version'] = self.version
+        if 'tags' not in kwargs:
+            kwargs_copy['tags'] = self.tags
         with open(file_path, 'w', newline=self.newline) as fp:
-            dump_all(self.data, stream=fp, register=self.register, **kwargs)
+            dump_all(self.data, stream=fp, register=self.register, **kwargs_copy)
 
     @classmethod
-    def load_yaml(cls, file_path):
+    def load_yaml(cls, file_path, try_preserve_types=False):
+        """
+        :param file_path: Path to the file to load
+        :param try_preserve_types: If true, will deserialize what seems to be int and float types to the same Python
+            data types instead of deserializing them all as the string type. When/if this value is later serialized
+            back it might be represented differently in some corner cases.
+        """
+        loader_cls = SmartUnityLoader if try_preserve_types else UnityLoader
         register = UnityScalarRegister()
         with open(file_path, newline='') as fp:
-            data = [d for d in load_all(fp, register)]
+            loader = loader_cls(fp)
+            loader.check_data()
+            fp.seek(0)
+            version = loader.yaml_version
+            tags = loader.non_default_tags
+            data = [d for d in load_all(fp, register, loader_cls)]
             # use document line endings if no mixed lien endings found, else default to linux
             line_endings = UNIX_LINE_ENDINGS if isinstance(fp.newlines, tuple) else fp.newlines
-        doc = UnityDocument(data, newline=line_endings, file_path=file_path, register=register)
+        doc = UnityDocument(data, newline=line_endings, file_path=file_path, register=register, version=version,
+                            tags=tags)
         return doc
 
     # region Filtering
@@ -95,12 +116,12 @@ def assert_or_raise(condition, exception):
         raise exception
 
 
-def load_all(stream, register=None):
+def load_all(stream, register=None, loader_cls=UnityLoader):
     """
     Parse all YAML documents in a stream
     and produce corresponding Python objects.
     """
-    loader = UnityLoader(stream, register)
+    loader = loader_cls(stream, register)
     try:
         while loader.check_data():
             yield loader.get_data()


Test patch:
diff --git a/unityparser/tests/test_scalar_value_types.py b/unityparser/tests/test_scalar_value_types.py
index 1aa0192..66323bc 100644
--- a/unityparser/tests/test_scalar_value_types.py
+++ b/unityparser/tests/test_scalar_value_types.py
@@ -1,14 +1,13 @@
 import py
-import pytest
 
-from unityparser.utils import UnityDocument, UnityDocumentError
+from unityparser.utils import UnityDocument
 
 
-class TestScalarValueTypes:
+class TestSmartScalarValueTypes:
 
     def test_types(self, fixtures):
         base_file_path = py.path.local(fixtures['MultipleTypesDoc.asset'])
-        doc = UnityDocument.load_yaml(str(base_file_path))
+        doc = UnityDocument.load_yaml(str(base_file_path), try_preserve_types=True)
         multi_types = doc.entry
 
         count_map = {'int': 0, 'str': 0, 'float': 0}
@@ -34,7 +33,7 @@ class TestScalarValueTypes:
 
     def test_sum_int_type(self, fixtures):
         base_file_path = py.path.local(fixtures['MultipleTypesDoc.asset'])
-        doc = UnityDocument.load_yaml(str(base_file_path))
+        doc = UnityDocument.load_yaml(str(base_file_path), try_preserve_types=True)
         multi_types = doc.entry
 
         multi_types.scalar_int_001 += 1
diff --git a/unityparser/tests/test_unity_document.py b/unityparser/tests/test_unity_document.py
index d619558..a8b9868 100644
--- a/unityparser/tests/test_unity_document.py
+++ b/unityparser/tests/test_unity_document.py
@@ -30,6 +30,14 @@ class TestUnityDocumentSerialization:
 
         assert base_file_path.read() == dumped_file_path.read()
 
+    def test_preserve_missing_yaml_tags_ok(self, fixtures, tmpdir):
+        base_file_path = py.path.local(fixtures['MetaFileWithoutTags.meta'])
+        doc = UnityDocument.load_yaml(str(base_file_path))
+        dumped_file_path = tmpdir.join('MetaFileWithoutTags.meta')
+        doc.dump_yaml(file_path=str(dumped_file_path))
+
+        assert base_file_path.read() == dumped_file_path.read()
+
 
 class TestUnityDocumentFilters: