Repository: Zapix/mtpylon (language: python)

Issue:
Dump/Load objects/functions from schema
- [ ] Load object by schema
- [ ] Load optional fields
- [ ] Load list types
- [ ] Raise error if object hasn't been found
- [ ] Raise error if wrong arguments passed by schema
- [x] Dump object by schema 
- [x] Raise error if object has wrong attributes to dump
- [ ] Dump optional fields 
- [ ] Dump list types


Discussion / hints:

Fix patch:
diff --git a/mtpylon/exceptions.py b/mtpylon/exceptions.py
index 93071aa..79ffb67 100644
--- a/mtpylon/exceptions.py
+++ b/mtpylon/exceptions.py
@@ -26,3 +26,10 @@ class SchemaChangeError(Exception):
     Raises when user tries add or delete combinator, function after schema
     initialization
     """
+
+
+class DumpError(Exception):
+    """
+    Raises error when can't dump object or function call  by schema
+    """
+    pass
diff --git a/mtpylon/serialization/__init__.py b/mtpylon/serialization/__init__.py
index 40a96af..431fe59 100644
--- a/mtpylon/serialization/__init__.py
+++ b/mtpylon/serialization/__init__.py
@@ -1,1 +1,4 @@
 # -*- coding: utf-8 -*-
+from .schema import dump, load
+
+__all__ = ['dump', 'load']
diff --git a/mtpylon/serialization/schema.py b/mtpylon/serialization/schema.py
new file mode 100644
index 0000000..e70ef64
--- /dev/null
+++ b/mtpylon/serialization/schema.py
@@ -0,0 +1,262 @@
+# -*- coding: utf-8 -*-
+from dataclasses import dataclass
+from typing import (
+    cast,
+    overload,
+    Any,
+    Callable,
+    Union,
+    Dict,
+    Optional,
+    List,
+    Type,
+    NamedTuple
+)
+from functools import partial
+from inspect import isfunction
+
+from .bytes import dump as dump_bytes, load as load_bytes
+from .int import dump as dump_int, load as load_int
+from .long import dump as dump_long, load as load_long
+from .int128 import dump as dump_int128, load as load_int128
+from .int256 import dump as dump_int256, load as load_int256
+from .double import dump as dump_double, load as load_double
+from .string import dump as dump_string, load as load_string
+from .vector import dump as dump_vector, load as load_vector
+from .loaded import LoadedValue
+from ..utils import (
+    long,
+    int128,
+    int256,
+    is_list_type,
+    is_optional_type,
+    AttrDescription,
+)
+from ..schema import Schema, FunctionData, CombinatorData
+from ..exceptions import DumpError
+
+
+DumpFunction = Callable[[Any], bytes]
+
+
+@dataclass
+class CallableFunc:
+    func: Callable
+    params: Dict[str, Any]
+
+
+class Value2Dump(NamedTuple):
+    value: Any
+    param: AttrDescription
+
+
+def get_flag_ids(origin: Type, param_name: str) -> int:
+    """
+    Get flag index for combinator
+
+    Args:
+        origin - type of origin comibnator
+        param_name - name of param
+    """
+    flags = origin.Meta.flags
+    return flags[param_name]
+
+
+def get_flag_value(flag_number: int, param_name: str, origin: Type) -> int:
+    """
+    Get flag value by param name
+    """
+    return (flag_number >> get_flag_ids(origin, param_name)) & 1
+
+
+def set_flag_value(flag_number: int, param_name: str, origin: Type) -> int:
+    """
+    Updates flag value and returns it
+    """
+    return flag_number | (1 << get_flag_ids(origin, param_name))
+
+
+@overload
+def dump(schema: Schema, value: Any) -> bytes:
+    ...
+
+
+@overload
+def dump(schema: Schema, value: Callable, **kwargs: Any) -> bytes:
+    ...
+
+
+def dump(schema, value, **kwargs):
+    """
+    Dumps basic or boxed types by schema
+    """
+    def dump_by_type(
+            dump_value: Any,
+            origin: Type,
+            type_name: str,
+    ) -> bytes:
+        dump_map: Dict[Any, DumpFunction] = {
+            int: dump_int,
+            long: dump_long,
+            int128: dump_int128,
+            int256: dump_int256,
+            float: dump_double,
+            bytes: dump_bytes,
+            str: dump_string
+        }
+
+        try:
+            if is_list_type(origin):
+                item_origin = origin.__args__[0]
+                dump_item: DumpFunction = dump_map.get(
+                    item_origin,
+                    partial(dump, schema)
+                )
+                return dump_vector(dump_item, dump_value)
+            else:
+                dump_func: DumpFunction = dump_map.get(
+                    origin,
+                    partial(dump, schema)
+                )
+                return dump_func(dump_value)
+        except Exception:
+            raise DumpError(f'Can`t dump {value} as {type_name}')
+
+    data: Optional[Union[CombinatorData, FunctionData]] = None
+    values_2_dump: List[Value2Dump] = []
+
+    dumped_values: List[bytes] = []
+    flags: Dict[str, int] = {}
+    flags_value: Optional[int] = None
+
+    if isfunction(value):
+        try:
+            data = schema[value]
+        except KeyError:
+            raise DumpError('Can`t dump function that not in schema')
+
+        values_2_dump = [
+            Value2Dump(kwargs.get(param.name), param)
+            for param in data.params
+        ]
+    else:
+        try:
+            data = schema[type(value)]
+        except KeyError:
+            raise DumpError('Can`t dump combinator that not in schema')
+
+        flags = getattr(data.origin.Meta, 'flags', {})
+
+        values_2_dump = [
+            Value2Dump(getattr(value, param.name, None), param)
+            for param in data.params
+        ]
+
+    for item in values_2_dump:
+        attr_value = item.value
+        param = item.param
+
+        if param.name == 'flags':
+            flags_value = 0
+            continue
+
+        if param.name in flags:
+            if attr_value is None:
+                continue
+            origin = param.origin.__args__[0]
+            type_name = param.type
+            flags_value = set_flag_value(flags_value, param.name, data.origin)
+        else:
+            origin = param.origin
+            type_name = param.type
+
+        dumped_values.append(dump_by_type(attr_value, origin, type_name))
+
+    dumped_flag = dump_int(flags_value) if flags_value is not None else b''
+
+    dumped = b''.join(
+        [
+            dump_int(data.id),
+            dumped_flag
+        ] +
+        dumped_values
+    )
+    return dumped
+
+
+def load(
+        schema: Schema,
+        input: bytes
+) -> LoadedValue[Union[CallableFunc, Any]]:
+    """
+    Loads object or function with params from bytes input
+
+    Args:
+        schema - schema that will be used to load
+        input - serialized value
+
+    Raises:
+        ValueError - when can't load data
+    """
+    load_map = {
+        bytes: load_bytes,
+        int: load_int,
+        long: load_long,
+        int128: load_int128,
+        int256: load_int256,
+        float: load_double,
+        str: load_string,
+    }
+
+    def get_load_func(x) -> Callable[[bytes], LoadedValue[Any]]:
+        return load_map.get(x, partial(load, schema))
+
+    def load_empty(x: bytes) -> LoadedValue[None]:
+        return LoadedValue(None, 0)
+
+    offset = 0
+    loaded_combinator = load_int(input)
+    offset += loaded_combinator.offset
+    combinator = loaded_combinator.value
+
+    if combinator not in schema:
+        raise ValueError(f'Can`t find combinator {combinator} in schema')
+
+    data = schema[combinator]
+    print(data)
+
+    flag_number: Optional[int] = None
+    params: Dict[str, Any] = {}
+
+    for param in data.params:
+        if param.name == 'flags':
+            loaded = load_int(input[offset:])
+            flag_number = loaded.value
+            offset += loaded.offset
+            continue
+
+        if is_optional_type(param.origin) and flag_number is not None:
+            if get_flag_value(
+                    flag_number,
+                    param.name,
+                    cast(Type, data.origin),
+            ):
+                load_func = get_load_func(param.origin.__args__[0])
+            else:
+                load_func = load_empty
+        elif is_list_type(param.origin):
+            load_item_func = get_load_func(param.origin.__args__[0])
+            load_func = partial(load_vector, load_item_func)
+        else:
+            load_func = get_load_func(param.origin)
+
+        loaded = load_func(input[offset:])
+        params[param.name] = loaded.value
+        offset += loaded.offset
+
+    if isfunction(data.origin):
+        value = CallableFunc(data.origin, params)
+    else:
+        value = data.origin(**params)
+
+    return LoadedValue(value=value, offset=offset)


Test patch:
diff --git a/tests/simpleschema.py b/tests/simpleschema.py
index 9f8b107..1e8e7d1 100644
--- a/tests/simpleschema.py
+++ b/tests/simpleschema.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 from dataclasses import dataclass
-from typing import Annotated, Union, List
+from typing import Annotated, Union, List, Optional
 
 from mtpylon import Schema
 
@@ -28,10 +28,14 @@ class AuthorizedUser:
     id: int
     username: str
     password: str
+    avatar_url: Optional[str]
 
     class Meta:
         name = 'authorizedUser'
-        order = ('id', 'username', 'password')
+        order = ('id', 'username', 'password', 'avatar_url')
+        flags = {
+            'avatar_url': 0
+        }
 
 
 @dataclass
@@ -68,7 +72,8 @@ async def register(username: str, password: str) -> User:
     return AuthorizedUser(
         id=1,
         username=username,
-        password=password
+        password=password,
+        avatar_url=None
     )
 
 
@@ -77,7 +82,8 @@ async def login(username: str, password: str) -> User:
         return AuthorizedUser(
             id=1,
             username=username,
-            password=password
+            password=password,
+            avatar_url=None
         )
 
     return AnonymousUser()
diff --git a/tests/test_schema.py b/tests/test_schema.py
index 38e0cb4..1f1afde 100644
--- a/tests/test_schema.py
+++ b/tests/test_schema.py
@@ -117,7 +117,7 @@ def test_function_not_in_schema():
 
 
 def test_combinator_number_in_schema():
-    assert 1503360568 in schema
+    assert 0x5d8fe2e9 in schema
 
 
 def test_function_number_in_schema():
diff --git a/tests/test_serialization/test_schema.py b/tests/test_serialization/test_schema.py
new file mode 100644
index 0000000..e6f6895
--- /dev/null
+++ b/tests/test_serialization/test_schema.py
@@ -0,0 +1,276 @@
+# -*- coding: utf-8 -*-
+from typing import no_type_check
+
+import pytest
+
+from mtpylon.serialization import dump, load
+from mtpylon.serialization.schema import CallableFunc
+from mtpylon.exceptions import DumpError
+
+from ..simpleschema import (
+    schema,
+    BoolTrue,
+    BoolFalse,
+    Task,
+    TaskList,
+    AuthorizedUser,
+    get_task_list,
+    login
+)
+
+
+class WrongObject:
+    pass
+
+
+def wrong_function():
+    pass
+
+
+def test_dump_bool_true():
+    assert dump(schema, BoolTrue()) == b'\xb5\x75\x72\x99'
+
+
+def test_dump_bool_false():
+    assert dump(schema, BoolFalse()) == b'\x37\x97\x79\xbc'
+
+
+def test_dump_task_object():
+    task = Task(
+        id=12,
+        content='dump by schema',
+        completed=BoolTrue()
+    )
+
+    assert dump(schema, task) == (
+        b'\xc2\x87b\x00' +  # task combinator number
+        b'\x0c\x00\x00\x00' +  # id number - int
+        b'\x0edump by schema\x00' +  # content - str
+        b'\xb5\x75\x72\x99'  # completed - Bool
+    )
+
+
+def test_dump_authroized_user_with_optional_field():
+    authorized_user = AuthorizedUser(
+        id=12,
+        username='zapix',
+        password='123123',
+        avatar_url='http://example.com/1.jpg'
+    )
+    assert dump(schema, authorized_user) == (
+        b'\xe9\xe2\x8f\x5d' +  # combinator id
+        b'\x01\x00\x00\x00' +  # flags
+        b'\x0c\x00\x00\x00' +  # user id
+        b'\x05zapix\x00\x00' +  # username
+        b'\x06123123\x00' +  # password
+        b'\x18http://example.com/1.jpg\x00\x00\x00'
+    )
+
+
+def test_dump_authorized_user_without_optional_field():
+    authorized_user = AuthorizedUser(
+        id=12,
+        username='zapix',
+        password='123123',
+        avatar_url=None,
+    )
+    assert dump(schema, authorized_user) == (
+        b'\xe9\xe2\x8f\x5d' +  # combinator id
+        b'\x00\x00\x00\x00' +  # flags
+        b'\x0c\x00\x00\x00' +  # user id
+        b'\x05zapix\x00\x00' +  # username
+        b'\x06123123\x00'  # password
+    )
+
+
+def test_dump_task_list():
+    task_list = TaskList(
+        tasks=[
+            Task(
+                id=12,
+                content='dump by schema',
+                completed=BoolTrue()
+            ),
+            Task(
+                id=16,
+                content='dump list',
+                completed=BoolFalse()
+            )
+        ]
+    )
+    assert dump(schema, task_list) == (
+        b'\x46\x00\x98\x66' +  # combinator id
+        b'\x15\xc4\xb5\x1c' +  # vector id
+        b'\x02\x00\x00\x00' +  # vector size
+        b'\xc2\x87b\x00' +  # task combinator number
+        b'\x0c\x00\x00\x00' +  # id number - int
+        b'\x0edump by schema\x00' +  # content - str
+        b'\xb5\x75\x72\x99'  # completed - Bool
+        b'\xc2\x87b\x00' +  # task combinator number
+        b'\x10\x00\x00\x00' +  # id number - int
+        b'\x09dump list\x00\x00' +  # content - str
+        b'\x37\x97\x79\xbc'  # completed - Bool
+    )
+
+
+def test_dump_get_task_list_function():
+    assert dump(schema, get_task_list) == b'S)"\xca'
+
+
+def test_dump_login_function():
+    assert dump(schema, login, username='root', password='root') == (
+        b'C~]\x1c' +  # function login
+        b'\x04root\x00\x00\x00' +  # username
+        b'\x04root\x00\x00\x00'  # password
+    )
+
+
+def test_dump_wrong_object():
+    obj = WrongObject()
+    with pytest.raises(DumpError):
+        dump(schema, obj)
+
+
+def test_dump_wrong_function():
+    with pytest.raises(DumpError):
+        dump(schema, wrong_function)
+
+
+def test_wrong_function_params():
+    with pytest.raises(DumpError):
+        dump(schema, login, username='root', password=123)
+
+
+@no_type_check
+def test_dump_wrong_object_params():
+    task = Task(id=12, content=12312, completed=BoolFalse())
+    with pytest.raises(DumpError):
+        dump(schema, task)
+
+
+def test_load_bool_true():
+    input = b'\xb5\x75\x72\x99'
+    loaded = load(schema, input)
+    assert loaded.value == BoolTrue()
+    assert loaded.offset == 4
+
+
+def test_load_bool_false():
+    input = b'\x37\x97\x79\xbc'
+    loaded = load(schema, input)
+    assert loaded.value == BoolFalse()
+    assert loaded.offset == 4
+
+
+def test_load_task():
+    input = (
+        b'\xc2\x87b\x00' +  # task combinator number
+        b'\x0c\x00\x00\x00' +  # id number - int
+        b'\x0edump by schema\x00' +  # content - str
+        b'\xb5\x75\x72\x99'  # completed - Bool
+    )
+
+    loaded = load(schema, input)
+    assert isinstance(loaded.value, Task)
+    assert loaded.value.id == 12
+    assert loaded.value.content == 'dump by schema'
+    assert loaded.value.completed == BoolTrue()
+    assert loaded.offset == 28
+
+
+def test_load_authorized_user_with_avatar_url():
+    input = (
+        b'\xe9\xe2\x8f\x5d' +  # combinator id
+        b'\x01\x00\x00\x00' +  # flags
+        b'\x0c\x00\x00\x00' +  # user id
+        b'\x05zapix\x00\x00' +  # username
+        b'\x06123123\x00' +  # password
+        b'\x18http://example.com/1.jpg\x00\x00\x00'
+    )
+
+    loaded = load(schema, input)
+
+    assert isinstance(loaded.value, AuthorizedUser)
+    assert loaded.value.id == 12
+    assert loaded.value.username == 'zapix'
+    assert loaded.value.password == '123123'
+    assert loaded.value.avatar_url == 'http://example.com/1.jpg'
+    assert loaded.offset == 56
+
+
+def test_load_authorized_user_without_avatar_url():
+    input = (
+        b'\xe9\xe2\x8f\x5d' +  # combinator id
+        b'\x00\x00\x00\x00' +  # flags
+        b'\x0c\x00\x00\x00' +  # user id
+        b'\x05zapix\x00\x00' +  # username
+        b'\x06123123\x00'  # password
+    )
+
+    loaded = load(schema, input)
+
+    assert isinstance(loaded.value, AuthorizedUser)
+    assert loaded.value.id == 12
+    assert loaded.value.username == 'zapix'
+    assert loaded.value.password == '123123'
+    assert loaded.offset == 28
+
+
+def test_load_task_list():
+    input = (
+        b'\x46\x00\x98\x66' +  # combinator id
+        b'\x15\xc4\xb5\x1c' +  # vector id
+        b'\x02\x00\x00\x00' +  # vector size
+        b'\xc2\x87b\x00' +  # task combinator number
+        b'\x0c\x00\x00\x00' +  # id number - int
+        b'\x0edump by schema\x00' +  # content - str
+        b'\xb5\x75\x72\x99'  # completed - Bool
+        b'\xc2\x87b\x00' +  # task combinator number
+        b'\x10\x00\x00\x00' +  # id number - int
+        b'\x09dump list\x00\x00' +  # content - str
+        b'\x37\x97\x79\xbc'  # completed - Bool
+    )
+
+    loaded = load(schema, input)
+
+    assert isinstance(loaded.value, TaskList)
+    assert len(loaded.value.tasks) == 2
+    assert loaded.offset == 64
+
+
+def test_load_get_task_list():
+    input = (
+        b'S)"\xca'
+    )
+
+    loaded = load(schema, input)
+
+    assert isinstance(loaded.value, CallableFunc)
+    assert loaded.value.func == get_task_list
+    assert loaded.value.params == {}
+    assert loaded.offset == 4
+
+
+def test_load_login_func():
+    input = (
+        b'C~]\x1c' +  # function login
+        b'\x04root\x00\x00\x00' +  # username
+        b'\x04root\x00\x00\x00'  # password
+    )
+
+    loaded = load(schema, input)
+
+    assert isinstance(loaded.value, CallableFunc)
+    assert loaded.value.func == login
+    assert loaded.value.params['username'] == 'root'
+    assert loaded.value.params['password'] == 'root'
+    assert loaded.offset == 20
+
+
+def test_no_value():
+    input = (
+        b'\x01\x00\x00\x00'
+    )
+
+    with pytest.raises(ValueError):
+        load(schema, input)
diff --git a/tests/test_serializers.py b/tests/test_serializers.py
index 692685d..2644d2b 100644
--- a/tests/test_serializers.py
+++ b/tests/test_serializers.py
@@ -56,10 +56,14 @@
       "params": []
     },
     {
-      "id": 1503360568,
+      "id": 0x5d8fe2e9,
       "predicate": "authorizedUser",
       "type": "User",
       "params": [
+        {
+            "name": "flags",
+            "type": "#"
+        },
         {
           "name": "id",
           "type": "int"
@@ -71,6 +75,10 @@
         {
           "name": "password",
           "type": "string"
+        },
+        {
+            'name': 'avatar_url',
+            'type': 'flags.0?string'
         }
       ]
     },
@@ -166,7 +174,7 @@
 tl_combinators = [
     'boolTrue#997275b5 = Bool;',
     'boolFalse#bc799737 = Bool;',
-    'authorizedUser#599b7638 id:int username:string password:string = User;',
+    'authorizedUser#5d8fe2e9 flags:# id:int username:string password:string avatar_url:flags.0?string = User;',  # noqa: E501
     'anonymousUser#5d328bc3 = User;',
     'task#006287c2 id:int content:string completed:Bool = Task;',
     'taskList#66980046 tasks:Vector<Task> = TaskList;',