Repository: microsoft/ptvsd

Issue:
Return values not reported on Python 3.7
Reflected in `test_return_values` test failures.

This is a 3.7-specific issue, which is why it doesn't show up in our automated runs. It fails consistently in local runs.



Discussion:
I'm going to slap a `skipif` on the test - this needs to be removed once the bug is fixed.

Resolving patch:
diff --git a/src/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_resolver.py b/src/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_resolver.py
--- a/src/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_resolver.py
+++ b/src/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_resolver.py
@@ -7,6 +7,7 @@
 
 from functools import partial
 from _pydevd_bundle.pydevd_constants import dict_iter_items, dict_keys, xrange
+from _pydevd_bundle.pydevd_safe_repr import SafeRepr
 
 # Note: 300 is already a lot to see in the outline (after that the user should really use the shell to get things)
 # and this also means we'll pass less information to the client side (which makes debugging faster).
@@ -88,7 +89,7 @@ class DefaultResolver:
     def resolve(self, var, attribute):
         return getattr(var, attribute)
 
-    def get_contents_debug_adapter_protocol(self, obj):
+    def get_contents_debug_adapter_protocol(self, obj, fmt={}):
         if MethodWrapperType:
             dct, used___dict__ = self._get_py_dictionary(obj)
         else:
@@ -259,13 +260,17 @@ def resolve(self, dict, key):
 
         raise UnableToResolveVariableException()
 
-    def key_to_str(self, key):
+    def key_to_str(self, key, fmt={}):
+        if fmt.get('hex', False):
+            safe_repr = SafeRepr()
+            safe_repr.convert_to_hex = True
+            return safe_repr(key)
         return '%r' % (key,)
 
     def init_dict(self):
         return {}
 
-    def get_contents_debug_adapter_protocol(self, dct):
+    def get_contents_debug_adapter_protocol(self, dct, fmt={}):
         '''
         This method is to be used in the case where the variables are all saved by its id (and as
         such don't need to have the `resolve` method called later on, so, keys don't need to
@@ -280,15 +285,16 @@ def get_contents_debug_adapter_protocol(self, dct):
         i = 0
         for key, val in dict_iter_items(dct):
             i += 1
-            key_as_str = self.key_to_str(key)
-            ret.append((key_as_str, val, '[%s]' % (key_as_str,)))
+            key_as_str = self.key_to_str(key, fmt)
+            eval_key_str = self.key_to_str(key) # do not format the key
+            ret.append((key_as_str, val, '[%s]' % (eval_key_str,)))
             if i > MAX_ITEMS_TO_HANDLE:
                 ret.append((TOO_LARGE_ATTR, TOO_LARGE_MSG))
                 break
 
         ret.append(('__len__', len(dct), partial(_apply_evaluate_name, evaluate_name='len(%s)')))
         # in case the class extends built-in type and has some additional fields
-        from_default_resolver = defaultResolver.get_contents_debug_adapter_protocol(dct)
+        from_default_resolver = defaultResolver.get_contents_debug_adapter_protocol(dct, fmt)
 
         if from_default_resolver:
             ret = from_default_resolver + ret
@@ -336,7 +342,7 @@ def resolve(self, var, attribute):
         except:
             return getattr(var, attribute)
 
-    def get_contents_debug_adapter_protocol(self, lst):
+    def get_contents_debug_adapter_protocol(self, lst, fmt={}):
         '''
         This method is to be used in the case where the variables are all saved by its id (and as
         such don't need to have the `resolve` method called later on, so, keys don't need to
@@ -350,6 +356,8 @@ def get_contents_debug_adapter_protocol(self, lst):
         ret = []
 
         format_str = '%0' + str(int(len(str(l - 1)))) + 'd'
+        if fmt is not None and fmt.get('hex', False):
+            format_str = '0x%0' + str(int(len(hex(l).lstrip('0x')))) + 'x'
 
         for i, item in enumerate(lst):
             ret.append((format_str % i, item, '[%s]' % i))
@@ -365,11 +373,13 @@ def get_contents_debug_adapter_protocol(self, lst):
             ret = from_default_resolver + ret
         return ret
 
-    def get_dictionary(self, var):
+    def get_dictionary(self, var, fmt={}):
         l = len(var)
         d = {}
 
         format_str = '%0' + str(int(len(str(l - 1)))) + 'd'
+        if fmt is not None and fmt.get('hex', False):
+            format_str = '0x%0' + str(int(len(hex(l).lstrip('0x')))) + 'x'
 
         for i, item in enumerate(var):
             d[format_str % i] = item
diff --git a/src/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_suspended_frames.py b/src/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_suspended_frames.py
--- a/src/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_suspended_frames.py
+++ b/src/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_suspended_frames.py
@@ -94,7 +94,7 @@ def get_children_variables(self, fmt=None):
         if resolver is not None:  # i.e.: it's a container.
             if hasattr(resolver, 'get_contents_debug_adapter_protocol'):
                 # The get_contents_debug_adapter_protocol needs to return sorted.
-                lst = resolver.get_contents_debug_adapter_protocol(self.value)
+                lst = resolver.get_contents_debug_adapter_protocol(self.value, fmt=fmt)
             else:
                 # If there's no special implementation, the default is sorting the keys.
                 dct = resolver.get_dictionary(self.value)


Test patch:
diff --git a/src/ptvsd/_vendored/pydevd/tests_python/resources/_debugger_case_local_variables_hex.py b/src/ptvsd/_vendored/pydevd/tests_python/resources/_debugger_case_local_variables_hex.py
new file mode 100644
--- /dev/null
+++ b/src/ptvsd/_vendored/pydevd/tests_python/resources/_debugger_case_local_variables_hex.py
@@ -0,0 +1,12 @@
+def Call():
+    variables_for_test_1 = 100
+    variables_for_test_2 = [1, 10, 100]
+    variables_for_test_3 = {10: 10, 100: 100, 1000: 1000}
+    variables_for_test_4 = {(1, 10, 100): (10000, 100000, 100000)}
+
+    all_vars_set = True  # Break here
+
+
+if __name__ == '__main__':
+    Call()
+    print('TEST SUCEEDED!')
\ No newline at end of file
diff --git a/src/ptvsd/_vendored/pydevd/tests_python/test_debugger_json.py b/src/ptvsd/_vendored/pydevd/tests_python/test_debugger_json.py
--- a/src/ptvsd/_vendored/pydevd/tests_python/test_debugger_json.py
+++ b/src/ptvsd/_vendored/pydevd/tests_python/test_debugger_json.py
@@ -736,6 +736,86 @@ def test_stack_and_variables(case_setup):
         writer.finished_ok = True
 
 
+def test_hex_variables(case_setup):
+    with case_setup.test_file('_debugger_case_local_variables_hex.py') as writer:
+        json_facade = JsonFacade(writer)
+
+        writer.write_set_protocol('http_json')
+        writer.write_add_breakpoint(writer.get_line_index_with_content('Break here'))
+
+        json_facade.write_make_initial_run()
+
+        hit = writer.wait_for_breakpoint_hit()
+
+        # : :type stack_trace_response: StackTraceResponse
+        # : :type stack_trace_response_body: StackTraceResponseBody
+        # : :type stack_frame: StackFrame
+        stack_trace_request = json_facade.write_request(
+            pydevd_schema.StackTraceRequest(pydevd_schema.StackTraceArguments(threadId=hit.thread_id)))
+        stack_trace_response = json_facade.wait_for_response(stack_trace_request)
+        stack_trace_response_body = stack_trace_response.body
+        assert len(stack_trace_response_body.stackFrames) == 2
+        stack_frame = next(iter(stack_trace_response_body.stackFrames))
+        assert stack_frame['name'] == 'Call'
+        assert stack_frame['source']['path'].endswith('_debugger_case_local_variables_hex.py')
+
+        scopes_request = json_facade.write_request(pydevd_schema.ScopesRequest(
+            pydevd_schema.ScopesArguments(stack_frame['id'])))
+
+        scopes_response = json_facade.wait_for_response(scopes_request)
+        scopes = scopes_response.body.scopes
+        assert len(scopes) == 1
+        scope = pydevd_schema.Scope(**next(iter(scopes)))
+        assert scope.name == 'Locals'
+        assert not scope.expensive
+        frame_variables_reference = scope.variablesReference
+        assert isinstance(frame_variables_reference, int)
+
+        fmt = {'hex': True}
+        variables_request = json_facade.write_request(
+            pydevd_schema.VariablesRequest(pydevd_schema.VariablesArguments(frame_variables_reference, format=fmt)))
+        variables_response = json_facade.wait_for_response(variables_request)
+
+        # : :type variables_response: VariablesResponse
+        variable_for_test_1, variable_for_test_2, variable_for_test_3, variable_for_test_4 = sorted(list(
+            v for v in variables_response.body.variables if v['name'].startswith('variable_for_test')
+        ), key=lambda v: v['name'])
+        assert variable_for_test_1 == [{
+            'name': 'variable_for_test_1',
+            'value': "0x64",
+            'type': 'int',
+            'evaluateName': 'variable_for_test_1'
+        }]
+
+        assert isinstance(variable_for_test_2.pop('variablesReference'), int)
+        assert variable_for_test_2 == [{
+            'name': 'variable_for_test_2',
+            'value': "[0x1, 0xa, 0x64]",
+            'type': 'list',
+            'evaluateName': 'variable_for_test_2'
+        }]
+
+        assert isinstance(variable_for_test_3.pop('variablesReference'), int)
+        assert variable_for_test_3 == [{
+            'name': 'variable_for_test_3',
+            'value': '{0xa: 0xa, 0x64: 0x64, 0x3e8: 0x3e8}',
+            'type': 'dict',
+            'evaluateName': 'variable_for_test_3'
+        }]
+
+        assert isinstance(variable_for_test_4.pop('variablesReference'), int)
+        assert variable_for_test_4 == [{
+            'name': 'variable_for_test_4',
+            'value': '{(0x1, 0xa, 0x64): (0x2710, 0x186a0, 0x186a0)}',
+            'type': 'dict',
+            'evaluateName': 'variable_for_test_4'
+        }]
+
+        writer.write_run_thread(hit.thread_id)
+
+        writer.finished_ok = True
+
+
 def test_path_translation_and_source_reference(case_setup):
 
     def get_file_in_client(writer):
diff --git a/src/ptvsd/_vendored/pydevd/tests_python/test_resolvers.py b/src/ptvsd/_vendored/pydevd/tests_python/test_resolvers.py
--- a/src/ptvsd/_vendored/pydevd/tests_python/test_resolvers.py
+++ b/src/ptvsd/_vendored/pydevd/tests_python/test_resolvers.py
@@ -21,6 +21,16 @@ def test_dict_resolver():
         assert contents_debug_adapter_protocol == [
             ("'22'", 22, "['22']"), ('(1, 2)', 2, '[(1, 2)]')]
 
+def test_dict_resolver_hex():
+    from _pydevd_bundle.pydevd_resolver import DictResolver
+    dict_resolver = DictResolver()
+    dct = {(1, 10, 100): (10000, 100000, 100000)}
+    contents_debug_adapter_protocol = dict_resolver.get_contents_debug_adapter_protocol(dct, fmt={'hex': True})
+    len_entry = contents_debug_adapter_protocol.pop(-1)
+    check_len_entry(len_entry, ('__len__', 1))
+    assert contents_debug_adapter_protocol == [
+        ('(0x1, 0xa, 0x64)', (10000, 100000, 100000), '[(1, 10, 100)]'), ]
+
 
 def test_object_resolver_simple():
     from _pydevd_bundle.pydevd_resolver import DefaultResolver
@@ -130,6 +140,7 @@ def __init__(self):
 def test_tuple_resolver():
     from _pydevd_bundle.pydevd_resolver import TupleResolver
     tuple_resolver = TupleResolver()
+    fmt={'hex': True}
     lst = tuple(range(11))
     contents_debug_adapter_protocol = tuple_resolver.get_contents_debug_adapter_protocol(lst)
     len_entry = contents_debug_adapter_protocol.pop(-1)
@@ -163,6 +174,51 @@ def test_tuple_resolver():
         '__len__': 11
     }
 
+    lst = tuple(range(17))
+    contents_debug_adapter_protocol = tuple_resolver.get_contents_debug_adapter_protocol(lst, fmt=fmt)
+    len_entry = contents_debug_adapter_protocol.pop(-1)
+    assert contents_debug_adapter_protocol == [
+        ('0x00', 0, '[0]'),
+        ('0x01', 1, '[1]'),
+        ('0x02', 2, '[2]'),
+        ('0x03', 3, '[3]'),
+        ('0x04', 4, '[4]'),
+        ('0x05', 5, '[5]'),
+        ('0x06', 6, '[6]'),
+        ('0x07', 7, '[7]'),
+        ('0x08', 8, '[8]'),
+        ('0x09', 9, '[9]'),
+        ('0x0a', 10, '[10]'),
+        ('0x0b', 11, '[11]'),
+        ('0x0c', 12, '[12]'),
+        ('0x0d', 13, '[13]'),
+        ('0x0e', 14, '[14]'),
+        ('0x0f', 15, '[15]'),
+        ('0x10', 16, '[16]'),
+    ]
+    check_len_entry(len_entry, ('__len__', 17))
+
+    assert tuple_resolver.get_dictionary(lst, fmt=fmt) == {
+        '0x00': 0,
+        '0x01': 1,
+        '0x02': 2,
+        '0x03': 3,
+        '0x04': 4,
+        '0x05': 5,
+        '0x06': 6,
+        '0x07': 7,
+        '0x08': 8,
+        '0x09': 9,
+        '0x0a': 10,
+        '0x0b': 11,
+        '0x0c': 12,
+        '0x0d': 13,
+        '0x0e': 14,
+        '0x0f': 15,
+        '0x10': 16,
+        '__len__': 17
+    }
+
     lst = tuple(range(10))
     contents_debug_adapter_protocol = tuple_resolver.get_contents_debug_adapter_protocol(lst)
     len_entry = contents_debug_adapter_protocol.pop(-1)
@@ -194,3 +250,32 @@ def test_tuple_resolver():
         '__len__': 10
     }
 
+    contents_debug_adapter_protocol = tuple_resolver.get_contents_debug_adapter_protocol(lst, fmt=fmt)
+    len_entry = contents_debug_adapter_protocol.pop(-1)
+    assert contents_debug_adapter_protocol == [
+        ('0x0', 0, '[0]'),
+        ('0x1', 1, '[1]'),
+        ('0x2', 2, '[2]'),
+        ('0x3', 3, '[3]'),
+        ('0x4', 4, '[4]'),
+        ('0x5', 5, '[5]'),
+        ('0x6', 6, '[6]'),
+        ('0x7', 7, '[7]'),
+        ('0x8', 8, '[8]'),
+        ('0x9', 9, '[9]'),
+    ]
+    check_len_entry(len_entry, ('__len__', 10))
+
+    assert tuple_resolver.get_dictionary(lst, fmt=fmt) == {
+        '0x0': 0,
+        '0x1': 1,
+        '0x2': 2,
+        '0x3': 3,
+        '0x4': 4,
+        '0x5': 5,
+        '0x6': 6,
+        '0x7': 7,
+        '0x8': 8,
+        '0x9': 9,
+        '__len__': 10
+    }
\ No newline at end of file
diff --git a/tests/func/test_evaluate.py b/tests/func/test_evaluate.py
--- a/tests/func/test_evaluate.py
+++ b/tests/func/test_evaluate.py
@@ -4,7 +4,6 @@
 
 from __future__ import print_function, with_statement, absolute_import
 
-import pytest
 import sys
 
 from tests.helpers import get_marked_line_numbers, print
@@ -244,7 +243,6 @@ def code_to_debug():
         session.wait_for_exit()
 
 
-@pytest.mark.skipif(sys.version_info >= (3, 7), reason='https://github.com/Microsoft/ptvsd/issues/1207')
 def test_return_values(pyfile, run_as, start_method):
 
     @pyfile
@@ -371,3 +369,120 @@ def code_to_debug():
 
         session.send_request('continue').wait_for_response(freeze=False)
         session.wait_for_exit()
+
+
+def test_hex_numbers(pyfile, run_as, start_method):
+    @pyfile
+    def code_to_debug():
+        from dbgimporter import import_and_enable_debugger
+        import_and_enable_debugger()
+        a = 100
+        b = [1, 10, 100]
+        c = {10: 10, 100: 100, 1000: 1000}
+        d = {(1, 10, 100): (10000, 100000, 100000)}
+        print((a, b, c, d)) #@bp
+
+    line_numbers = get_marked_line_numbers(code_to_debug)
+    print(line_numbers)
+
+    with DebugSession() as session:
+        session.initialize(
+            target=(run_as, code_to_debug),
+            start_method=start_method,
+            ignore_unobserved=[Event('continued')],
+        )
+        session.set_breakpoints(code_to_debug, [line_numbers['bp']])
+        session.start_debugging()
+        hit = session.wait_for_thread_stopped()
+
+        resp_scopes = session.send_request('scopes', arguments={
+            'frameId': hit.frame_id
+        }).wait_for_response()
+        scopes = resp_scopes.body['scopes']
+        assert len(scopes) > 0
+
+        resp_variables = session.send_request('variables', arguments={
+            'variablesReference': scopes[0]['variablesReference'],
+            'format': {'hex': True}
+        }).wait_for_response()
+        variables = list(v for v in resp_variables.body['variables']
+                         if v['name'] in ('a', 'b', 'c', 'd'))
+        a, b, c, d = sorted(variables, key=lambda v: v['name'])
+        assert a == ANY.dict_with({
+            'name': 'a',
+            'value': "0x64",
+            'type': 'int',
+            'evaluateName': 'a'
+        })
+
+        assert b == ANY.dict_with({
+            'name': 'b',
+            'value': "[0x1, 0xa, 0x64]",
+            'type': 'list',
+            'evaluateName': 'b',
+            'variablesReference': ANY.int,
+        })
+
+        resp_variables = session.send_request('variables', arguments={
+            'variablesReference': b['variablesReference'],
+            'format': {'hex': True}
+        }).wait_for_response()
+        b_children = resp_variables.body['variables']
+        assert b_children == [
+            {'name': '0x0', 'value': '0x1', 'type': 'int', 'evaluateName': 'b[0]'},
+            {'name': '0x1', 'value': '0xa', 'type': 'int', 'evaluateName': 'b[1]'},
+            {'name': '0x2', 'value': '0x64', 'type': 'int', 'evaluateName': 'b[2]'},
+            {'name': '__len__', 'value': '0x3', 'type': 'int', 'evaluateName': 'len(b)'},
+        ]
+
+        assert c == ANY.dict_with({
+            'name': 'c',
+            'value': '{0xa: 0xa, 0x64: 0x64, 0x3e8: 0x3e8}',
+            'type': 'dict',
+            'evaluateName': 'c',
+            'variablesReference': ANY.int,
+        })
+
+        resp_variables = session.send_request('variables', arguments={
+            'variablesReference': c['variablesReference'],
+            'format': {'hex': True}
+        }).wait_for_response()
+        c_children = resp_variables.body['variables']
+        assert c_children == [
+            {'name': '0x3e8', 'value': '0x3e8', 'type': 'int', 'evaluateName': 'c[1000]'},
+            {'name': '0x64', 'value': '0x64', 'type': 'int', 'evaluateName': 'c[100]'},
+            {'name': '0xa', 'value': '0xa', 'type': 'int', 'evaluateName': 'c[10]'},
+            {'name': '__len__', 'value': '0x3', 'type': 'int', 'evaluateName': 'len(c)'}
+        ]
+
+        assert d == ANY.dict_with({
+            'name': 'd',
+            'value': '{(0x1, 0xa, 0x64): (0x2710, 0x186a0, 0x186a0)}',
+            'type': 'dict',
+            'evaluateName': 'd',
+            'variablesReference': ANY.int,
+        })
+        resp_variables = session.send_request('variables', arguments={
+            'variablesReference': d['variablesReference'],
+            'format': {'hex': True}
+        }).wait_for_response()
+        d_children = resp_variables.body['variables']
+        assert d_children == [
+            {'name': '(0x1, 0xa, 0x64)', 'value': '(0x2710, 0x186a0, 0x186a0)', 'type': 'tuple', 'evaluateName': 'd[(1, 10, 100)]', 'variablesReference': ANY.int},
+            {'name': '__len__', 'value': '0x1', 'type': 'int', 'evaluateName': 'len(d)'}
+        ]
+
+        resp_variables = session.send_request('variables', arguments={
+            'variablesReference': d_children[0]['variablesReference'],
+            'format': {'hex': True}
+        }).wait_for_response()
+        d_child_of_child = resp_variables.body['variables']
+        assert d_child_of_child == [
+            {'name': '0x0', 'value': '0x2710', 'type': 'int', 'evaluateName': 'd[(1, 10, 100)][0]'},
+            {'name': '0x1', 'value': '0x186a0', 'type': 'int', 'evaluateName': 'd[(1, 10, 100)][1]'},
+            {'name': '0x2', 'value': '0x186a0', 'type': 'int', 'evaluateName': 'd[(1, 10, 100)][2]'},
+            {'name': '__len__', 'value': '0x3', 'type': 'int', 'evaluateName': 'len(d[(1, 10, 100)])'}
+        ]
+
+        session.send_request('continue').wait_for_response(freeze=False)
+        session.wait_for_exit()