Repository: joke2k/faker
Pull request #1247 (2020-08-19 07:17:49)

JSON and Fixed Width argument group and parser support.
### What does this changes

The new Argument Groups is a much cleaner way to handle parsing arguments to provider methods when building data structures like DSV, JSON and Fixed Width.  It also allows for the JSON provider to now accept a Dictionary that utilizes the Argument Groups, which is a much simpler way to build complex data structures.

```Python
fake.set_arguments('top_half', {'min_value': 50,'max_value': 100})

fake.json(data_columns={'Name': 'name', 'Score': 'pyint:top_half'})
fake.fixed_width(data_columns=[(20, 'name'), (3, 'pyint:top_half')])
```

If string tokens are used then complex Strings can be created by using pystr_format under the hood.

```Python
fake.set_arguments('red_rgb', {'hue': 'red', 'color_format': 'rgb'})
fake.json(data_columns={'Address': 'address', 'Feature': '### - {{ color:red_rgb }}'})
```

All previous support for lists with tuple records is still in tacked and will not be broken.  More tests have been added to ensure absolutely safety for anyone that begins to use JSON with list format or dict format, and Fixed Width generation.


Patch:
diff --git a/faker/generator.py b/faker/generator.py
index 46f176d1..2c7be5e7 100644
--- a/faker/generator.py
+++ b/faker/generator.py
@@ -99,7 +99,9 @@ class Generator:
     def set_arguments(self, group, argument, value=None):
         """
         Creates an argument group, with an individual argument or a dictionary
-        of arguments.  Used with the Generator.parse method.
+        of arguments. The argument groups is used to apply arguments to tokens,
+        when using the generator.parse() method. To further manage argument
+        groups, use get_arguments() and del_arguments() methods.
 
         generator.set_arguments('small', 'max_value', 10)
         generator.set_arguments('small', {'min_value': 5, 'max_value': 10})
@@ -117,7 +119,8 @@ class Generator:
     def get_arguments(self, group, argument=None):
         """
         Get the value of an argument configured within a argument group, or
-        the entire group as a dictionary.
+        the entire group as a dictionary. Used in conjunction with the
+        set_arguments() method.
 
         generator.get_arguments('small', 'max_value')
         generator.get_arguments('small')
@@ -131,8 +134,8 @@ class Generator:
 
     def del_arguments(self, group, argument=None):
         """
-        Delete an argument from an argument group or the entire
-        argument group.
+        Delete an argument from an argument group or the entire argument group.
+        Used in conjunction with the set_arguments() method.
 
         generator.del_arguments('small')
         generator.del_arguments('small', 'max_value')
@@ -149,9 +152,10 @@ class Generator:
 
     def parse(self, text):
         """
-        Replaces tokens (like '{{ tokenName }}' or '{{tokenName}}')
-        with the result from the token method call. Arguments can be
-        parsed by using an argument group. '{{ tokenName:group }}'
+        Replaces tokens like '{{ tokenName }}' or '{{tokenName}}' in a string with
+        the result from the token method call. Arguments can be parsed by using an
+        argument group. For more information on the use of argument groups, please
+        refer to the set_arguments() method.
 
         Example:
 
diff --git a/faker/providers/address/es_ES/__init__.py b/faker/providers/address/es_ES/__init__.py
index 1fd46d26..4438dbed 100644
--- a/faker/providers/address/es_ES/__init__.py
+++ b/faker/providers/address/es_ES/__init__.py
@@ -63,6 +63,30 @@ class Provider(AddressProvider):
         'Zamora',
         'Zaragoza')
 
+    # Source:
+    # https://administracionelectronica.gob.es/ctt/resources/Soluciones
+    # /238/Descargas/Catalogo-de-Comunidades-Autonomas.xlsx
+    regions = (
+        'Andalucía',
+        'Aragón',
+        'Principado de Asturias',
+        'Illes Balears',
+        'Canarias',
+        'Cantabria',
+        'Castilla y León',
+        'Castilla-La Mancha',
+        'Cataluña',
+        'Comunitat Valenciana',
+        'Extremadura',
+        'Galicia',
+        'Comunidad de Madrid',
+        'Región de Murcia',
+        'Comunidad Foral de Navarra',
+        'País Vasco',
+        'La Rioja',
+        'Ciudad Autónoma de Ceuta',
+        'Ciudad Autónoma de Melilla')
+
     city_formats = (
         '{{state_name}}',
     )
@@ -94,3 +118,8 @@ class Provider(AddressProvider):
 
     def state(self):
         return self.random_element(self.states)
+
+    def region(self):
+        return self.random_element(self.regions)
+
+    autonomous_community = region
diff --git a/faker/providers/misc/__init__.py b/faker/providers/misc/__init__.py
index fe6e24bd..cb129ce5 100644
--- a/faker/providers/misc/__init__.py
+++ b/faker/providers/misc/__init__.py
@@ -7,8 +7,6 @@ import tarfile
 import uuid
 import zipfile
 
-from collections import OrderedDict
-
 from .. import BaseProvider
 
 localized = True
@@ -298,7 +296,12 @@ class Provider(BaseProvider):
         The ``header`` argument expects a list or a tuple of strings that will serve as the header row
         if supplied. The ``data_columns`` argument expects a list or a tuple of string tokens, and these
         string tokens will be passed to  :meth:`pystr_format() <faker.providers.python.Provider.pystr_format>`
-        for data generation. Both ``header`` and ``data_columns`` myst be of the same length.
+        for data generation. Argument Groups are used to pass arguments to the provider methods.
+        Both ``header`` and ``data_columns`` must be of the same length.
+
+        Example:
+            fake.set_arguments('top_half', {'min_value': 50, 'max_value': 100})
+            fake.dsv(data_columns=('{{ name }}', '{{ pyint:top_half }}'))
 
         The ``num_rows`` argument controls how many rows of data to generate, and the ``include_row_ids``
         argument may be set to ``True`` to include a sequential row ID column.
@@ -385,24 +388,37 @@ class Provider(BaseProvider):
         )
 
     def json(self,
-             data_columns: list = [('name', 'name'), ('residency', 'address')],
+             data_columns: list = None,
              num_rows: int = 10,
              indent: int = None) -> str:
         """
-        Generate random JSON structured key/values
+        Generate random JSON structure values.
 
-        Using a list of records that is passed as ``data_columns``, you define the structure that
-        will be generated. Parameters are provider specific, and should be a dictionary that will
-        be passed to the provider method.
+        Using a dictionary or list of records that is passed as ``data_columns``,
+        define the structure that is used to build JSON structures.  For complex
+        data structures it is recommended to use the dictionary format.
 
-        Data Columns format
-            [('field_name', 'provider_name', {'parameters'})]
+        Data Column Dictionary format:
+            {'key name': 'definition'}}
 
-        The provider_name can also be a list of records, to create a list within the JSON data.
-        For value only entries within the list, set the 'field_name' to None.
+        The definition can simply be the 'name:argument_group' of a provider
+        method, or can also be string {{ tokens }} that are passed to python
+        provider pystr_format() method for complex string generation.
+        Argument Groups are used to pass arguments to the provider methods.
 
-        :param spec: specification for the data structure
-        :type data_columns: list
+        Example:
+            fake.set_arguments('top_half', {'min_value': 50, 'max_value': 100})
+            fake.json(data_columns={'Name': 'name', 'Score': 'pyint:top_half'})
+
+        Data Column List format:
+            [('key name', 'definition', {'arguments'})]
+
+        With the list format the definition can be a list of records, to create
+        a list within the structure data.  For literal entries within the list,
+        set the 'field_name' to None.
+
+        :param data_columns: specification for the data structure
+        :type data_columns: dict
         :param num_rows: number of rows the returned
         :type num_rows: int
         :param indent: number of spaces to indent the fields
@@ -410,50 +426,99 @@ class Provider(BaseProvider):
         :return: Serialized JSON data
         :rtype: str
 
-        :sample: data_columns=[('id', 'pyint', {'max_value': 20})], num_rows=3
-        :sample: data_columns=[('id', 'pyint'), ('details', (('name', 'name'),))], num_rows=1
-        :sample: data_columns=[('id', 'pyint'), ('details', [(None, 'name'), (None, 'name')])], num_rows=1
-        :sample: data_columns=[('id', 'pyint'), ('details', [('name', 'name'), ('name', 'name')])], num_rows=1
+        :sample: data_columns={'ID': 'pyint', 'Details': {'Name': 'name',
+                'Address': 'address'}}, num_rows=1
+        :sample: data_columns={'Candidates': ['name', 'name', 'name']},
+                num_rows=1
+        :sample: data_columns=[('Name', 'name'), ('Points', 'pyint',
+                {'min_value': 50, 'max_value': 100})], num_rows=1
         """
+        default_data_columns = {
+            'name': '{{name}}',
+            'residency': '{{address}}',
+        }
+        data_columns = data_columns if data_columns else default_data_columns
+
+        def process_list_structure(data: list) -> dict:
+            entry = {}
+
+            for name, definition, *arguments in data:
+                kwargs = arguments[0] if arguments else {}
 
-        def create_json_entry(data_columns: list) -> OrderedDict:
-            entry = OrderedDict()
-            for field_name, provider_name, *parameters in data_columns:
-                kwargs = parameters[0] if parameters else {}
                 if not isinstance(kwargs, dict):
-                    raise TypeError("Parameters must be a dictionary")
+                    raise TypeError('Invalid arguments type. Must be a dictionary')
+
+                if name is None:
+                    return self._format_selection(definition, **kwargs)
+
+                if isinstance(definition, tuple):
+                    entry[name] = process_list_structure(definition)
+                elif isinstance(definition, (list, set)):
+                    entry[name] = [process_list_structure([item])
+                                   for item in definition]
+                else:
+                    entry[name] = self._format_selection(definition, **kwargs)
+            return entry
+
+        def process_dict_structure(data: dict) -> dict:
+            entry = {}
 
-                if field_name is None:
-                    return self.generator.format(provider_name, **kwargs)
+            if isinstance(data, str):
+                return self._format_selection(data)
 
-                if isinstance(provider_name, tuple):
-                    entry[field_name] = create_json_entry(provider_name)
-                elif isinstance(provider_name, list):
-                    entry[field_name] = [create_json_entry([item])
-                                         for item in provider_name]
+            if isinstance(data, (float, int)):
+                return data
+
+            for name, definition in data.items():
+                if isinstance(definition, (tuple, list)):
+                    entry[name] = [process_dict_structure(item)
+                                   for item in definition]
+                elif isinstance(definition, (dict, int, float)):
+                    entry[name] = process_dict_structure(definition)
                 else:
-                    entry[field_name] = self.generator.format(provider_name, **kwargs)
+                    entry[name] = self._format_selection(definition)
+
             return entry
 
+        def create_json_structure(data_columns) -> dict:
+            if isinstance(data_columns, dict):
+                return process_dict_structure(data_columns)
+
+            if isinstance(data_columns, list):
+                return process_list_structure(data_columns)
+
+            raise TypeError('Invalid data_columns type. Must be a dictionary or list')
+
         if num_rows == 1:
-            return json.dumps(create_json_entry(data_columns), indent=indent)
+            return json.dumps(create_json_structure(data_columns), indent=indent)
 
-        data = [create_json_entry(data_columns) for _ in range(num_rows)]
+        data = [create_json_structure(data_columns) for _ in range(num_rows)]
         return json.dumps(data, indent=indent)
 
     def fixed_width(self,
-                    data_columns: list = [(20, 'name'), (3, 'pyint', {'max_value': 20})],
+                    data_columns: list = None,
                     num_rows: int = 10,
                     align: str = 'left') -> str:
         """
         Generate random fixed width values.
 
-        Using a list of records that is passed as ``data_columns``, you define the structure that
-        will be generated. ``parameters`` are provider specific, and should be a dictionary that will
-        be passed to the provider method.
+        Using a list of tuple records that is passed as ``data_columns``, that
+        defines the structure that will be generated. Arguments within the
+        record are provider specific, and should be a dictionary that will be
+        passed to the provider method.
+
+        Data Column List format
+            [('field width', 'definition', {'arguments'})]
+
+        The definition can simply be the 'name:argument_group' of a provider
+        method, or can also be string tokens that are passed to python
+        provider method pystr_format() for data generation.
+        Argument Groups can be used to pass arguments to the provider methods,
+        but will override the arguments supplied in the tuple record.
 
-        Data Columns format
-            [('field_width', 'provider_name', {'parameters'})]
+        Example:
+            fake.set_arguments('top_half', {'min_value': 50, 'max_value': 100})
+            fake.fixed_width(data_columns=[(20, 'name'), (3, 'pyint:top_half')])
 
         :param data_columns: specification for the data structure
         :type data_columns: list
@@ -464,25 +529,48 @@ class Provider(BaseProvider):
         :return: Serialized Fixed Width data
         :rtype: str
 
-        :sample: align='right', data_columns=[(20, 'name'), (3, 'pyint', {'max_value': 20})], num_rows=3
+        :sample: data_columns=[(20, 'name'), (3, 'pyint', {'min_value': 50,
+                'max_value': 100})], align='right', num_rows=1
         """
+        default_data_columns = [
+            (20, 'name'),
+            (3, 'pyint', {'max_value': 20}),
+        ]
+        data_columns = data_columns if data_columns else default_data_columns
         align_map = {
             'left': '<',
             'middle': '^',
             'right': '>',
         }
-
         data = []
+
         for _ in range(num_rows):
             row = []
-            for field_width, provider_name, *parameters in data_columns:
-                kwargs = parameters[0] if parameters else {}
+
+            for width, definition, *arguments in data_columns:
+                kwargs = arguments[0] if arguments else {}
+
                 if not isinstance(kwargs, dict):
-                    raise TypeError("Parameters must be a dictionary")
+                    raise TypeError('Invalid arguments type. Must be a dictionary')
 
-                result = self.generator.format(provider_name, **kwargs)
-                field = "{0:%s%s}" % (align_map.get(align, '<'), field_width)
-                row.append(field.format(result)[:field_width])
-            data.append(''.join(row))
+                result = self._format_selection(definition, **kwargs)
+                field = "{0:%s%s}" % (align_map.get(align, '<'), width)
+                row.append(field.format(result)[:width])
 
+            data.append(''.join(row))
         return '\n'.join(data)
+
+    def _format_selection(self, definition, **kwargs):
+        """
+        Formats the string with PyStr Format if special characters are found.
+        """
+        if '{{' in definition and '}}' in definition:
+            return self.generator.pystr_format(definition)
+
+        if definition.count(':') == 1:
+            definition, argument_group = definition.split(':')
+            arguments = self.generator.get_arguments(argument_group.strip())
+
+            return self.generator.format(definition.strip(), **arguments)
+
+        return self.generator.format(definition, **kwargs)


Test patch:
diff --git a/tests/providers/test_address.py b/tests/providers/test_address.py
index 2e3f52cb..e705f359 100644
--- a/tests/providers/test_address.py
+++ b/tests/providers/test_address.py
@@ -463,6 +463,16 @@ class TestEsES(unittest.TestCase):
         secondary_address = self.fake.secondary_address()
         assert isinstance(secondary_address, str)
 
+    def test_regions(self):
+        region = self.fake.region()
+        assert isinstance(region, str)
+        assert region in EsEsProvider.regions
+
+        # Spanish regions, also known as "autonomous communities"
+        autonomous_community = self.fake.autonomous_community()
+        assert isinstance(autonomous_community, str)
+        assert autonomous_community in EsEsProvider.regions
+
 
 class TestEsMX(unittest.TestCase):
     """ Tests the addresses in the es_MX locale """
diff --git a/tests/providers/test_misc.py b/tests/providers/test_misc.py
index 3c5814e3..f81e7136 100644
--- a/tests/providers/test_misc.py
+++ b/tests/providers/test_misc.py
@@ -12,10 +12,25 @@ from unittest.mock import patch
 from faker import Faker
 
 
+class _FooBarProvider:
+
+    def foo_bar(self, param: str = None) -> str:
+        return 'FooBar' + str(param) if param else 'FooBar'
+
+    def test_integer(self, multi=1) -> int:
+        return 1 * multi
+
+    def test_float(self, multi=1) -> float:
+        return 1.1 * multi
+
+
 class TestMisc(unittest.TestCase):
     """Tests miscellaneous generators"""
     def setUp(self):
         self.fake = Faker()
+        self.fake.add_provider(_FooBarProvider())
+        self.fake.set_arguments('argument_group', 'param', 'Baz')
+        self.fake.set_arguments('double', 'multi', 2)
         Faker.seed(0)
 
     def test_uuid4_str(self):
@@ -426,76 +441,199 @@ class TestMisc(unittest.TestCase):
             kwargs['delimiter'] = '|'
             mock_dsv.assert_called_once_with(**kwargs)
 
-    def test_json_single_entry(self):
+    def test_json_with_arguments(self):
         kwargs = {
-            'data_columns': [('word', 'name'), ('number', 'pyint', {'max_value': 20})],
+            'data_columns': [
+                ('item1', '{{ foo_bar:argument_group }}'),
+                ('item2', 'foo_bar', {'param': 'BAZ'}),
+            ],
             'num_rows': 1,
         }
-        json_string = self.fake.json(**kwargs)
-        json_data = json.loads(json_string)
+        json_data = json.loads(self.fake.json(**kwargs))
 
-        assert isinstance(json_data, dict)
-        assert 'word' in json_data
-        assert 'number' in json_data
+        assert json_data.get('item1') == 'FooBarBaz'
+        assert json_data.get('item2') == 'FooBarBAZ'
 
-    def test_json_multiple_entries(self):
+    def test_json_multiple_rows(self):
         kwargs = {
-            'data_columns': [('word', 'name'), ('number', 'pyint', {'max_value': 20})],
+            'data_columns': {'item': 'foo_bar'},
             'num_rows': 2,
         }
-        json_string = self.fake.json(**kwargs)
-        json_data = json.loads(json_string)
+        json_data = json.loads(self.fake.json(**kwargs))
+
+        assert isinstance(json_data, list) and len(json_data) == 2
 
-        assert isinstance(json_data, list)
-        for entry in json_data:
-            assert 'word' in entry
-            assert 'number' in entry
+    def test_json_passthrough_int_float(self):
+        kwargs = {
+            'data_columns': {
+                'item1': 1,
+                'item2': 1.0,
+            },
+            'num_rows': 1,
+        }
+        json_data = json.loads(self.fake.json(**kwargs))
+
+        assert json_data['item1'] == 1
+        assert json_data['item2'] == 1.0
+
+    def test_json_type_integrity_int(self):
+        kwargs = {
+            'data_columns': {
+                'item1': 'test_integer',
+                'item2': 'test_integer:double',
+            },
+            'num_rows': 1,
+        }
+        json_data = json.loads(self.fake.json(**kwargs))
+        assert isinstance(json_data['item1'], int)
+        assert json_data['item2'] == 2
+
+    def test_json_type_integrity_float(self):
+        kwargs = {
+            'data_columns': {
+                'item1': 'test_float',
+                'item2': 'test_float:double',
+            },
+            'num_rows': 1,
+        }
+        json_data = json.loads(self.fake.json(**kwargs))
+        assert isinstance(json_data['item1'], float)
+        assert json_data['item2'] == 2.2
+
+    def test_json_invalid_data_columns(self):
+        kwargs = {
+            'data_columns': (('item', 'foo_bar'),),
+            'num_rows': 1,
+        }
+        with self.assertRaises(TypeError) as excinfo:
+            json.loads(self.fake.json(**kwargs))
+        assert str(excinfo.exception) == 'Invalid data_columns type. Must be a dictionary or list'
+
+    def test_json_list_format_invalid_arguments_type(self):
+        kwargs = {
+            'data_columns': [('item', 'foo_bar', ['wrong'])],
+            'num_rows': 1,
+        }
+        with self.assertRaises(TypeError) as excinfo:
+            self.fake.json(**kwargs)
+        assert str(excinfo.exception) == 'Invalid arguments type. Must be a dictionary'
 
-    def test_json_depth_structure(self):
+    def test_json_list_format_nested_list_of_values(self):
         kwargs = {
             'data_columns': [
-                ('list1', [(None, 'pyint'), (None, 'pyint')]),
-                ('list2', [('number', 'pyint'), ('number', 'pyint')]),
-                ('dict', (('number', 'pyint'), ('number', 'pyint'))),
+                (
+                    'list', [
+                        (None, '{{ foo_bar }}s'),
+                        (None, 'foo_bar'),
+                    ],
+                ),
             ],
             'num_rows': 1,
         }
+        json_data = json.loads(self.fake.json(**kwargs))
 
-        json_string = self.fake.json(**kwargs)
-        json_data = json.loads(json_string)
+        assert json_data['list'][0] == 'FooBars'
+        assert json_data['list'][1] == 'FooBar'
 
-        assert isinstance(json_data['list1'], list)
-        assert isinstance(json_data['list2'], list)
-        assert isinstance(json_data['dict'], dict)
+    def test_json_list_format_nested_list_of_objects(self):
+        kwargs = {
+            'data_columns': [
+                (
+                    'list', [
+                        ('item', '{{ foo_bar }}s'),
+                        ('item', 'foo_bar'),
+                    ],
+                ),
+            ],
+            'num_rows': 1,
+        }
+        json_data = json.loads(self.fake.json(**kwargs))
+
+        assert json_data['list'][0]['item'] == 'FooBars'
+        assert json_data['list'][1]['item'] == 'FooBar'
+
+    def test_json_list_format_nested_objects(self):
+        kwargs = {
+            'data_columns': [
+                (
+                    'dict', (
+                        ('item1', '{{ foo_bar }}s'),
+                        ('item2', 'foo_bar'),
+                    ),
+                ),
+            ],
+            'num_rows': 1,
+        }
+        json_data = json.loads(self.fake.json(**kwargs))
 
-        # Check the Lists have values, and key/values
-        for item in json_data['list1']:
-            assert isinstance(item, int)
-        for item in json_data['list2']:
-            assert isinstance(item, dict)
+        assert json_data['dict']['item1'] == 'FooBars'
+        assert json_data['dict']['item2'] == 'FooBar'
 
-    def test_json_invalid_parameter_type(self):
+    def test_json_dict_format_nested_list_of_values(self):
         kwargs = {
-            'data_columns': [('word', 'name', ['wrong'])],
+            'data_columns': {
+                'list': [
+                    '{{ foo_bar }}s',
+                    'foo_bar',
+                ],
+            },
             'num_rows': 1,
         }
-        with self.assertRaises(TypeError):
-            self.fake.json(**kwargs)
+        json_data = json.loads(self.fake.json(**kwargs))
 
-    def test_fixed_width_row_lengths(self):
+        assert json_data['list'][0] == 'FooBars'
+        assert json_data['list'][1] == 'FooBar'
+
+    def test_json_dict_format_nested_list_of_objects(self):
+        kwargs = {
+            'data_columns': {
+                'list': [
+                    {'item': '{{ foo_bar }}s'},
+                    {'item': 'foo_bar'},
+                ],
+            },
+            'num_rows': 1,
+        }
+        json_data = json.loads(self.fake.json(**kwargs))
+
+        assert json_data['list'][0]['item'] == 'FooBars'
+        assert json_data['list'][1]['item'] == 'FooBar'
+
+    def test_json_dict_format_nested_objects(self):
+        kwargs = {
+            'data_columns': {
+                'dict': {
+                    'item1': '{{ foo_bar }}s',
+                    'item2': 'foo_bar',
+                },
+            },
+            'num_rows': 1,
+        }
+        json_data = json.loads(self.fake.json(**kwargs))
+
+        assert json_data['dict']['item1'] == 'FooBars'
+        assert json_data['dict']['item2'] == 'FooBar'
+
+    def test_fixed_width_with_arguments(self):
         kwargs = {
-            'data_columns': [(20, 'name'), (3, 'pyint', {'max_value': 20})],
+            'data_columns': [
+                (9, '{{ foo_bar:argument_group }}'),
+                (9, 'foo_bar', {'param': 'BAR'}),
+            ],
             'num_rows': 2,
         }
         fixed_width_string = self.fake.fixed_width(**kwargs)
 
         for row in fixed_width_string.split('\n'):
-            assert len(row) == 23
+            assert len(row) == 18
+            assert row[0:9].strip() == 'FooBarBaz'
+            assert row[9:18].strip() == 'FooBarBAR'
 
-    def test_fixed_width_invalid_parameter_type(self):
+    def test_fixed_width_invalid_arguments_type(self):
         kwargs = {
-            'data_columns': [(20, 'name'), (3, 'pyint', ['error'])],
+            'data_columns': [(9, 'foo_bar', ['wrong'])],
             'num_rows': 1,
         }
-        with self.assertRaises(TypeError):
+        with self.assertRaises(TypeError) as excinfo:
             self.fake.fixed_width(**kwargs)
+        assert str(excinfo.exception) == 'Invalid arguments type. Must be a dictionary'