Repository: joke2k/faker

Issue:
Public IP blocks used with ipv4_public encompass reserved address namespaces
The `ipv4_public` faker generates IP addresses from public blocks, excluding private blocks configured in `IPV4_PUBLIC_NETS` and `IPV4_PRIVATE_NET_BLOCKS`, however it often results in addresses coming from reserved spaces.

One example is 192.0.0.145, or in general 192.0.0.0/24 addresses from reserved space, per https://en.wikipedia.org/wiki/Reserved_IP_addresses

This clashes with many popular libraries checking whether IP is public and routable such as `django-ipware`.

### Steps to reproduce

```
>>> startswith = False
>>> while not startswith:
...   ip = fake.ipv4_public()
...   if ip.startswith('192.0.0'):
...     startswith = True
...     print(ip)
...
192.0.0.145
```

### Expected behavior

No address starting with `192.0.0` generated

### Actual behavior

`192.0.0.145` comes up often.


Discussion:

Patch:
diff --git a/docs/communityproviders.rst b/docs/communityproviders.rst
index cd6cdfa5..08509a98 100644
--- a/docs/communityproviders.rst
+++ b/docs/communityproviders.rst
@@ -18,11 +18,7 @@ Here's a list of Providers written by the community:
 +---------------+--------------------------+----------------------------------+
 | Wi-Fi ESSID   | Fake Wi-Fi ESSIDs.       | `faker_wifi_essid`_              +
 +---------------+--------------------------+----------------------------------+
-| Credit Score  | Fake credit score data   | `faker_credit_score`_            |
-|               | for testing purposes     |                                  |
-+---------------+--------------------------+----------------------------------+
 
 .. _faker_web: https://pypi.org/project/faker_web/
 .. _faker_cloud: https://pypi.org/project/faker-cloud/
 .. _faker_wifi_essid: https://pypi.org/project/faker-wifi-essid/
-.. _faker_credit_score: https://pypi.org/project/faker-credit-score/
diff --git a/faker/providers/address/id_ID/__init__.py b/faker/providers/address/id_ID/__init__.py
index 4bd022f8..031c85b3 100644
--- a/faker/providers/address/id_ID/__init__.py
+++ b/faker/providers/address/id_ID/__init__.py
@@ -29,15 +29,15 @@ class Provider(AddressProvider):
     # From
     # http://elibrary.dephub.go.id/elibrary/media/catalog/0010-021500000000135/swf/618/Lampiran%20E%20Data%20Bandung.pdf
     streets = (
-        'Abdul Muis', 'Antapani Lama', 'Asia Afrika', 'Astana Anyar', 'BKR',
+        'Abdul Muis ', 'Antapani Lama', 'Asia Afrika', 'Astana Anyar', 'BKR',
         'Cihampelas', 'Cikapayang', 'Cikutra Barat', 'Cikutra Timur',
         'Ciumbuleuit', 'Ciwastra', 'Dipatiukur', 'Dipenogoro', 'Dr. Djunjunan',
         'Gardujati', 'Gedebage Selatan', 'Gegerkalong Hilir',
-        'HOS. Cokroaminoto', 'Ir. H. Djuanda', 'Jakarta', 'Jamika',
+        'HOS. Cokroaminoto', 'Ir. H. Djuanda', 'Jakarta ', 'Jamika',
         'Jend. A. Yani', 'Jend. Sudirman', 'K.H. Wahid Hasyim', 'Kebonjati',
         'Kiaracondong', 'Laswi', 'Lembong', 'Merdeka', 'Moch. Ramdan',
         'Moch. Toha', 'Pacuan Kuda', 'Pasir Koja', 'Pasirkoja', 'Pasteur',
-        'Pelajar Pejuang', 'Peta', 'PHH. Mustofa', 'Rajawali Barat',
+        'Pelajar Pejuang', 'Peta', 'PHH. Mustofa ', 'Rajawali Barat',
         'Rajawali Timur', 'Raya Setiabudhi', 'Raya Ujungberung', 'Rumah Sakit',
         'Sadang Serang', 'Sentot Alibasa', 'Setiabudhi', 'Siliwangi',
         'Soekarno Hatta', 'Sukabumi', 'Sukajadi', 'Suniaraja', 'Surapati',
diff --git a/faker/providers/internet/__init__.py b/faker/providers/internet/__init__.py
index 0856b267..82dbc136 100644
--- a/faker/providers/internet/__init__.py
+++ b/faker/providers/internet/__init__.py
@@ -5,7 +5,7 @@ from text_unidecode import unidecode
 
 from .. import BaseProvider
 
-from ipaddress import ip_address, ip_network, IPV4LENGTH, IPV6LENGTH
+from ipaddress import ip_address, ip_network, IPv4Address, IPV4LENGTH, IPV6LENGTH
 
 # from faker.generator import random
 # from faker.providers.lorem.la import Provider as Lorem
@@ -15,36 +15,58 @@ from faker.utils.decorators import lowercase, slugify, slugify_unicode
 localized = True
 
 
-IPV4_PUBLIC_NETS = {
-    'a': ip_network('0.0.0.0/1'),
-    'b': ip_network('128.0.0.0/2'),
-    'c': ip_network('192.0.0.0/3')
-}
-IPV4_PRIVATE_NET_BLOCKS = {
-    'a': (
-        ip_network('0.0.0.0/8'),
+class _IPv4Constants:
+    """
+    IPv4 network constants used to group networks into different categories.
+    Structure derived from `ipaddress._IPv4Constants`.
+
+    Excluded network list is updated to comply with current IANA list of
+    private and reserved networks.
+    """
+    _network_classes = {
+        'a': ip_network('0.0.0.0/1'),
+        'b': ip_network('128.0.0.0/2'),
+        'c': ip_network('192.0.0.0/3')
+    }
+
+    _linklocal_network = ip_network('169.254.0.0/16')
+
+    _loopback_network = ip_network('127.0.0.0/8')
+
+    _multicast_network = ip_network('224.0.0.0/4')
+
+    # Three common private networks from class A, B and CIDR
+    # to generate private addresses from.
+    _private_networks = [
         ip_network('10.0.0.0/8'),
+        ip_network('172.16.0.0/12'),
+        ip_network('192.168.0.0/16')
+    ]
+
+    # List of networks from which IP addresses will never be generated,
+    # includes other private IANA and reserved networks from
+    # ttps://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml
+    _excluded_networks = [
+        ip_network('0.0.0.0/8'),
+        ip_network('100.64.0.0/10'),
         ip_network('127.0.0.0/8'),
-    ),
-    'b': (
         ip_network('169.254.0.0/16'),
-        ip_network('172.16.0.0/12')
-    ),
-    'c': (
-        ip_network('192.0.0.0/29'),
-        ip_network('192.0.0.170/31'),
+        ip_network('192.0.0.0/24'),
         ip_network('192.0.2.0/24'),
-        ip_network('192.168.0.0/16'),
+        ip_network('192.31.196.0/24'),
+        ip_network('192.52.193.0/24'),
+        ip_network('192.88.99.0/24'),
+        ip_network('192.175.48.0/24'),
         ip_network('198.18.0.0/15'),
         ip_network('198.51.100.0/24'),
-        ip_network('203.0.113.0/24')
-    ),
-}
-IPV4_NET_CLASSES = {
-    'a': (167772160, 184549375, 24),
-    'b': (2886729728, 2887778303, 20),
-    'c': (3232235520, 3232301055, 16)
-}
+        ip_network('203.0.113.0/24'),
+        ip_network('240.0.0.0/4'),
+        ip_network('255.255.255.255/32')
+    ] + [
+        _linklocal_network,
+        _loopback_network,
+        _multicast_network
+    ]
 
 
 class Provider(BaseProvider):
@@ -213,6 +235,80 @@ class Provider(BaseProvider):
 
         return self.generator.parse(pattern)
 
+    def _random_ipv4_address_from_subnet(self, subnet, network=False):
+        """
+        Produces a random IPv4 address or network with a valid CIDR
+        from within a given subnet.
+
+        :param subnet: IPv4Network to choose from within
+        :param network: Return a network address, and not an IP address
+        """
+        address = str(
+            subnet[self.generator.random.randint(
+                0, subnet.num_addresses - 1
+            )]
+        )
+
+        if network:
+            address += '/' + str(self.generator.random.randint(
+                subnet.prefixlen,
+                subnet.max_prefixlen
+            ))
+            address = str(ip_network(address, strict=False))
+
+        return address
+
+    def _exclude_ipv4_networks(self, networks, networks_to_exclude):
+        """
+        Exclude the list of networks from another list of networks
+        and return a flat list of new networks.
+
+        :param networks: List of IPv4 networks to exclude from
+        :param networks_to_exclude: List of IPv4 networks to exclude
+        :returns: Flat list of IPv4 networks
+        """
+        for network_to_exclude in networks_to_exclude:
+            def _exclude_ipv4_network(network):
+                """
+                Exclude a single network from another single network
+                and return a list of networks. Network to exclude
+                comes from the outer scope.
+
+                :param network: Network to exclude from
+                :returns: Flat list of IPv4 networks after exclusion.
+                          If exclude fails because networks do not
+                          overlap, a single element list with the
+                          orignal network is returned. If it overlaps,
+                          even partially, the network is excluded.
+                """
+                try:
+                    return list(network.address_exclude(network_to_exclude))
+                except ValueError:
+                    # If networks overlap partially, `address_exclude`
+                    # will fail, but the network still must not be used
+                    # in generation.
+                    if network.overlaps(network_to_exclude):
+                        return []
+                    else:
+                        return [network]
+
+            networks = list(map(_exclude_ipv4_network, networks))
+
+            # flatten list of lists
+            networks = [
+                item for nested in networks for item in nested
+            ]
+
+        return networks
+
+    def ipv4_network_class(self):
+        """
+        Returns a IPv4 network class 'a', 'b' or 'c'.
+
+        :returns: IPv4 network class
+        """
+        return self.random_element('abc')
+
     def ipv4(self, network=False, address_class=None, private=None):
         """
         Produce a random IPv4 address or network with a valid CIDR.
@@ -228,24 +324,25 @@ class Provider(BaseProvider):
         elif private is False:
             return self.ipv4_public(address_class=address_class,
                                     network=network)
-        if address_class is None:
-            ip_range = (0, (2 ** IPV4LENGTH) - 1)
+
+        # if neither private nor public is required explicitly,
+        # generate from whole requested address space
+        if address_class:
+            all_networks = [_IPv4Constants._network_classes[address_class]]
         else:
-            net = IPV4_PUBLIC_NETS[address_class]
-            ip_range = (net._ip[0], net.ip[-1])
-        address = str(ip_address(self.generator.random.randint(*ip_range)))
-        if network:
-            address += '/' + str(self.generator.random.randint(0, IPV4LENGTH))
-            address = str(ip_network(address, strict=False))
-        return address
+            # if no address class is choosen, use whole IPv4 pool
+            all_networks = [ip_network('0.0.0.0/0')]
 
-    def ipv4_network_class(self):
-        """
-        Returns a IPv4 network class 'a', 'b' or 'c'.
+        # exclude special networks
+        all_networks = self._exclude_ipv4_networks(
+            all_networks,
+            _IPv4Constants._excluded_networks
+        )
 
-        :returns: IPv4 network class
-        """
-        return self.random_element('abc')
+        # choose random network from the list
+        random_network = self.generator.random.choice(all_networks)
+
+        return self._random_ipv4_address_from_subnet(random_network, network)
 
     def ipv4_private(self, network=False, address_class=None):
         """
@@ -255,14 +352,26 @@ class Provider(BaseProvider):
         :param address_class: IPv4 address class (a, b, or c)
         :returns: Private IPv4
         """
-        address_class = address_class or self.ipv4_network_class()
-        min_, max_, netmask = IPV4_NET_CLASSES[address_class]
-        address = str(ip_address(
-            self.generator.random.randint(min_, max_)))
-        if network:
-            address += '/' + str(self.generator.random.randint(netmask, 31))
-            address = str(ip_network(address, strict=False))
-        return address
+        # compute private networks from given class
+        supernet = _IPv4Constants._network_classes[
+            address_class or self.ipv4_network_class()
+        ]
+
+        private_networks = [
+            subnet for subnet in _IPv4Constants._private_networks
+            if subnet.overlaps(supernet)
+        ]
+
+        # exclude special networks
+        private_networks = self._exclude_ipv4_networks(
+            private_networks,
+            _IPv4Constants._excluded_networks
+        )
+
+        # choose random private network from the list
+        private_network = self.generator.random.choice(private_networks)
+
+        return self._random_ipv4_address_from_subnet(private_network, network)
 
     def ipv4_public(self, network=False, address_class=None):
         """
@@ -272,34 +381,22 @@ class Provider(BaseProvider):
         :param address_class: IPv4 address class (a, b, or c)
         :returns: Public IPv4
         """
-        address_class = address_class or self.ipv4_network_class()
-        public_net = IPV4_PUBLIC_NETS[address_class]
-        private_blocks = IPV4_PRIVATE_NET_BLOCKS[address_class]
-        # Make valid IP ranges, created from private block exclusion
-        net_ranges = []
-        ## Starts at end of 1st block if it's 1st of class
-        if public_net[0] != private_blocks[0][0]:
-            net_ranges = [(public_net[0]._ip, private_blocks[0][0]._ip-1)]
-        ## Loop on private blocks and guess available ranges
-        for i, block in enumerate(private_blocks):
-            if i+1 == len(private_blocks):
-                break
-            net_range = (block[-1]._ip+1, private_blocks[i+1][0]._ip-1)
-            net_ranges.append(net_range)
-        ## Add last range
-        net_ranges.append((private_blocks[-1][-1]._ip-1, public_net[-1]._ip-1))
-        net_ranges = [(i, j) for i, j in net_ranges if (j-i) > 0]
-        # Choose a range
-        min_, max_ = self.generator.random.choice(net_ranges)
-        address = str(ip_address(
-            self.generator.random.randint(min_, max_)))
-        # Add network mask
-        if network:
-            net_masks = {'a': 8, 'b': 16, 'c': 24}
-            min_net_mask = net_masks[address_class]
-            address += '/' + str(self.generator.random.randint(min_net_mask, 31))
-            address = str(ip_network(address, strict=False))
-        return address
+        # compute public networks
+        public_networks = [_IPv4Constants._network_classes[
+            address_class or self.ipv4_network_class()
+        ]]
+
+        # exclude private and excluded special networks
+        public_networks = self._exclude_ipv4_networks(
+            public_networks,
+            _IPv4Constants._private_networks +
+            _IPv4Constants._excluded_networks
+        )
+
+        # choose random public network from the list
+        public_network = self.generator.random.choice(public_networks)
+
+        return self._random_ipv4_address_from_subnet(public_network, network)
 
     def ipv6(self, network=False):
         """Produce a random IPv6 address or network with a valid CIDR"""
diff --git a/faker/providers/person/dk_DK/__init__.py b/faker/providers/person/dk_DK/__init__.py
index 22f1cf5f..16077411 100644
--- a/faker/providers/person/dk_DK/__init__.py
+++ b/faker/providers/person/dk_DK/__init__.py
@@ -29,9 +29,9 @@ class Provider(PersonProvider):
         'Brian', 'Bruno', 'Bøje', 'Børge', 'Carl', 'Carlo', 'Carsten',
         'Casper', 'Christian', 'Christoffer', 'Christopher', 'Claus', 'Clavs', 'Curt',
         'Dan', 'Daniel', 'Danny', 'David', 'Dennis', 'Ebbe', 'Einar',
-        'Einer', 'Elias', 'Emil', 'Eric', 'Erik', 'Erling', 'Ernst',
-        'Esben', 'Finn', 'Flemming', 'Frank', 'Frans', 'Freddy', 'Frede',
-        'Frederik', 'Frode', 'Georg', 'George', 'Gert', 'Gorm', 'Gunnar',
+        'Einer', 'Elias', 'Emil ', 'Eric', 'Erik', 'Erling', 'Ernst',
+        'Esben', 'Finn', 'Flemming ', 'Frank', 'Frans', 'Freddy', 'Frede',
+        'Frederik', 'Frode', 'Georg ', 'George', 'Gert', 'Gorm', 'Gunnar',
         'Gunner', 'Gustav', 'Hans', 'Helge', 'Henrik', 'Henry', 'Herbert',
         'Herman', 'Hjalte', 'Holger', 'Hugo', 'Ib', 'Ivan', 'Iver',
         'Jack', 'Jacob', 'Jakob', 'James', 'Jan', 'Jano', 'Jarl',
@@ -75,14 +75,14 @@ class Provider(PersonProvider):
         'Helen', 'Helle', 'Henriette', 'Herdis', 'Iben', 'Ida', 'Inga',
         'Inge', 'Ingelise', 'Inger', 'Ingrid', 'Irma', 'Isabella', 'Jacobine',
         'Jacqueline', 'Janne', 'Janni', 'Jannie', 'Jasmin', 'Jean', 'Jenny',
-        'Joan', 'Johanne', 'Jonna', 'Josefine', 'Josephine', 'Julie', 'Justina',
+        'Joan', 'Johanne', 'Jonna', 'Josefine', 'Josephine ', 'Julie', 'Justina',
         'Jytte', 'Karen', 'Karin', 'Karina', 'Karla', 'Karoline', 'Katcha',
         'Katja', 'Katrine', 'Kirsten', 'Kirstin', 'Kirstine', 'Klara', 'Kristina',
         'Kristine', 'Laura', 'Lea', 'Lena', 'Lene', 'Leonora', 'Line',
         'Liva', 'Lona', 'Lone', 'Lotte', 'Louise', 'Lærke', 'Maiken',
         'Maja', 'Majken', 'Malene', 'Malou', 'Maren', 'Margit', 'Margrethe',
         'Maria', 'Marianne', 'Marie', 'Marlene', 'Mathilde', 'Maya', 'Merete',
-        'Merethe', 'Mette', 'Mia', 'Michala', 'Michelle', 'Mie', 'Mille',
+        'Merethe', 'Mette ', 'Mia', 'Michala', 'Michelle', 'Mie', 'Mille',
         'Mimi', 'Minna', 'Nadia', 'Naja', 'Nana', 'Nanna', 'Nanni',
         'Natasha', 'Natasja', 'Nete', 'Nicoline', 'Nina', 'Nora', 'Oda',
         'Odeline', 'Odette', 'Ofelia', 'Olga', 'Olivia', 'Patricia', 'Paula',
diff --git a/faker/providers/person/tr_TR/__init__.py b/faker/providers/person/tr_TR/__init__.py
index 014e55d5..6c930ffa 100644
--- a/faker/providers/person/tr_TR/__init__.py
+++ b/faker/providers/person/tr_TR/__init__.py
@@ -120,7 +120,8 @@ class Provider(PersonProvider):
                           'Şahıgül', 'Şamiha', 'Şayan', 'Şazime', 'Şefiye', 'Şehreban', 'Şehza',
                           'Şelâle', 'Şemsinisa', 'Şendoğan', 'Şennur', 'Şeref', 'Şerman',
                           'Şevketfeza', 'Şeyda', 'Şilan', 'Şirivan', 'Şöhret', 'Şüküfe'
-    )
+
+                          )
 
     first_names_male = ('Abdiş', 'Abdulbekir', 'Abdulgazi', 'Abdulkadir',
                         'Abdulmenaf', 'Abdulsemet', 'Abdurrahman', 'Abdülahat', 'Abdülcemal',
@@ -253,7 +254,8 @@ class Provider(PersonProvider):
                         'Şahmettin', 'Şali', 'Şanlı', 'Şavki', 'Şefi', 'Şehamet', 'Şekim',
                         'Şemsettin', 'Şendoğan', 'Şenkal', 'Şerafeddin', 'Şevket', 'Şide',
                         'Şinasi', 'Şuayp', 'Şükri'
-    )
+
+                        )
 
     first_names = first_names_male + first_names_female
 


Test patch:
diff --git a/tests/test_factory.py b/tests/test_factory.py
index 39ed0084..a5c3633b 100644
--- a/tests/test_factory.py
+++ b/tests/test_factory.py
@@ -6,7 +6,7 @@ import re
 import unittest
 import string
 import sys
-from ipaddress import ip_address, ip_network
+from ipaddress import ip_address, ip_network, IPv4Address
 
 import logging
 
@@ -441,7 +441,7 @@ class FactoryTestCase(unittest.TestCase):
             self.assertIn(klass, 'abc')
 
     def test_ipv4_private(self):
-        from faker.providers.internet import Provider
+        from faker.providers.internet import Provider, _IPv4Constants
         provider = Provider(self.generator)
 
         for _ in range(999):
@@ -463,8 +463,12 @@ class FactoryTestCase(unittest.TestCase):
                 re.compile(r'^(\d{1,3}\.){3}\d{1,3}/\d{1,2}$').search(address))
 
     def test_ipv4_private_class_a(self):
-        from faker.providers.internet import Provider
+        from faker.providers.internet import Provider, _IPv4Constants
         provider = Provider(self.generator)
+        
+        class_network = _IPv4Constants._network_classes['a']
+        class_min = class_network.network_address
+        class_max = class_network.broadcast_address
 
         for _ in range(999):
             address = provider.ipv4_private(address_class='a')
@@ -472,12 +476,16 @@ class FactoryTestCase(unittest.TestCase):
             self.assertTrue(len(address) >= 7)
             self.assertTrue(len(address) <= 15)
             self.assertTrue(ip_address(address).is_private)
-            self.assertTrue(
-                re.compile(r'^10\.(\d{1,3}\.){2}\d{1,3}$').search(address))
+            self.assertTrue(ip_address(address) >= class_min)
+            self.assertTrue(ip_address(address) <= class_max)
 
     def test_ipv4_private_class_b(self):
-        from faker.providers.internet import Provider
+        from faker.providers.internet import Provider, _IPv4Constants
         provider = Provider(self.generator)
+        
+        class_network = _IPv4Constants._network_classes['b']
+        class_min = class_network.network_address
+        class_max = class_network.broadcast_address
 
         for _ in range(999):
             address = provider.ipv4_private(address_class='b')
@@ -485,12 +493,16 @@ class FactoryTestCase(unittest.TestCase):
             self.assertTrue(len(address) >= 7)
             self.assertTrue(len(address) <= 15)
             self.assertTrue(ip_address(address).is_private)
-            self.assertTrue(
-                re.compile(r'^172\.(\d{1,3}\.){2}\d{1,3}$').search(address))
+            self.assertTrue(ip_address(address) >= class_min)
+            self.assertTrue(ip_address(address) <= class_max)
 
     def test_ipv4_private_class_c(self):
-        from faker.providers.internet import Provider
+        from faker.providers.internet import Provider, _IPv4Constants
         provider = Provider(self.generator)
+        
+        class_network = _IPv4Constants._network_classes['c']
+        class_min = class_network.network_address
+        class_max = class_network.broadcast_address
 
         for _ in range(999):
             address = provider.ipv4_private(address_class='c')
@@ -498,12 +510,12 @@ class FactoryTestCase(unittest.TestCase):
             self.assertTrue(len(address) >= 7)
             self.assertTrue(len(address) <= 15)
             self.assertTrue(ip_address(address).is_private)
-            self.assertTrue(
-                re.compile(r'^192\.168\.\d{1,3}\.\d{1,3}$').search(address))
+            self.assertTrue(ip_address(address) >= class_min)
+            self.assertTrue(ip_address(address) <= class_max)
 
     def test_ipv4_public(self):
-        from faker.providers.internet import Provider
-        provider = Provider(self.generator)
+        from faker.providers.internet import Provider, _IPv4Constants
+        provider = Provider(self.generator)   
 
         for _ in range(999):
             address = provider.ipv4_public()