Repository: pallets/werkzeug

Issue:
runaway memory usage on 0.15.x
I've narrowed it down to this little script, haven't dug into it yet but the smoking gun is the new function compilation stuff

```python
from werkzeug.routing import Map, Rule


def main():
    while True:
        Map([Rule('/a/<string:b>')])


if __name__ == '__main__':
    exit(main())
```


Discussion:
@edk0 
Seems you're saying that rules aren't being GC'd correctly?

I'm not clear under what real circumstances this would happen. Typically you define a set of rules and then use them, they're  not arbitrarily created and deleted.
our usage here involves a set of redirects defined in a configuration file, this configuration file changes periodically as we add or remove vanity routes

instead of deploying the application each time we want to add a vanity route, we simply update the configuration file and the application rebuilds a route mapping (which is used by a `flask` app to serve redirects)
Hmm, that's usually discouraged because changes to the map aren't synchronized in multiprocess workers. I'd probably implement it instead as an error handler for 404, to check if a redirect should be returned instead. Not saying it shouldn't be fixed, just that it's not a use case I'd heard of.
each individual worker checks the configuration file periodically and reloads it -- it isn't being injected directly into the flask app as far as I can tell

either way, these objects probably shouldn't leak :laughing: -- I'm looking into what could cause that -- I suspect either the functions being compiled have issue or the hashing of those (since the `Map` seems to also be involved in some way)
Of note, this leak doesn't happen without the `<string:b>` portion
here's the disassembly of the function objects it creates:
```pycon
>>> x = Rule('/a/<string:b>')
>>> from werkzeug.routing import Map
>>> y = Map([x])
>>> x._build
<function <builder:'/a/<string:b>'> at 0x7f16d62d1730>
>>> import dis
>>> dis.dis(x._build)
  1           0 LOAD_CONST               0 ('')
              2 LOAD_CONST               1 ('/a/')
              4 LOAD_CONST               2 (<bound method BaseConverter.to_url of <werkzeug.routing.UnicodeConverter object at 0x7f16d2c9a0f0>>)
              6 LOAD_FAST                0 (b)
              8 CALL_FUNCTION            1
             10 BUILD_STRING             2
             12 BUILD_TUPLE              2
             14 RETURN_VALUE
>>> dis.dis(x._build_unknown)
  1           0 LOAD_CONST               0 ('')
              2 LOAD_CONST               1 ('/a/')
              4 LOAD_CONST               2 (<bound method BaseConverter.to_url of <werkzeug.routing.UnicodeConverter object at 0x7f16d2c9a0f0>>)
              6 LOAD_FAST                0 (b)
              8 CALL_FUNCTION            1
             10 LOAD_FAST                1 (.keyword_arguments)
             12 JUMP_IF_TRUE_OR_POP     20
             14 LOAD_CONST               0 ('')
             16 DUP_TOP
             18 JUMP_FORWARD            10 (to 30)
        >>   20 LOAD_CONST               3 (functools.partial(<function url_encode at 0x7f16d2d5d510>, charset='utf-8', sort=False, key=None))
             22 ROT_TWO
             24 CALL_FUNCTION            1
             26 LOAD_CONST               4 ('?')
             28 ROT_TWO
        >>   30 BUILD_STRING             4
             32 BUILD_TUPLE              2
             34 RETURN_VALUE
```
Adjusting the script slightly:

```python
import collections
import gc
import pprint
from werkzeug.routing import Map, Rule


def main():
    for _ in range(10000):
        Map([Rule('/a/<string:b>')])
    for _ in range(5):
        gc.collect()
    counts = collections.Counter(type(o) for o in gc.get_objects())
    pprint.pprint(counts.most_common(15))


if __name__ == '__main__':
    exit(main())
```

it looks like it is leaking (at the very least, probably more in the other common types above it as well) the `Map`, `Rule`, as well as a `functools.partial` and a `UnicodeConverter` per call:

```console
$ ./venv/bin/python t.py
[(<class 'dict'>, 62085),
 (<class 'list'>, 50514),
 (<class 'function'>, 24085),
 (<class 'tuple'>, 21811),
 (<class 'method'>, 20032),
 (<class 'set'>, 10518),
 (<class 'functools.partial'>, 10002),
 (<class 'werkzeug.routing.UnicodeConverter'>, 10000),
 (<class 'werkzeug.routing.Map'>, 10000),
 (<class 'werkzeug.routing.Rule'>, 10000),
 (<class 'weakref'>, 1306),
 (<class 'wrapper_descriptor'>, 1131),
 (<class 'method_descriptor'>, 879),
 (<class 'builtin_function_or_method'>, 839),
 (<class 'getset_descriptor'>, 740)]
```
here's some graphs of the things keeping this alive in gc:

```python
def graph(obj, ids, *, seen=None, indent='', limit=10):
    if seen is None:
        seen = set()

    for referrer in gc.get_referrers(obj):
        # the main frame which has a hard reference to ths object
        if (
                type(referrer).__name__ == 'frame' and
                referrer.f_globals['__name__'] == '__main__'
        ):
            continue
        # objects only present due to traversal of gc referrers
        elif id(referrer) not in ids:
            continue
        elif id(referrer) in seen:
            print(f'{indent}(already seen) {id(referrer)}')
            continue

        seen.add(id(referrer))

        if indent == '':
            print('=' * 79)
        print(f'{indent}type: {type(referrer).__name__} ({id(referrer)})')
        fmted = repr(referrer)  #pprint.pformat(referrer)
        print(indent + fmted.replace('\n', f'\n{indent}'))

        if limit:
            graph(
                referrer, ids,
                seen=seen, indent='==' + indent, limit=limit - 1,
            )

...

    ids = {id(o) for o in gc.get_objects()}
    obj = next(iter(o for o in gc.get_objects() if isinstance(o, Map)))
    graph(obj, ids)
```

```
===============================================================================
type: dict (140272699432680)
{'map': Map([<Rule '/a/<b>' -> None>]), 'regex': '[^/]{1,}'}
==type: UnicodeConverter (140272699175656)
==<werkzeug.routing.UnicodeConverter object at 0x7f93c867f2e8>
====type: method (140272750734216)
====<bound method BaseConverter.to_url of <werkzeug.routing.UnicodeConverter object at 0x7f93c867f2e8>>
======type: tuple (140272726348928)
======('', '/a/', <bound method BaseConverter.to_url of <werkzeug.routing.UnicodeConverter object at 0x7f93c867f2e8>>)
====type: method (140272749846664)
====<bound method BaseConverter.to_url of <werkzeug.routing.UnicodeConverter object at 0x7f93c867f2e8>>
======type: tuple (140272726372424)
======('', '/a/', <bound method BaseConverter.to_url of <werkzeug.routing.UnicodeConverter object at 0x7f93c867f2e8>>, functools.partial(<function url_encode at 0x7f93c877eae8>, charset='utf-8', sort=False, key=None), '?')
====type: dict (140272723298056)
===={'b': <werkzeug.routing.UnicodeConverter object at 0x7f93c867f2e8>}
======type: dict (140272749460432)
======{'rule': '/a/<string:b>', 'is_leaf': True, 'map': Map([<Rule '/a/<b>' -> None>]), 'strict_slashes': True, 'subdomain': '', 'host': None, 'defaults': None, 'build_only': False, 'alias': False, 'methods': None, 'endpoint': None, 'redirect_to': None, 'arguments': {'b'}, '_trace': [(False, '|'), (False, '/a/'), (True, 'b')], '_converters': {'b': <werkzeug.routing.UnicodeConverter object at 0x7f93c867f2e8>}, '_regex': re.compile('^\\|\\/a\\/(?P<b>[^/]{1,})$'), '_argument_weights': [100], '_static_weights': [(0, -1)], '_build': <function <builder:'/a/<string:b>'> at 0x7f93c9d880d0>, '_build_unknown': <function <builder:'/a/<string:b>'> at 0x7f93c9d88158>}
========type: Rule (140272749480984)
========<Rule '/a/<b>' -> None>
==========type: list (140272699123848)
==========[<Rule '/a/<b>' -> None>]
============type: dict (140272726280736)
============{'_rules': [<Rule '/a/<b>' -> None>], '_rules_by_endpoint': {None: [<Rule '/a/<b>' -> None>]}, '_remap': False, '_remap_lock': <unlocked _thread.lock object at 0x7f93cb6d9da0>, 'default_subdomain': '', 'charset': 'utf-8', 'encoding_errors': 'replace', 'strict_slashes': True, 'redirect_defaults': True, 'host_matching': False, 'converters': {'default': <class 'werkzeug.routing.UnicodeConverter'>, 'string': <class 'werkzeug.routing.UnicodeConverter'>, 'any': <class 'werkzeug.routing.AnyConverter'>, 'path': <class 'werkzeug.routing.PathConverter'>, 'int': <class 'werkzeug.routing.IntegerConverter'>, 'float': <class 'werkzeug.routing.FloatConverter'>, 'uuid': <class 'werkzeug.routing.UUIDConverter'>}, 'sort_parameters': False, 'sort_key': None}
==============type: Map (140272749480872)
==============Map([<Rule '/a/<b>' -> None>])
================(already seen) 140272699432680
================(already seen) 140272749460432
==========type: list (140272700110792)
==========[<Rule '/a/<b>' -> None>]
============type: dict (140272726280808)
============{None: [<Rule '/a/<b>' -> None>]}
==============(already seen) 140272726280736
(already seen) 140272749460432

```
This is sufficient to break the cycle and eliminate the memory leak:

```diff
diff --git a/src/werkzeug/routing.py b/src/werkzeug/routing.py
index c7cff94d..8176ddfe 100644
--- a/src/werkzeug/routing.py
+++ b/src/werkzeug/routing.py
@@ -1254,13 +1254,13 @@ class BaseConverter(object):
     weight = 100
 
     def __init__(self, map):
-        self.map = map
+        self.charset = map.charset
 
     def to_python(self, value):
         return value
 
     def to_url(self, value):
-        return _fast_url_quote(text_type(value).encode(self.map.charset))
+        return _fast_url_quote(text_type(value).encode(self.charset))
 
 
 class UnicodeConverter(BaseConverter):
```

though I believe the actual cause of the cycle is that `LOAD_CONST` is being used in the code objects against objects which aren't themselves constants, I believe this confuses the gc (assuming that the referred code objects are live, when they are actually not)

Resolving patch:
diff --git a/src/werkzeug/routing.py b/src/werkzeug/routing.py
--- a/src/werkzeug/routing.py
+++ b/src/werkzeug/routing.py
@@ -95,14 +95,11 @@
     :copyright: 2007 Pallets
     :license: BSD-3-Clause
 """
+import ast
 import difflib
-import dis
 import posixpath
 import re
-import sys
-import types
 import uuid
-from functools import partial
 from pprint import pformat
 from threading import Lock
 
@@ -110,7 +107,6 @@
 from ._compat import iteritems
 from ._compat import itervalues
 from ._compat import native_string_result
-from ._compat import PY2
 from ._compat import string_types
 from ._compat import text_type
 from ._compat import to_bytes
@@ -490,6 +486,29 @@ def get_rules(self, map):
                 )
 
 
+def _prefix_names(src):
+    """ast parse and prefix names with `.` to avoid collision with user vars"""
+    tree = ast.parse(src).body[0]
+    if isinstance(tree, ast.Expr):
+        tree = tree.value
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Name):
+            node.id = "." + node.id
+    return tree
+
+
+_CALL_CONVERTER_CODE_FMT = "self._converters[{elem!r}].to_url()"
+_IF_KWARGS_URL_ENCODE_CODE = """\
+if kwargs:
+    q = '?'
+    params = self._encode_query_vars(kwargs)
+else:
+    q = params = ''
+"""
+_IF_KWARGS_URL_ENCODE_AST = _prefix_names(_IF_KWARGS_URL_ENCODE_CODE)
+_URL_ENCODE_AST_NAMES = (_prefix_names("q"), _prefix_names("params"))
+
+
 @implements_to_string
 class Rule(RuleFactory):
     """A Rule represents one URL pattern.  There are some options for `Rule`
@@ -719,6 +738,14 @@ def get_converter(self, variable_name, converter_name, args, kwargs):
             raise LookupError("the converter %r does not exist" % converter_name)
         return self.map.converters[converter_name](self.map, *args, **kwargs)
 
+    def _encode_query_vars(self, query_vars):
+        return url_encode(
+            query_vars,
+            charset=self.map.charset,
+            sort=self.map.sort_parameters,
+            key=self.map.sort_key,
+        )
+
     def compile(self):
         """Compiles the regular expression and stores it."""
         assert self.map is not None, "rule not bound"
@@ -764,8 +791,8 @@ def _build_regex(rule):
         if not self.is_leaf:
             self._trace.append((False, "/"))
 
-        self._build = self._compile_builder(False)
-        self._build_unknown = self._compile_builder(True)
+        self._build = self._compile_builder(False).__get__(self, None)
+        self._build_unknown = self._compile_builder(True).__get__(self, None)
 
         if self.build_only:
             return
@@ -825,308 +852,102 @@ def match(self, path, method=None):
 
                 return result
 
-    class BuilderCompiler:
-        JOIN_EMPTY = "".join
-        if sys.version_info >= (3, 6):
-            OPARG_SIZE = 256
-            OPARG_VARI = False
-        else:
-            OPARG_SIZE = 65536
-            OPARG_VARI = True
-
-        def __init__(self, rule):
-            self.rule = rule
-            self.consts = []
-            self.const_table = {}
-            self.var = []
-            self.var_table = {}
-            self.argdefs = ()
-            self.defaults = dict(self.rule.defaults or {})
-
-        def get_const(self, x):
-            """Return a constant ID for an object, adding it to the pool
-            if not already present.
-            """
-            if x not in self.const_table:
-                self.const_table[x] = len(self.consts)
-                self.consts.append(x)
-            return self.const_table[x]
-
-        def get_var(self, x):
-            """Return a local variable ID for a name, adding it to the
-            pool if not already present.
-
-            Our only use for local variables is as function arguments:
-            any variable name that exists before the call to
-            ``add_defaults()`` will become one.
-            """
-            x = str(x)
-            if x not in self.var_table:
-                self.var_table[x] = len(self.var)
-                self.var.append(x)
-            return self.var_table[x]
-
-        def add_defaults(self):
-            """A rule builder is allowed to receive any of its defaults
-            as arguments. We don't bother to check that they match
-            anywhere, since ``suitable_for()`` should have already done
-            that, but we do need them to be optional arguments. Since
-            their values are known at compile-time, the builder will
-            never refer to these arguments.
-            """
-            # ensure every default exists
-            for k in self.defaults.keys():
-                self.get_var(k)
-            # reorder to put anything with a default at the end
-            req = []
-            opt = []
-            defs = []
-            for k in self.var:
-                if k in self.defaults:
-                    opt.append(k)
-                    defs.append(self.defaults[k])
-                else:
-                    req.append(k)
-            self.var = req + opt
-            self.argdefs = tuple(defs)
-            for i, k in enumerate(self.var):
-                self.var_table[k] = i
-
-        def collapse_constants(self, opl):
-            """Given a list of build operations, spit out a new list
-            with runs of constant elements joined."""
-            new = []
-            for op, elem in opl:
-                if op is not None:
-                    new.append((op, elem))
-                    continue
-                if elem == "":
-                    continue
-                if not new or new[-1][0] is not None:
-                    new.append((op, elem))
-                    continue
-                new[-1] = (None, new[-1][1] + elem)
-            if not new:
-                new.append((None, ""))
-            return new
-
-        def build_op(self, op, arg=None):
-            """Return a byte representation of a Python instruction."""
-            if isinstance(op, str):
-                op = dis.opmap[op]
-            if arg is None and op >= dis.HAVE_ARGUMENT:
-                raise ValueError("Operation requires an argument: %s" % dis.opname[op])
-            if arg is not None and op < dis.HAVE_ARGUMENT:
-                raise ValueError("Operation takes no argument: %s" % dis.opname[op])
-            if arg is None:
-                arg = 0
-            # Python 3.6 changed the argument to an 8-bit integer, so this
-            # could be a practical consideration
-            if arg >= self.OPARG_SIZE:
-                return self.build_op(
-                    "EXTENDED_ARG", arg // self.OPARG_SIZE
-                ) + self.build_op(op, arg % self.OPARG_SIZE)
-            if not self.OPARG_VARI:
-                return bytearray((op, arg))
-            elif op >= dis.HAVE_ARGUMENT:
-                return bytearray((op, arg % 256, arg // 256))
-            else:
-                return bytearray((op,))
-
-        def build_string(self, n):
-            """Return the correct opcode(s) for building a string from
-            ``n`` elements. If the ``''.join`` crutch is needed, it must
-            already be immediately below the string elements on the
-            stack.
-            """
-            if "BUILD_STRING" in dis.opmap:
-                return self.build_op("BUILD_STRING", n)
-            else:
-                return self.build_op("BUILD_TUPLE", n) + self.build_op(
-                    "CALL_FUNCTION", 1
-                )
-
-        def emit_build(
-            self, ind, opl, append_unknown=False, encode_query_vars=None, kwargs=None
-        ):
-            ops = b""
-            n = len(opl)
-            stack = 0
-            stack_overhead = 0
-
-            for op, elem in opl:
-                if op is None:
-                    ops += self.build_op("LOAD_CONST", self.get_const(elem))
-                    stack_overhead = 0
-                    continue
-                ops += self.build_op("LOAD_CONST", self.get_const(op))
-                ops += self.build_op("LOAD_FAST", self.get_var(elem))
-                ops += self.build_op("CALL_FUNCTION", 1)
-                stack_overhead = 2
-
-            stack += len(opl)
-            peak_stack = stack + stack_overhead
-            dont_build_string = False
-            needs_build_string = "BUILD_STRING" not in dis.opmap
-
-            if n <= 1:
-                dont_build_string = True
-                needs_build_string = False
+    def _compile_builder(self, append_unknown=True):
+        defaults = self.defaults or {}
+        dom_ops = []
+        url_ops = []
 
-            if append_unknown:
-                if "BUILD_STRING" not in dis.opmap:
-                    needs_build_string = True
-                    ops = (
-                        self.build_op("LOAD_CONST", self.get_const(self.JOIN_EMPTY))
-                        + ops
-                    )
-                ops += self.build_op("LOAD_FAST", kwargs)
-
-                # assemble this in its own buffers because we need to
-                # jump over it
-                uops = bytearray()  # run if kwargs. TOS=kwargs
-                uops += self.build_op("LOAD_CONST", self.get_const(encode_query_vars))
-                uops += self.build_op("ROT_TWO")
-                uops += self.build_op("CALL_FUNCTION", 1)
-                uops += self.build_op("LOAD_CONST", self.get_const("?"))
-                uops += self.build_op("ROT_TWO")
-                if dont_build_string:
-                    uops += self.build_string(n + 2)
-
-                nops = bytearray()  # otherwise
-                if not dont_build_string:
-                    # if we're going to build a string, we need to pad out to
-                    # a constant length
-                    nops += self.build_op("LOAD_CONST", self.get_const(""))
-                    nops += self.build_op("DUP_TOP")
-                elif needs_build_string:
-                    # we inserted the ''.join reference at the bottom of the
-                    # stack, but we don't want to call it: throw it away
-                    nops += self.build_op("ROT_TWO")
-                    nops += self.build_op("POP_TOP")
-                nops += self.build_op("JUMP_FORWARD", len(uops))
-
-                # this jump needs to take its own length into account. the
-                # simple way to do that is to compute a minimal guess for the
-                # length of the jump instruction, and keep revising it upward
-                jump_op = self.build_op("JUMP_IF_TRUE_OR_POP", 0)
-                while True:
-                    jump_len = len(jump_op)
-                    jump_target = ind + len(ops) + jump_len + len(nops)
-                    jump_op = self.build_op("JUMP_IF_TRUE_OR_POP", jump_target)
-                    assert len(jump_op) >= jump_len
-                    if len(jump_op) == jump_len:
-                        break
-
-                ops += jump_op
-                ops += nops
-                ops += uops
-                stack += 1
-                n += 2
-                peak_stack = max(peak_stack, stack + 2)
-            elif needs_build_string:
-                ops = self.build_op("LOAD_CONST", self.get_const(self.JOIN_EMPTY)) + ops
-                peak_stack += 1
-            if not dont_build_string:
-                ops += self.build_string(n)
-            return peak_stack, ops
-
-        def compile(self, append_unknown=True):
-            flags = 0x08
-            dom_ops = []
-            url_ops = []
-            opl = dom_ops
-            if append_unknown:
-                encode_query_vars = partial(
-                    url_encode,
-                    charset=self.rule.map.charset,
-                    sort=self.rule.map.sort_parameters,
-                    key=self.rule.map.sort_key,
+        opl = dom_ops
+        for is_dynamic, data in self._trace:
+            if data == "|" and opl is dom_ops:
+                opl = url_ops
+                continue
+            # this seems like a silly case to ever come up but:
+            # if a default is given for a value that appears in the rule,
+            # resolve it to a constant ahead of time
+            if is_dynamic and data in defaults:
+                data = self._converters[data].to_url(defaults[data])
+                opl.append((False, data))
+            elif not is_dynamic:
+                opl.append(
+                    (False, url_quote(to_bytes(data, self.map.charset), safe="/:|+"))
                 )
-            for is_dynamic, data in self.rule._trace:
-                if data == "|" and opl is dom_ops:
-                    opl = url_ops
-                    continue
-                # this seems like a silly case to ever come up but:
-                # if a default is given for a value that appears in the rule,
-                # resolve it to a constant ahead of time
-                if is_dynamic and data in self.defaults:
-                    data = self.rule._converters[data].to_url(self.defaults[data])
-                    opl.append((None, data))
-                    continue
-                elif not is_dynamic:
-                    opl.append(
-                        (
-                            None,
-                            url_quote(
-                                to_bytes(data, self.rule.map.charset), safe="/:|+"
-                            ),
-                        )
-                    )
-                    continue
-                opl.append((self.rule._converters[data].to_url, data))
-            dom_ops = self.collapse_constants(dom_ops)
-            url_ops = self.collapse_constants(url_ops)
-            for op, elem in dom_ops + url_ops:
-                if op is not None:
-                    self.get_var(elem)
-            self.add_defaults()
-            argcount = len(self.var)
-            # invalid name for paranoia reasons
-            self.get_var(".keyword_arguments")
-            stack = 0
-            peak_stack = 0
-            ops = b""
-            if (
-                not append_unknown
-                and len(dom_ops) == len(url_ops) == 1
-                and dom_ops[0][0] is url_ops[0][0] is None
-            ):
-                # shortcut: just return the constant
-                stack = peak_stack = 1
-                constant_value = (dom_ops[0][1], url_ops[0][1])
-                ops += self.build_op("LOAD_CONST", self.get_const(constant_value))
             else:
-                ps, rv = self.emit_build(len(ops), dom_ops)
-                ops += rv
-                peak_stack = max(stack + ps, peak_stack)
-                stack += 1
-                if append_unknown:
-                    ps, rv = self.emit_build(
-                        len(ops), url_ops, append_unknown, encode_query_vars, argcount
-                    )
-                else:
-                    ps, rv = self.emit_build(len(ops), url_ops)
-                ops += rv
-                peak_stack = max(stack + ps, peak_stack)
-                ops += self.build_op("BUILD_TUPLE", 2)
-            ops += self.build_op("RETURN_VALUE")
-            code_args = [  # CodeType doesn't take keywords
-                argcount,  # argcount
-                len(self.var),  # nlocals
-                peak_stack + len(self.var),  # stacksize
-                flags,  # flags
-                bytes(ops),  # codestring
-                tuple(self.consts),  # constants
-                (),  # names
-                tuple(self.var),  # varnames
-                "<werkzeug routing>",  # filename, coverage ignores "<"
-                "<builder:%r>" % self.rule.rule,  # name
-                1,  # firstlineno
-                b"",  # lnotab
+                opl.append((True, data))
+
+        def _convert(elem):
+            ret = _prefix_names(_CALL_CONVERTER_CODE_FMT.format(elem=elem))
+            ret.args = [ast.Name(str(elem), ast.Load())]  # str for py2
+            return ret
+
+        def _parts(ops):
+            parts = [
+                _convert(elem) if is_dynamic else ast.Str(s=elem)
+                for is_dynamic, elem in ops
             ]
-            if not PY2:
-                code_args[1:1] = [0]  # kwonlyargcount
-            co = types.CodeType(*code_args)
-            fn = types.FunctionType(co, {}, None, self.argdefs)
-            return fn
+            parts = parts or [ast.Str("")]
+            # constant fold
+            ret = [parts[0]]
+            for p in parts[1:]:
+                if isinstance(p, ast.Str) and isinstance(ret[-1], ast.Str):
+                    ret[-1] = ast.Str(ret[-1].s + p.s)
+                else:
+                    ret.append(p)
+            return ret
 
-    def _compile_builder(self, append_unknown=True):
-        """Generate a function that builds this rule.
+        dom_parts = _parts(dom_ops)
+        url_parts = _parts(url_ops)
+        if not append_unknown:
+            body = []
+        else:
+            body = [_IF_KWARGS_URL_ENCODE_AST]
+            url_parts.extend(_URL_ENCODE_AST_NAMES)
+
+        def _join(parts):
+            if len(parts) == 1:  # shortcut
+                return parts[0]
+            elif hasattr(ast, "JoinedStr"):  # py36+
+                return ast.JoinedStr(parts)
+            else:
+                call = _prefix_names('"".join()')
+                call.args = [ast.Tuple(parts, ast.Load())]
+                return call
 
-        :internal:
-        """
-        return self.BuilderCompiler(self).compile(append_unknown)
+        body.append(
+            ast.Return(ast.Tuple([_join(dom_parts), _join(url_parts)], ast.Load()))
+        )
+
+        # str is necessary for python2
+        pargs = [
+            str(elem)
+            for is_dynamic, elem in dom_ops + url_ops
+            if is_dynamic and elem not in defaults
+        ]
+        kargs = [str(k) for k in defaults]
+
+        func_ast = _prefix_names("def _(): pass")
+        func_ast.name = "<builder:{!r}>".format(self.rule)
+        if hasattr(ast, "arg"):  # py3
+            func_ast.args.args.append(ast.arg(".self", None))
+            for arg in pargs + kargs:
+                func_ast.args.args.append(ast.arg(arg, None))
+            func_ast.args.kwarg = ast.arg(".kwargs", None)
+        else:
+            func_ast.args.args.append(ast.Name(".self", ast.Load()))
+            for arg in pargs + kargs:
+                func_ast.args.args.append(ast.Name(arg, ast.Load()))
+            func_ast.args.kwarg = ".kwargs"
+        for _ in kargs:
+            func_ast.args.defaults.append(ast.Str(""))
+        func_ast.body = body
+
+        module = ast.fix_missing_locations(ast.Module([func_ast]))
+        code = compile(module, "<werkzeug routing>", "exec")
+
+        globs, locs = {}, {}
+        exec(code, globs, locs)
+
+        return locs[func_ast.name]
 
     def build(self, values, append_unknown=True):
         """Assembles the relative url for that rule and the subdomain.


Test patch:
diff --git a/tests/test_routing.py b/tests/test_routing.py
--- a/tests/test_routing.py
+++ b/tests/test_routing.py
@@ -8,6 +8,7 @@
     :copyright: 2007 Pallets
     :license: BSD-3-Clause
 """
+import gc
 import uuid
 
 import pytest
@@ -1069,3 +1070,36 @@ def test_error_message_suggestion():
     with pytest.raises(r.BuildError) as excinfo:
         adapter.build("world", {"id": 2}, method="POST")
     assert "Did you mean to use methods ['GET', 'HEAD']?" in str(excinfo.value)
+
+
+def test_no_memory_leak_from_Rule_builder():
+    """See #1520"""
+
+    # generate a bunch of objects that *should* get collected
+    for _ in range(100):
+        r.Map([r.Rule("/a/<string:b>")])
+
+    # ensure that the garbage collection has had a chance to collect cyclic
+    # objects
+    for _ in range(5):
+        gc.collect()
+
+    # assert they got collected!
+    count = sum(1 for obj in gc.get_objects() if isinstance(obj, r.Rule))
+    assert count == 0
+
+
+def test_build_url_with_arg_self():
+    map = r.Map([r.Rule("/foo/<string:self>", endpoint="foo")])
+    adapter = map.bind("example.org", "/", subdomain="blah")
+
+    ret = adapter.build("foo", {"self": "bar"})
+    assert ret == "http://example.org/foo/bar"
+
+
+def test_build_url_with_arg_keyword():
+    map = r.Map([r.Rule("/foo/<string:class>", endpoint="foo")])
+    adapter = map.bind("example.org", "/", subdomain="blah")
+
+    ret = adapter.build("foo", {"class": "bar"})
+    assert ret == "http://example.org/foo/bar"