# Buggy code
def state(
    name,
    tgt,
    ssh=False,
    tgt_type=None,
    expr_form=None,
    ret="",
    highstate=None,
    sls=None,
    top=None,
    env=None,
    test=False,
    pillar=None,
    expect_minions=False,
    fail_minions=None,
    allow_fail=0,
    concurrent=False,
    timeout=None,
):
    """
    Invoke a state run on a given target

    name
        An arbitrary name used to track the state execution

    tgt
        The target specification for the state run.

    tgt_type | expr_form
        The target type to resolve, defaults to glob

    ret
        Optionally set a single or a list of returners to use

    highstate
        Defaults to None, if set to True the target systems will ignore any
        sls references specified in the sls option and call state.highstate
        on the targeted minions

    top
        Should be the name of a top file. If set state.top is called with this
        top file instead of state.sls.

    sls
        A group of sls files to execute. This can be defined as a single string
        containing a single sls file, or a list of sls files

    test
        Pass ``test=true`` through to the state function

    pillar
        Pass the ``pillar`` kwarg through to the state function

    saltenv
        The default salt environment to pull sls files from

    ssh
        Set to `True` to use the ssh client instead of the standard salt client

    roster
        In the event of using salt-ssh, a roster system can be set

    expect_minions
        An optional boolean for failing if some minions do not respond

    fail_minions
        An optional list of targeted minions where failure is an option

    allow_fail
        Pass in the number of minions to allow for failure before setting
        the result of the execution to False

    concurrent
        Allow multiple state runs to occur at once.

        WARNING: This flag is potentially dangerous. It is designed
        for use when multiple state runs can safely be run at the same
        Do not use this flag for performance optimization.

    Examples:

    Run a list of sls files via :py:func:`state.sls <salt.state.sls>` on target
    minions:

    .. code-block:: yaml

        webservers:
          salt.state:
            - tgt: 'web*'
            - sls:
              - apache
              - django
              - core
            - saltenv: prod

    Run a full :py:func:`state.highstate <salt.state.highstate>` on target
    mininons.

    .. code-block:: yaml

        databases:
          salt.state:
            - tgt: role:database
            - tgt_type: grain
            - highstate: True
    """
    cmd_kw = {"arg": [], "kwarg": {}, "ret": ret, "timeout": timeout}

    ret = {"name": name, "changes": {}, "comment": "", "result": True}

    try:
        allow_fail = int(allow_fail)
    except ValueError:
        ret["result"] = False
        ret["comment"] = "Passed invalid value for 'allow_fail', must be an int"
        return ret

    if env is not None:
        msg = (
            "Passing a salt environment should be done using 'saltenv' not "
            "'env'. This warning will go away in Salt Boron and this "
            "will be the default and expected behavior. Please update your "
            "state files."
        )
        salt.utils.warn_until("Boron", msg)
        ret.setdefault("warnings", []).append(msg)
        # No need to set __env__ = env since that's done in the state machinery

    if expr_form and tgt_type:
        ret.setdefault("warnings", []).append(
            "Please only use 'tgt_type' or 'expr_form' not both. "
            "Preferring 'tgt_type' over 'expr_form'"
        )
        expr_form = None
    elif expr_form and not tgt_type:
        tgt_type = expr_form
    elif not tgt_type and not expr_form:
        tgt_type = "glob"

    cmd_kw["expr_form"] = tgt_type
    cmd_kw["ssh"] = ssh
    cmd_kw["expect_minions"] = expect_minions
    if highstate:
        fun = "state.highstate"
    elif top:
        fun = "state.top"
        cmd_kw["arg"].append(top)
    elif sls:
        fun = "state.sls"
        if isinstance(sls, list):
            sls = ",".join(sls)
        cmd_kw["arg"].append(sls)
    else:
        ret["comment"] = "No highstate or sls specified, no execution made"
        ret["result"] = False
        return ret

    if test:
        cmd_kw["kwarg"]["test"] = test

    if pillar:
        cmd_kw["kwarg"]["pillar"] = pillar

    cmd_kw["kwarg"]["saltenv"] = __env__

    if isinstance(concurrent, bool):
        cmd_kw["kwarg"]["concurrent"] = concurrent
    else:
        ret["comment"] = "Must pass in boolean for value of 'concurrent'"
        ret["result"] = False
        return ret

    if __opts__["test"] is True:
        ret["comment"] = ("{0} will be run on target {1} as test={2}").format(
            fun == "state.highstate"
            and "Highstate"
            or "States " + ",".join(cmd_kw["arg"]),
            tgt,
            str(test),
        )
        ret["result"] = None
        return ret
    cmd_ret = __salt__["saltutil.cmd"](tgt, fun, **cmd_kw)

    changes = {}
    fail = set()
    failures = {}
    no_change = set()

    if fail_minions is None:
        fail_minions = ()
    elif isinstance(fail_minions, string_types):
        fail_minions = [minion.strip() for minion in fail_minions.split(",")]
    elif not isinstance(fail_minions, list):
        ret.setdefault("warnings", []).append(
            "'fail_minions' needs to be a list or a comma separated string. Ignored."
        )
        fail_minions = ()

    for minion, mdata in six.iteritems(cmd_ret):
        if mdata.get("out", "") != "highstate":
            log.warning("Output from salt state not highstate")

        m_ret = False

        if "return" in mdata and "ret" not in mdata:
            mdata["ret"] = mdata.pop("return")

        if mdata.get("failed", False):
            m_state = False
        else:
            m_ret = mdata["ret"]
            m_state = salt.utils.check_state_result(m_ret)

        if not m_state:
            if minion not in fail_minions:
                fail.add(minion)
            failures[minion] = m_ret and m_ret or "Minion did not respond"
            continue
        for state_item in six.itervalues(m_ret):
            if state_item["changes"]:
                changes[minion] = m_ret
                break
        else:
            no_change.add(minion)

    if changes:
        ret["changes"] = {"out": "highstate", "ret": changes}
    if len(fail) > allow_fail:
        ret["result"] = False
        ret["comment"] = "Run failed on minions: {0}".format(", ".join(fail))
    else:
        ret["comment"] = "States ran successfully."
        if changes:
            ret["comment"] += " Updating {0}.".format(", ".join(changes))
        if no_change:
            ret["comment"] += " No changes made to {0}.".format(", ".join(no_change))
    if failures:
        ret["comment"] += "\nFailures:\n"
        for minion, failure in six.iteritems(failures):
            ret["comment"] += "\n".join(
                (" " * 4 + l)
                for l in salt.output.out_format(
                    {minion: failure},
                    "highstate",
                    __opts__,
                ).splitlines()
            )
            ret["comment"] += "\n"
    return ret


# Traceback (KeyError)
[2015-10-15 06:58:04,096] UID: 0 WARNING: Output from salt state not highstate
[2015-10-15 06:58:04,098] UID: 0 ERROR: An exception occurred in this state: Traceback (most recent call last):
File "/usr/lib/python2.6/site-packages/salt/state.py", line 1560, in call
**cdata['kwargs'])
File "/usr/lib64/python2.6/contextlib.py", line 34, in __exit__
self.gen.throw(type, value, traceback)
File "/usr/lib/python2.6/site-packages/salt/utils/context.py", line 42, in func_globals_inject
yield
File "/usr/lib/python2.6/site-packages/salt/state.py", line 1560, in call
**cdata['kwargs'])
File "/usr/lib/python2.6/site-packages/salt/states/saltmod.py", line 254, in state
m_ret = mdata['ret']
KeyError: 'ret'

# Fixed code
def state(
    name,
    tgt,
    ssh=False,
    tgt_type=None,
    expr_form=None,
    ret="",
    highstate=None,
    sls=None,
    top=None,
    env=None,
    test=False,
    pillar=None,
    expect_minions=False,
    fail_minions=None,
    allow_fail=0,
    concurrent=False,
    timeout=None,
):
    """
    Invoke a state run on a given target

    name
        An arbitrary name used to track the state execution

    tgt
        The target specification for the state run.

    tgt_type | expr_form
        The target type to resolve, defaults to glob

    ret
        Optionally set a single or a list of returners to use

    highstate
        Defaults to None, if set to True the target systems will ignore any
        sls references specified in the sls option and call state.highstate
        on the targeted minions

    top
        Should be the name of a top file. If set state.top is called with this
        top file instead of state.sls.

    sls
        A group of sls files to execute. This can be defined as a single string
        containing a single sls file, or a list of sls files

    test
        Pass ``test=true`` through to the state function

    pillar
        Pass the ``pillar`` kwarg through to the state function

    saltenv
        The default salt environment to pull sls files from

    ssh
        Set to `True` to use the ssh client instead of the standard salt client

    roster
        In the event of using salt-ssh, a roster system can be set

    expect_minions
        An optional boolean for failing if some minions do not respond

    fail_minions
        An optional list of targeted minions where failure is an option

    allow_fail
        Pass in the number of minions to allow for failure before setting
        the result of the execution to False

    concurrent
        Allow multiple state runs to occur at once.

        WARNING: This flag is potentially dangerous. It is designed
        for use when multiple state runs can safely be run at the same
        Do not use this flag for performance optimization.

    Examples:

    Run a list of sls files via :py:func:`state.sls <salt.state.sls>` on target
    minions:

    .. code-block:: yaml

        webservers:
          salt.state:
            - tgt: 'web*'
            - sls:
              - apache
              - django
              - core
            - saltenv: prod

    Run a full :py:func:`state.highstate <salt.state.highstate>` on target
    mininons.

    .. code-block:: yaml

        databases:
          salt.state:
            - tgt: role:database
            - tgt_type: grain
            - highstate: True
    """
    cmd_kw = {"arg": [], "kwarg": {}, "ret": ret, "timeout": timeout}

    ret = {"name": name, "changes": {}, "comment": "", "result": True}

    try:
        allow_fail = int(allow_fail)
    except ValueError:
        ret["result"] = False
        ret["comment"] = "Passed invalid value for 'allow_fail', must be an int"
        return ret

    if env is not None:
        msg = (
            "Passing a salt environment should be done using 'saltenv' not "
            "'env'. This warning will go away in Salt Boron and this "
            "will be the default and expected behavior. Please update your "
            "state files."
        )
        salt.utils.warn_until("Boron", msg)
        ret.setdefault("warnings", []).append(msg)
        # No need to set __env__ = env since that's done in the state machinery

    if expr_form and tgt_type:
        ret.setdefault("warnings", []).append(
            "Please only use 'tgt_type' or 'expr_form' not both. "
            "Preferring 'tgt_type' over 'expr_form'"
        )
        expr_form = None
    elif expr_form and not tgt_type:
        tgt_type = expr_form
    elif not tgt_type and not expr_form:
        tgt_type = "glob"

    cmd_kw["expr_form"] = tgt_type
    cmd_kw["ssh"] = ssh
    cmd_kw["expect_minions"] = expect_minions
    if highstate:
        fun = "state.highstate"
    elif top:
        fun = "state.top"
        cmd_kw["arg"].append(top)
    elif sls:
        fun = "state.sls"
        if isinstance(sls, list):
            sls = ",".join(sls)
        cmd_kw["arg"].append(sls)
    else:
        ret["comment"] = "No highstate or sls specified, no execution made"
        ret["result"] = False
        return ret

    if test:
        cmd_kw["kwarg"]["test"] = test

    if pillar:
        cmd_kw["kwarg"]["pillar"] = pillar

    cmd_kw["kwarg"]["saltenv"] = __env__

    if isinstance(concurrent, bool):
        cmd_kw["kwarg"]["concurrent"] = concurrent
    else:
        ret["comment"] = "Must pass in boolean for value of 'concurrent'"
        ret["result"] = False
        return ret

    if __opts__["test"] is True:
        ret["comment"] = ("{0} will be run on target {1} as test={2}").format(
            fun == "state.highstate"
            and "Highstate"
            or "States " + ",".join(cmd_kw["arg"]),
            tgt,
            str(test),
        )
        ret["result"] = None
        return ret
    cmd_ret = __salt__["saltutil.cmd"](tgt, fun, **cmd_kw)

    changes = {}
    fail = set()
    failures = {}
    no_change = set()

    if fail_minions is None:
        fail_minions = ()
    elif isinstance(fail_minions, string_types):
        fail_minions = [minion.strip() for minion in fail_minions.split(",")]
    elif not isinstance(fail_minions, list):
        ret.setdefault("warnings", []).append(
            "'fail_minions' needs to be a list or a comma separated string. Ignored."
        )
        fail_minions = ()

    for minion, mdata in six.iteritems(cmd_ret):
        if mdata.get("out", "") != "highstate":
            log.warning("Output from salt state not highstate")

        m_ret = False

        if "return" in mdata and "ret" not in mdata:
            mdata["ret"] = mdata.pop("return")

        if mdata.get("failed", False):
            m_state = False
        else:
            try:
                m_ret = mdata["ret"]
            except KeyError:
                m_state = False
            if not m_state:
                m_state = salt.utils.check_state_result(m_ret)

        if not m_state:
            if minion not in fail_minions:
                fail.add(minion)
            failures[minion] = m_ret and m_ret or "Minion did not respond"
            continue
        for state_item in six.itervalues(m_ret):
            if state_item["changes"]:
                changes[minion] = m_ret
                break
        else:
            no_change.add(minion)

    if changes:
        ret["changes"] = {"out": "highstate", "ret": changes}
    if len(fail) > allow_fail:
        ret["result"] = False
        ret["comment"] = "Run failed on minions: {0}".format(", ".join(fail))
    else:
        ret["comment"] = "States ran successfully."
        if changes:
            ret["comment"] += " Updating {0}.".format(", ".join(changes))
        if no_change:
            ret["comment"] += " No changes made to {0}.".format(", ".join(no_change))
    if failures:
        ret["comment"] += "\nFailures:\n"
        for minion, failure in six.iteritems(failures):
            ret["comment"] += "\n".join(
                (" " * 4 + l)
                for l in salt.output.out_format(
                    {minion: failure},
                    "highstate",
                    __opts__,
                ).splitlines()
            )
            ret["comment"] += "\n"
    return ret