Commit message: netfilter: nf_tables_offload: incorrect flow offload action array size

immediate verdict expression needs to allocate one slot in the flow offload
action array, however, immediate data expression does not need to do so.

fwd and dup expression need to allocate one slot, this is missing.

Add a new offload_action interface to report if this expression needs to
allocate one slot in the flow offload action array.

Fixes: be2861dc36d7 ("netfilter: nft_{fwd,dup}_netdev: add offload support")
Reported-and-tested-by: Nick Gregory <Nick.Gregory@Sophos.com>
Signed-off-by: Pablo Neira Ayuso <pablo@netfilter.org>
CVE: CVE-2022-25636
CWE: CWE-269
Vulnerability description: net/netfilter/nf_dup_netdev.c in the Linux kernel 5.4 through 5.6.10 allows local users to gain privileges because of a heap out-of-bounds write. This is related to nf_tables_offload.

Project: nf
File: nf_tables_offload.c

int nft_chain_offload_priority(struct nft_base_chain *basechain)
{
	if (basechain->ops.priority <= 0 ||
	    basechain->ops.priority > USHRT_MAX)
		return -1;

	return 0;
}

Project: nf
File: nft_fwd_netdev.c

static int nft_fwd_neigh_dump(struct sk_buff *skb, const struct nft_expr *expr)
{
	struct nft_fwd_neigh *priv = nft_expr_priv(expr);

	if (nft_dump_register(skb, NFTA_FWD_SREG_DEV, priv->sreg_dev) ||
	    nft_dump_register(skb, NFTA_FWD_SREG_ADDR, priv->sreg_addr) ||
	    nla_put_be32(skb, NFTA_FWD_NFPROTO, htonl(priv->nfproto)))
		goto nla_put_failure;

	return 0;

nla_put_failure:
	return -1;
}

Project: nf
File: nft_dup_netdev.c

static int nft_dup_netdev_dump(struct sk_buff *skb, const struct nft_expr *expr)
{
	struct nft_dup_netdev *priv = nft_expr_priv(expr);

	if (nft_dump_register(skb, NFTA_DUP_SREG_DEV, priv->sreg_dev))
		goto nla_put_failure;

	return 0;

nla_put_failure:
	return -1;
}

Project: nf
File: nft_fwd_netdev.c

static int nft_fwd_netdev_init(const struct nft_ctx *ctx,
			       const struct nft_expr *expr,
			       const struct nlattr * const tb[])
{
	struct nft_fwd_netdev *priv = nft_expr_priv(expr);

	if (tb[NFTA_FWD_SREG_DEV] == NULL)
		return -EINVAL;

	return nft_parse_register_load(tb[NFTA_FWD_SREG_DEV], &priv->sreg_dev,
				       sizeof(int));
}

Project: nf
File: nf_tables_offload.c

void nft_offload_exit(void)
{
	unregister_netdevice_notifier(&nft_offload_netdev_notifier);
}

Project: nf
File: nf_tables_offload.c

static int nft_flow_offload_rule(const struct nft_chain *chain,
				 struct nft_rule *rule,
				 struct nft_flow_rule *flow,
				 enum flow_cls_command command)
{
	struct flow_cls_offload cls_flow;

	return nft_flow_offload_cmd(chain, rule, flow, command, &cls_flow);
}

Project: nf
File: nf_tables_offload.c

static int nft_flow_offload_bind(struct flow_block_offload *bo,
				 struct nft_base_chain *basechain)
{
	list_splice(&bo->cb_list, &basechain->flow_block.cb_list);
	return 0;
}

Project: nf
File: nft_dup_netdev.c

static bool nft_dup_netdev_offload_action(const struct nft_expr *expr)
{
	return true;
}

Project: nf
File: nft_fwd_netdev.c

static int nft_fwd_netdev_dump(struct sk_buff *skb, const struct nft_expr *expr)
{
	struct nft_fwd_netdev *priv = nft_expr_priv(expr);

	if (nft_dump_register(skb, NFTA_FWD_SREG_DEV, priv->sreg_dev))
		goto nla_put_failure;

	return 0;

nla_put_failure:
	return -1;
}

Project: nf
File: nf_tables_offload.c

struct nft_flow_rule *nft_flow_rule_create(struct net *net,
					   const struct nft_rule *rule)
{
	struct nft_offload_ctx *ctx;
	struct nft_flow_rule *flow;
	int num_actions = 0, err;
	struct nft_expr *expr;

	expr = nft_expr_first(rule);
	while (nft_expr_more(rule, expr)) {
		if (expr->ops->offload_action &&
		    expr->ops->offload_action(expr))
			num_actions++;

		expr = nft_expr_next(expr);
	}

	if (num_actions == 0)
		return ERR_PTR(-EOPNOTSUPP);

	flow = nft_flow_rule_alloc(num_actions);
	if (!flow)
		return ERR_PTR(-ENOMEM);

	expr = nft_expr_first(rule);

	ctx = kzalloc(sizeof(struct nft_offload_ctx), GFP_KERNEL);
	if (!ctx) {
		err = -ENOMEM;
		goto err_out;
	}
	ctx->net = net;
	ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;

	while (nft_expr_more(rule, expr)) {
		if (!expr->ops->offload) {
			err = -EOPNOTSUPP;
			goto err_out;
		}
		err = expr->ops->offload(ctx, flow, expr);
		if (err < 0)
			goto err_out;

		expr = nft_expr_next(expr);
	}
	nft_flow_rule_transfer_vlan(ctx, flow);

	flow->proto = ctx->dep.l3num;
	kfree(ctx);

	return flow;
err_out:
	kfree(ctx);
	nft_flow_rule_destroy(flow);

	return ERR_PTR(err);
}

Project: nf
File: nft_fwd_netdev.c

nft_fwd_select_ops(const struct nft_ctx *ctx,
		   const struct nlattr * const tb[])
{
	if (tb[NFTA_FWD_SREG_ADDR])
		return &nft_fwd_neigh_netdev_ops;
	if (tb[NFTA_FWD_SREG_DEV])
		return &nft_fwd_netdev_ops;

        return ERR_PTR(-EOPNOTSUPP);
}

Project: nf
File: nft_immediate.c

static void nft_immediate_destroy(const struct nft_ctx *ctx,
				  const struct nft_expr *expr)
{
	const struct nft_immediate_expr *priv = nft_expr_priv(expr);
	const struct nft_data *data = &priv->data;
	struct nft_rule *rule, *n;
	struct nft_ctx chain_ctx;
	struct nft_chain *chain;

	if (priv->dreg != NFT_REG_VERDICT)
		return;

	switch (data->verdict.code) {
	case NFT_JUMP:
	case NFT_GOTO:
		chain = data->verdict.chain;

		if (!nft_chain_is_bound(chain))
			break;

		chain_ctx = *ctx;
		chain_ctx.chain = chain;

		list_for_each_entry_safe(rule, n, &chain->rules, list)
			nf_tables_rule_release(&chain_ctx, rule);

		nf_tables_chain_destroy(&chain_ctx);
		break;
	default:
		break;
	}
}

Project: nf
File: nf_tables_offload.c

static void nft_flow_cls_offload_setup(struct flow_cls_offload *cls_flow,
				       const struct nft_base_chain *basechain,
				       const struct nft_rule *rule,
				       const struct nft_flow_rule *flow,
				       struct netlink_ext_ack *extack,
				       enum flow_cls_command command)
{
	__be16 proto = ETH_P_ALL;

	memset(cls_flow, 0, sizeof(*cls_flow));

	if (flow)
		proto = flow->proto;

	nft_flow_offload_common_init(&cls_flow->common, proto,
				     basechain->ops.priority, extack);
	cls_flow->command = command;
	cls_flow->cookie = (unsigned long) rule;
	if (flow)
		cls_flow->rule = flow->rule;
}

Project: nf
File: nf_tables_offload.c

static int nft_flow_offload_cmd(const struct nft_chain *chain,
				const struct nft_rule *rule,
				struct nft_flow_rule *flow,
				enum flow_cls_command command,
				struct flow_cls_offload *cls_flow)
{
	struct netlink_ext_ack extack = {};
	struct nft_base_chain *basechain;

	if (!nft_is_base_chain(chain))
		return -EOPNOTSUPP;

	basechain = nft_base_chain(chain);
	nft_flow_cls_offload_setup(cls_flow, basechain, rule, flow, &extack,
				   command);

	return nft_setup_cb_call(TC_SETUP_CLSFLOWER, cls_flow,
				 &basechain->flow_block.cb_list);
}

Project: nf
File: nft_dup_netdev.c

static int __init nft_dup_netdev_module_init(void)
{
	return nft_register_expr(&nft_dup_netdev_type);
}

Project: nf
File: nf_tables_offload.c

int nft_flow_rule_offload_commit(struct net *net)
{
	struct nftables_pernet *nft_net = nft_pernet(net);
	struct nft_trans *trans;
	int err = 0;
	u8 policy;

	list_for_each_entry(trans, &nft_net->commit_list, list) {
		if (trans->ctx.family != NFPROTO_NETDEV)
			continue;

		switch (trans->msg_type) {
		case NFT_MSG_NEWCHAIN:
			if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD) ||
			    nft_trans_chain_update(trans))
				continue;

			policy = nft_trans_chain_policy(trans);
			err = nft_flow_offload_chain(trans->ctx.chain, &policy,
						     FLOW_BLOCK_BIND);
			break;
		case NFT_MSG_DELCHAIN:
			if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
				continue;

			policy = nft_trans_chain_policy(trans);
			err = nft_flow_offload_chain(trans->ctx.chain, &policy,
						     FLOW_BLOCK_UNBIND);
			break;
		case NFT_MSG_NEWRULE:
			if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
				continue;

			if (trans->ctx.flags & NLM_F_REPLACE ||
			    !(trans->ctx.flags & NLM_F_APPEND)) {
				err = -EOPNOTSUPP;
				break;
			}
			err = nft_flow_offload_rule(trans->ctx.chain,
						    nft_trans_rule(trans),
						    nft_trans_flow_rule(trans),
						    FLOW_CLS_REPLACE);
			break;
		case NFT_MSG_DELRULE:
			if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
				continue;

			err = nft_flow_offload_rule(trans->ctx.chain,
						    nft_trans_rule(trans),
						    NULL, FLOW_CLS_DESTROY);
			break;
		}

		if (err) {
			nft_flow_rule_offload_abort(net, trans);
			break;
		}
	}

	return err;
}

Project: nf
File: nft_dup_netdev.c

static int nft_dup_netdev_offload(struct nft_offload_ctx *ctx,
				  struct nft_flow_rule *flow,
				  const struct nft_expr *expr)
{
	const struct nft_dup_netdev *priv = nft_expr_priv(expr);
	int oif = ctx->regs[priv->sreg_dev].data.data[0];

	return nft_fwd_dup_netdev_offload(ctx, flow, FLOW_ACTION_MIRRED, oif);
}

Project: nf
File: nft_fwd_netdev.c

static int nft_fwd_neigh_init(const struct nft_ctx *ctx,
			      const struct nft_expr *expr,
			      const struct nlattr * const tb[])
{
	struct nft_fwd_neigh *priv = nft_expr_priv(expr);
	unsigned int addr_len;
	int err;

	if (!tb[NFTA_FWD_SREG_DEV] ||
	    !tb[NFTA_FWD_SREG_ADDR] ||
	    !tb[NFTA_FWD_NFPROTO])
		return -EINVAL;

	priv->nfproto = ntohl(nla_get_be32(tb[NFTA_FWD_NFPROTO]));

	switch (priv->nfproto) {
	case NFPROTO_IPV4:
		addr_len = sizeof(struct in_addr);
		break;
	case NFPROTO_IPV6:
		addr_len = sizeof(struct in6_addr);
		break;
	default:
		return -EOPNOTSUPP;
	}

	err = nft_parse_register_load(tb[NFTA_FWD_SREG_DEV], &priv->sreg_dev,
				      sizeof(int));
	if (err < 0)
		return err;

	return nft_parse_register_load(tb[NFTA_FWD_SREG_ADDR], &priv->sreg_addr,
				       addr_len);
}

Project: nf
File: nft_immediate.c

static int nft_immediate_offload(struct nft_offload_ctx *ctx,
				 struct nft_flow_rule *flow,
				 const struct nft_expr *expr)
{
	const struct nft_immediate_expr *priv = nft_expr_priv(expr);

	if (priv->dreg == NFT_REG_VERDICT)
		return nft_immediate_offload_verdict(ctx, flow, priv);

	memcpy(&ctx->regs[priv->dreg].data, &priv->data, sizeof(priv->data));

	return 0;
}

Project: nf
File: nf_tables_offload.c

static struct nft_chain *__nft_offload_get_chain(const struct nftables_pernet *nft_net,
						 struct net_device *dev)
{
	struct nft_base_chain *basechain;
	struct nft_hook *hook, *found;
	const struct nft_table *table;
	struct nft_chain *chain;

	list_for_each_entry(table, &nft_net->tables, list) {
		if (table->family != NFPROTO_NETDEV)
			continue;

		list_for_each_entry(chain, &table->chains, list) {
			if (!nft_is_base_chain(chain) ||
			    !(chain->flags & NFT_CHAIN_HW_OFFLOAD))
				continue;

			found = NULL;
			basechain = nft_base_chain(chain);
			list_for_each_entry(hook, &basechain->hook_list, list) {
				if (hook->ops.dev != dev)
					continue;

				found = hook;
				break;
			}
			if (!found)
				continue;

			return chain;
		}
	}

	return NULL;
}

Project: nf
File: nft_immediate.c

static int nft_immediate_offload_verdict(struct nft_offload_ctx *ctx,
					 struct nft_flow_rule *flow,
					 const struct nft_immediate_expr *priv)
{
	struct flow_action_entry *entry;
	const struct nft_data *data;

	entry = &flow->rule->action.entries[ctx->num_actions++];

	data = &priv->data;
	switch (data->verdict.code) {
	case NF_ACCEPT:
		entry->id = FLOW_ACTION_ACCEPT;
		break;
	case NF_DROP:
		entry->id = FLOW_ACTION_DROP;
		break;
	default:
		return -EOPNOTSUPP;
	}

	return 0;
}

Project: nf
File: nft_fwd_netdev.c

static int nft_fwd_validate(const struct nft_ctx *ctx,
			    const struct nft_expr *expr,
			    const struct nft_data **data)
{
	return nft_chain_validate_hooks(ctx->chain, (1 << NF_NETDEV_INGRESS) |
						    (1 << NF_NETDEV_EGRESS));
}

Project: nf
File: nft_dup_netdev.c

static int nft_dup_netdev_init(const struct nft_ctx *ctx,
			       const struct nft_expr *expr,
			       const struct nlattr * const tb[])
{
	struct nft_dup_netdev *priv = nft_expr_priv(expr);

	if (tb[NFTA_DUP_SREG_DEV] == NULL)
		return -EINVAL;

	return nft_parse_register_load(tb[NFTA_DUP_SREG_DEV], &priv->sreg_dev,
				       sizeof(int));
}

Project: nf
File: nf_tables_offload.c

static struct nft_flow_rule *nft_flow_rule_alloc(int num_actions)
{
	struct nft_flow_rule *flow;

	flow = kzalloc(sizeof(struct nft_flow_rule), GFP_KERNEL);
	if (!flow)
		return NULL;

	flow->rule = flow_rule_alloc(num_actions);
	if (!flow->rule) {
		kfree(flow);
		return NULL;
	}

	flow->rule->match.dissector	= &flow->match.dissector;
	flow->rule->match.mask		= &flow->match.mask;
	flow->rule->match.key		= &flow->match.key;

	return flow;
}

Project: nf
File: nf_tables_offload.c

static int nft_flow_offload_unbind(struct flow_block_offload *bo,
				   struct nft_base_chain *basechain)
{
	struct flow_block_cb *block_cb, *next;
	struct flow_cls_offload cls_flow;
	struct netlink_ext_ack extack;
	struct nft_chain *chain;
	struct nft_rule *rule;

	chain = &basechain->chain;
	list_for_each_entry(rule, &chain->rules, list) {
		memset(&extack, 0, sizeof(extack));
		nft_flow_cls_offload_setup(&cls_flow, basechain, rule, NULL,
					   &extack, FLOW_CLS_DESTROY);
		nft_setup_cb_call(TC_SETUP_CLSFLOWER, &cls_flow, &bo->cb_list);
	}

	list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
		list_del(&block_cb->list);
		flow_block_cb_free(block_cb);
	}

	return 0;
}

Project: nf
File: nf_tables_offload.c

static int nft_flow_block_chain(struct nft_base_chain *basechain,
				const struct net_device *this_dev,
				enum flow_block_command cmd)
{
	struct net_device *dev;
	struct nft_hook *hook;
	int err, i = 0;

	list_for_each_entry(hook, &basechain->hook_list, list) {
		dev = hook->ops.dev;
		if (this_dev && this_dev != dev)
			continue;

		err = nft_chain_offload_cmd(basechain, dev, cmd);
		if (err < 0 && cmd == FLOW_BLOCK_BIND) {
			if (!this_dev)
				goto err_flow_block;

			return err;
		}
		i++;
	}

	return 0;

err_flow_block:
	list_for_each_entry(hook, &basechain->hook_list, list) {
		if (i-- <= 0)
			break;

		dev = hook->ops.dev;
		nft_chain_offload_cmd(basechain, dev, FLOW_BLOCK_UNBIND);
	}
	return err;
}

Project: nf
File: nft_immediate.c

static void nft_immediate_deactivate(const struct nft_ctx *ctx,
				     const struct nft_expr *expr,
				     enum nft_trans_phase phase)
{
	const struct nft_immediate_expr *priv = nft_expr_priv(expr);

	if (phase == NFT_TRANS_COMMIT)
		return;

	return nft_data_release(&priv->data, nft_dreg_to_type(priv->dreg));
}

Project: nf
File: nft_immediate.c

static int nft_immediate_validate(const struct nft_ctx *ctx,
				  const struct nft_expr *expr,
				  const struct nft_data **d)
{
	const struct nft_immediate_expr *priv = nft_expr_priv(expr);
	struct nft_ctx *pctx = (struct nft_ctx *)ctx;
	const struct nft_data *data;
	int err;

	if (priv->dreg != NFT_REG_VERDICT)
		return 0;

	data = &priv->data;

	switch (data->verdict.code) {
	case NFT_JUMP:
	case NFT_GOTO:
		pctx->level++;
		err = nft_chain_validate(ctx, data->verdict.chain);
		if (err < 0)
			return err;
		pctx->level--;
		break;
	default:
		break;
	}

	return 0;
}

Project: nf
File: nf_tables_offload.c

void nft_flow_rule_set_addr_type(struct nft_flow_rule *flow,
				 enum flow_dissector_key_id addr_type)
{
	struct nft_flow_match *match = &flow->match;
	struct nft_flow_key *mask = &match->mask;
	struct nft_flow_key *key = &match->key;

	if (match->dissector.used_keys & BIT(FLOW_DISSECTOR_KEY_CONTROL))
		return;

	key->control.addr_type = addr_type;
	mask->control.addr_type = 0xffff;
	match->dissector.used_keys |= BIT(FLOW_DISSECTOR_KEY_CONTROL);
	match->dissector.offset[FLOW_DISSECTOR_KEY_CONTROL] =
		offsetof(struct nft_flow_key, control);
}

Project: nf
File: nf_tables_offload.c

static int nft_offload_netdev_event(struct notifier_block *this,
				    unsigned long event, void *ptr)
{
	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
	struct nftables_pernet *nft_net;
	struct net *net = dev_net(dev);
	struct nft_chain *chain;

	if (event != NETDEV_UNREGISTER)
		return NOTIFY_DONE;

	nft_net = nft_pernet(net);
	mutex_lock(&nft_net->commit_mutex);
	chain = __nft_offload_get_chain(nft_net, dev);
	if (chain)
		nft_flow_block_chain(nft_base_chain(chain), dev,
				     FLOW_BLOCK_UNBIND);

	mutex_unlock(&nft_net->commit_mutex);

	return NOTIFY_DONE;
}

Project: nf
File: nf_tables_offload.c

int nft_offload_init(void)
{
	return register_netdevice_notifier(&nft_offload_netdev_notifier);
}

Project: nf
File: nft_immediate.c

static int nft_immediate_init(const struct nft_ctx *ctx,
			      const struct nft_expr *expr,
			      const struct nlattr * const tb[])
{
	struct nft_immediate_expr *priv = nft_expr_priv(expr);
	struct nft_data_desc desc;
	int err;

	if (tb[NFTA_IMMEDIATE_DREG] == NULL ||
	    tb[NFTA_IMMEDIATE_DATA] == NULL)
		return -EINVAL;

	err = nft_data_init(ctx, &priv->data, sizeof(priv->data), &desc,
			    tb[NFTA_IMMEDIATE_DATA]);
	if (err < 0)
		return err;

	priv->dlen = desc.len;

	err = nft_parse_register_store(ctx, tb[NFTA_IMMEDIATE_DREG],
				       &priv->dreg, &priv->data, desc.type,
				       desc.len);
	if (err < 0)
		goto err1;

	if (priv->dreg == NFT_REG_VERDICT) {
		struct nft_chain *chain = priv->data.verdict.chain;

		switch (priv->data.verdict.code) {
		case NFT_JUMP:
		case NFT_GOTO:
			if (nft_chain_is_bound(chain)) {
				err = -EBUSY;
				goto err1;
			}
			chain->bound = true;
			break;
		default:
			break;
		}
	}

	return 0;

err1:
	nft_data_release(&priv->data, desc.type);
	return err;
}

Project: nf
File: nft_fwd_netdev.c

static bool nft_fwd_netdev_offload_action(const struct nft_expr *expr)
{
	return true;
}

Project: nf
File: nf_tables_offload.c

static int nft_block_setup(struct nft_base_chain *basechain,
			   struct flow_block_offload *bo,
			   enum flow_block_command cmd)
{
	int err;

	switch (cmd) {
	case FLOW_BLOCK_BIND:
		err = nft_flow_offload_bind(bo, basechain);
		break;
	case FLOW_BLOCK_UNBIND:
		err = nft_flow_offload_unbind(bo, basechain);
		break;
	default:
		WARN_ON_ONCE(1);
		err = -EOPNOTSUPP;
	}

	return err;
}

Project: nf
File: nf_tables_offload.c

static void nft_flow_rule_transfer_vlan(struct nft_offload_ctx *ctx,
					struct nft_flow_rule *flow)
{
	struct nft_flow_match *match = &flow->match;
	struct nft_offload_ethertype ethertype = {
		.value	= match->key.basic.n_proto,
		.mask	= match->mask.basic.n_proto,
	};

	if (match->dissector.used_keys & BIT(FLOW_DISSECTOR_KEY_VLAN) &&
	    (match->key.vlan.vlan_tpid == htons(ETH_P_8021Q) ||
	     match->key.vlan.vlan_tpid == htons(ETH_P_8021AD))) {
		match->key.basic.n_proto = match->key.cvlan.vlan_tpid;
		match->mask.basic.n_proto = match->mask.cvlan.vlan_tpid;
		match->key.cvlan.vlan_tpid = match->key.vlan.vlan_tpid;
		match->mask.cvlan.vlan_tpid = match->mask.vlan.vlan_tpid;
		match->key.vlan.vlan_tpid = ethertype.value;
		match->mask.vlan.vlan_tpid = ethertype.mask;
		match->dissector.offset[FLOW_DISSECTOR_KEY_CVLAN] =
			offsetof(struct nft_flow_key, cvlan);
		match->dissector.used_keys |= BIT(FLOW_DISSECTOR_KEY_CVLAN);
	} else if (match->dissector.used_keys & BIT(FLOW_DISSECTOR_KEY_BASIC) &&
		   (match->key.basic.n_proto == htons(ETH_P_8021Q) ||
		    match->key.basic.n_proto == htons(ETH_P_8021AD))) {
		match->key.basic.n_proto = match->key.vlan.vlan_tpid;
		match->mask.basic.n_proto = match->mask.vlan.vlan_tpid;
		match->key.vlan.vlan_tpid = ethertype.value;
		match->mask.vlan.vlan_tpid = ethertype.mask;
		match->dissector.offset[FLOW_DISSECTOR_KEY_VLAN] =
			offsetof(struct nft_flow_key, vlan);
		match->dissector.used_keys |= BIT(FLOW_DISSECTOR_KEY_VLAN);
	}
}

Project: nf
File: nft_dup_netdev.c

static void nft_dup_netdev_eval(const struct nft_expr *expr,
				struct nft_regs *regs,
				const struct nft_pktinfo *pkt)
{
	struct nft_dup_netdev *priv = nft_expr_priv(expr);
	int oif = regs->data[priv->sreg_dev];

	nf_dup_netdev_egress(pkt, oif);
}

Project: nf
File: nft_fwd_netdev.c

static int __init nft_fwd_netdev_module_init(void)
{
	return nft_register_expr(&nft_fwd_netdev_type);
}

Project: nf
File: nft_fwd_netdev.c

static void nft_fwd_netdev_eval(const struct nft_expr *expr,
				struct nft_regs *regs,
				const struct nft_pktinfo *pkt)
{
	struct nft_fwd_netdev *priv = nft_expr_priv(expr);
	int oif = regs->data[priv->sreg_dev];
	struct sk_buff *skb = pkt->skb;

	/* This is used by ifb only. */
	skb->skb_iif = skb->dev->ifindex;
	skb_set_redirected(skb, nft_hook(pkt) == NF_NETDEV_INGRESS);

	nf_fwd_netdev_egress(pkt, oif);
	regs->verdict.code = NF_STOLEN;
}

Project: nf
File: nf_tables_offload.c

void nft_flow_rule_destroy(struct nft_flow_rule *flow)
{
	struct flow_action_entry *entry;
	int i;

	flow_action_for_each(i, entry, &flow->rule->action) {
		switch (entry->id) {
		case FLOW_ACTION_REDIRECT:
		case FLOW_ACTION_MIRRED:
			dev_put(entry->dev);
			break;
		default:
			break;
		}
	}
	kfree(flow->rule);
	kfree(flow);
}

Project: nf
File: nf_tables_offload.c

static void nft_flow_offload_common_init(struct flow_cls_common_offload *common,
					 __be16 proto, int priority,
					 struct netlink_ext_ack *extack)
{
	common->protocol = proto;
	common->prio = priority;
	common->extack = extack;
}

Project: nf
File: nf_tables_offload.c

static void nft_flow_block_offload_init(struct flow_block_offload *bo,
					struct net *net,
					enum flow_block_command cmd,
					struct nft_base_chain *basechain,
					struct netlink_ext_ack *extack)
{
	memset(bo, 0, sizeof(*bo));
	bo->net		= net;
	bo->block	= &basechain->flow_block;
	bo->command	= cmd;
	bo->binder_type	= FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS;
	bo->extack	= extack;
	bo->cb_list_head = &basechain->flow_block.cb_list;
	INIT_LIST_HEAD(&bo->cb_list);
}

Project: nf
File: nft_fwd_netdev.c

static void __exit nft_fwd_netdev_module_exit(void)
{
	nft_unregister_expr(&nft_fwd_netdev_type);
}

Project: nf
File: nf_tables_offload.c

static void nft_indr_block_cleanup(struct flow_block_cb *block_cb)
{
	struct nft_base_chain *basechain = block_cb->indr.data;
	struct net_device *dev = block_cb->indr.dev;
	struct netlink_ext_ack extack = {};
	struct nftables_pernet *nft_net;
	struct net *net = dev_net(dev);
	struct flow_block_offload bo;

	nft_flow_block_offload_init(&bo, dev_net(dev), FLOW_BLOCK_UNBIND,
				    basechain, &extack);
	nft_net = nft_pernet(net);
	mutex_lock(&nft_net->commit_mutex);
	list_del(&block_cb->driver_list);
	list_move(&block_cb->list, &bo.cb_list);
	nft_flow_offload_unbind(&bo, basechain);
	mutex_unlock(&nft_net->commit_mutex);
}

Project: nf
File: nft_immediate.c

static void nft_immediate_activate(const struct nft_ctx *ctx,
				   const struct nft_expr *expr)
{
	const struct nft_immediate_expr *priv = nft_expr_priv(expr);

	return nft_data_hold(&priv->data, nft_dreg_to_type(priv->dreg));
}

Project: nf
File: nf_tables_offload.c

int nft_flow_rule_stats(const struct nft_chain *chain,
			const struct nft_rule *rule)
{
	struct flow_cls_offload cls_flow = {};
	struct nft_expr *expr, *next;
	int err;

	err = nft_flow_offload_cmd(chain, rule, NULL, FLOW_CLS_STATS,
				   &cls_flow);
	if (err < 0)
		return err;

	nft_rule_for_each_expr(expr, next, rule) {
		if (expr->ops->offload_stats)
			expr->ops->offload_stats(expr, &cls_flow.stats);
	}

	return 0;
}

Project: nf
File: nft_fwd_netdev.c

static int nft_fwd_netdev_offload(struct nft_offload_ctx *ctx,
				  struct nft_flow_rule *flow,
				  const struct nft_expr *expr)
{
	const struct nft_fwd_netdev *priv = nft_expr_priv(expr);
	int oif = ctx->regs[priv->sreg_dev].data.data[0];

	return nft_fwd_dup_netdev_offload(ctx, flow, FLOW_ACTION_REDIRECT, oif);
}

Project: nf
File: nft_immediate.c

static bool nft_immediate_offload_action(const struct nft_expr *expr)
{
	const struct nft_immediate_expr *priv = nft_expr_priv(expr);

	if (priv->dreg == NFT_REG_VERDICT)
		return true;

	return false;
}

Project: nf
File: nf_tables_offload.c

static int nft_indr_block_offload_cmd(struct nft_base_chain *basechain,
				      struct net_device *dev,
				      enum flow_block_command cmd)
{
	struct netlink_ext_ack extack = {};
	struct flow_block_offload bo;
	int err;

	nft_flow_block_offload_init(&bo, dev_net(dev), cmd, basechain, &extack);

	err = flow_indr_dev_setup_offload(dev, NULL, TC_SETUP_BLOCK, basechain, &bo,
					  nft_indr_block_cleanup);
	if (err < 0)
		return err;

	if (list_empty(&bo.cb_list))
		return -EOPNOTSUPP;

	return nft_block_setup(basechain, &bo, cmd);
}

Project: nf
File: nf_tables_offload.c

static int nft_chain_offload_cmd(struct nft_base_chain *basechain,
				 struct net_device *dev,
				 enum flow_block_command cmd)
{
	int err;

	if (dev->netdev_ops->ndo_setup_tc)
		err = nft_block_offload_cmd(basechain, dev, cmd);
	else
		err = nft_indr_block_offload_cmd(basechain, dev, cmd);

	return err;
}

Project: nf
File: nft_immediate.c

void nft_immediate_eval(const struct nft_expr *expr,
			struct nft_regs *regs,
			const struct nft_pktinfo *pkt)
{
	const struct nft_immediate_expr *priv = nft_expr_priv(expr);

	nft_data_copy(&regs->data[priv->dreg], &priv->data, priv->dlen);
}

Project: nf
File: nf_tables_offload.c

static int nft_block_offload_cmd(struct nft_base_chain *chain,
				 struct net_device *dev,
				 enum flow_block_command cmd)
{
	struct netlink_ext_ack extack = {};
	struct flow_block_offload bo;
	int err;

	nft_flow_block_offload_init(&bo, dev_net(dev), cmd, chain, &extack);

	err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo);
	if (err < 0)
		return err;

	return nft_block_setup(chain, &bo, cmd);
}

Project: nf
File: nf_tables_offload.c

void nft_offload_update_dependency(struct nft_offload_ctx *ctx,
				   const void *data, u32 len)
{
	switch (ctx->dep.type) {
	case NFT_OFFLOAD_DEP_NETWORK:
		WARN_ON(len != sizeof(__u16));
		memcpy(&ctx->dep.l3num, data, sizeof(__u16));
		break;
	case NFT_OFFLOAD_DEP_TRANSPORT:
		WARN_ON(len != sizeof(__u8));
		memcpy(&ctx->dep.protonum, data, sizeof(__u8));
		break;
	default:
		break;
	}
	ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;
}

Project: nf
File: nft_immediate.c

static int nft_immediate_dump(struct sk_buff *skb, const struct nft_expr *expr)
{
	const struct nft_immediate_expr *priv = nft_expr_priv(expr);

	if (nft_dump_register(skb, NFTA_IMMEDIATE_DREG, priv->dreg))
		goto nla_put_failure;

	return nft_data_dump(skb, NFTA_IMMEDIATE_DATA, &priv->data,
			     nft_dreg_to_type(priv->dreg), priv->dlen);

nla_put_failure:
	return -1;
}

Project: nf
File: nft_dup_netdev.c

static void __exit nft_dup_netdev_module_exit(void)
{
	nft_unregister_expr(&nft_dup_netdev_type);
}

Project: nf
File: nf_tables_offload.c

void nft_offload_set_dependency(struct nft_offload_ctx *ctx,
				enum nft_offload_dep_type type)
{
	ctx->dep.type = type;
}

Project: nf
File: nft_fwd_netdev.c

static void nft_fwd_neigh_eval(const struct nft_expr *expr,
			      struct nft_regs *regs,
			      const struct nft_pktinfo *pkt)
{
	struct nft_fwd_neigh *priv = nft_expr_priv(expr);
	void *addr = &regs->data[priv->sreg_addr];
	int oif = regs->data[priv->sreg_dev];
	unsigned int verdict = NF_STOLEN;
	struct sk_buff *skb = pkt->skb;
	struct net_device *dev;
	int neigh_table;

	switch (priv->nfproto) {
	case NFPROTO_IPV4: {
		struct iphdr *iph;

		if (skb->protocol != htons(ETH_P_IP)) {
			verdict = NFT_BREAK;
			goto out;
		}
		if (skb_try_make_writable(skb, sizeof(*iph))) {
			verdict = NF_DROP;
			goto out;
		}
		iph = ip_hdr(skb);
		ip_decrease_ttl(iph);
		neigh_table = NEIGH_ARP_TABLE;
		break;
		}
	case NFPROTO_IPV6: {
		struct ipv6hdr *ip6h;

		if (skb->protocol != htons(ETH_P_IPV6)) {
			verdict = NFT_BREAK;
			goto out;
		}
		if (skb_try_make_writable(skb, sizeof(*ip6h))) {
			verdict = NF_DROP;
			goto out;
		}
		ip6h = ipv6_hdr(skb);
		ip6h->hop_limit--;
		neigh_table = NEIGH_ND_TABLE;
		break;
		}
	default:
		verdict = NFT_BREAK;
		goto out;
	}

	dev = dev_get_by_index_rcu(nft_net(pkt), oif);
	if (dev == NULL)
		return;

	skb->dev = dev;
	skb->tstamp = 0;
	neigh_xmit(neigh_table, dev, addr, skb);
out:
	regs->verdict.code = verdict;
}

Project: nf
File: nf_tables_offload.c

static int nft_setup_cb_call(enum tc_setup_type type, void *type_data,
			     struct list_head *cb_list)
{
	struct flow_block_cb *block_cb;
	int err;

	list_for_each_entry(block_cb, cb_list, list) {
		err = block_cb->cb(type, type_data, block_cb->cb_priv);
		if (err < 0)
			return err;
	}
	return 0;
}

Project: nf
File: nf_tables_offload.c

static void nft_flow_rule_offload_abort(struct net *net,
					struct nft_trans *trans)
{
	struct nftables_pernet *nft_net = nft_pernet(net);
	int err = 0;

	list_for_each_entry_continue_reverse(trans, &nft_net->commit_list, list) {
		if (trans->ctx.family != NFPROTO_NETDEV)
			continue;

		switch (trans->msg_type) {
		case NFT_MSG_NEWCHAIN:
			if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD) ||
			    nft_trans_chain_update(trans))
				continue;

			err = nft_flow_offload_chain(trans->ctx.chain, NULL,
						     FLOW_BLOCK_UNBIND);
			break;
		case NFT_MSG_DELCHAIN:
			if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
				continue;

			err = nft_flow_offload_chain(trans->ctx.chain, NULL,
						     FLOW_BLOCK_BIND);
			break;
		case NFT_MSG_NEWRULE:
			if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
				continue;

			err = nft_flow_offload_rule(trans->ctx.chain,
						    nft_trans_rule(trans),
						    NULL, FLOW_CLS_DESTROY);
			break;
		case NFT_MSG_DELRULE:
			if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
				continue;

			err = nft_flow_offload_rule(trans->ctx.chain,
						    nft_trans_rule(trans),
						    nft_trans_flow_rule(trans),
						    FLOW_CLS_REPLACE);
			break;
		}

		if (WARN_ON_ONCE(err))
			break;
	}
}

Project: nf
File: nf_tables_offload.c

static int nft_flow_offload_chain(struct nft_chain *chain, u8 *ppolicy,
				  enum flow_block_command cmd)
{
	struct nft_base_chain *basechain;
	u8 policy;

	if (!nft_is_base_chain(chain))
		return -EOPNOTSUPP;

	basechain = nft_base_chain(chain);
	policy = ppolicy ? *ppolicy : basechain->policy;

	/* Only default policy to accept is supported for now. */
	if (cmd == FLOW_BLOCK_BIND && policy == NF_DROP)
		return -EOPNOTSUPP;

	return nft_flow_block_chain(basechain, NULL, cmd);
}