miniflow_extract(struct dp_packet *packet, struct miniflow *dst)
{
    /* Add code to this function (or its callees) to extract new fields. */
    BUILD_ASSERT_DECL(FLOW_WC_SEQ == 42);

    const struct pkt_metadata *md = &packet->md;
    const void *data = dp_packet_data(packet);
    size_t size = dp_packet_size(packet);
    ovs_be32 packet_type = packet->packet_type;
    uint64_t *values = miniflow_values(dst);
    struct mf_ctx mf = { FLOWMAP_EMPTY_INITIALIZER, values,
                         values + FLOW_U64S };
    const char *frame;
    ovs_be16 dl_type = OVS_BE16_MAX;
    uint8_t nw_frag, nw_tos, nw_ttl, nw_proto;
    uint8_t *ct_nw_proto_p = NULL;
    ovs_be16 ct_tp_src = 0, ct_tp_dst = 0;

    /* Metadata. */
    if (flow_tnl_dst_is_set(&md->tunnel)) {
        miniflow_push_words(mf, tunnel, &md->tunnel,
                            offsetof(struct flow_tnl, metadata) /
                            sizeof(uint64_t));

        if (!(md->tunnel.flags & FLOW_TNL_F_UDPIF)) {
            if (md->tunnel.metadata.present.map) {
                miniflow_push_words(mf, tunnel.metadata, &md->tunnel.metadata,
                                    sizeof md->tunnel.metadata /
                                    sizeof(uint64_t));
            }
        } else {
            if (md->tunnel.metadata.present.len) {
                miniflow_push_words(mf, tunnel.metadata.present,
                                    &md->tunnel.metadata.present, 1);
                miniflow_push_words(mf, tunnel.metadata.opts.gnv,
                                    md->tunnel.metadata.opts.gnv,
                                    DIV_ROUND_UP(md->tunnel.metadata.present.len,
                                                 sizeof(uint64_t)));
            }
        }
    }
    if (md->skb_priority || md->pkt_mark) {
        miniflow_push_uint32(mf, skb_priority, md->skb_priority);
        miniflow_push_uint32(mf, pkt_mark, md->pkt_mark);
    }
    miniflow_push_uint32(mf, dp_hash, md->dp_hash);
    miniflow_push_uint32(mf, in_port, odp_to_u32(md->in_port.odp_port));
    if (md->ct_state) {
        miniflow_push_uint32(mf, recirc_id, md->recirc_id);
        miniflow_push_uint8(mf, ct_state, md->ct_state);
        ct_nw_proto_p = miniflow_pointer(mf, ct_nw_proto);
        miniflow_push_uint8(mf, ct_nw_proto, 0);
        miniflow_push_uint16(mf, ct_zone, md->ct_zone);
        miniflow_push_uint32(mf, ct_mark, md->ct_mark);
        miniflow_push_be32(mf, packet_type, packet_type);
        if (!ovs_u128_is_zero(md->ct_label)) {
            miniflow_push_words(mf, ct_label, &md->ct_label,
                                sizeof md->ct_label / sizeof(uint64_t));
        }
    } else {
        if (md->recirc_id) {
            miniflow_push_uint32(mf, recirc_id, md->recirc_id);
            miniflow_pad_to_64(mf, recirc_id);
        }
        miniflow_pad_from_64(mf, packet_type);
        miniflow_push_be32(mf, packet_type, packet_type);
    }

    /* Initialize packet's layer pointer and offsets. */
    frame = data;
    dp_packet_reset_offsets(packet);

    if (packet_type == htonl(PT_ETH)) {
        /* Must have full Ethernet header to proceed. */
        if (OVS_UNLIKELY(size < sizeof(struct eth_header))) {
            goto out;
        } else {
            /* Link layer. */
            ASSERT_SEQUENTIAL(dl_dst, dl_src);
            miniflow_push_macs(mf, dl_dst, data);

            /* VLAN */
            union flow_vlan_hdr vlans[FLOW_MAX_VLAN_HEADERS];
            size_t num_vlans = parse_vlan(&data, &size, vlans);

            dl_type = parse_ethertype(&data, &size);
            miniflow_push_be16(mf, dl_type, dl_type);
            miniflow_pad_to_64(mf, dl_type);
            if (num_vlans > 0) {
                miniflow_push_words_32(mf, vlans, vlans, num_vlans);
            }

        }
    } else {
        /* Take dl_type from packet_type. */
        dl_type = pt_ns_type_be(packet_type);
        miniflow_pad_from_64(mf, dl_type);
        miniflow_push_be16(mf, dl_type, dl_type);
        /* Do not push vlan_tci, pad instead */
        miniflow_pad_to_64(mf, dl_type);
    }

    /* Parse mpls. */
    if (OVS_UNLIKELY(eth_type_mpls(dl_type))) {
        int count;
        const void *mpls = data;

        packet->l2_5_ofs = (char *)data - frame;
        count = parse_mpls(&data, &size);
        miniflow_push_words_32(mf, mpls_lse, mpls, count);
    }

    /* Network layer. */
    packet->l3_ofs = (char *)data - frame;

    nw_frag = 0;
    if (OVS_LIKELY(dl_type == htons(ETH_TYPE_IP))) {
        const struct ip_header *nh = data;
        int ip_len;
        uint16_t tot_len;

        if (OVS_UNLIKELY(!ipv4_sanity_check(nh, size, &ip_len, &tot_len))) {
            goto out;
        }
        dp_packet_set_l2_pad_size(packet, size - tot_len);
        size = tot_len;   /* Never pull padding. */

        /* Push both source and destination address at once. */
        miniflow_push_words(mf, nw_src, &nh->ip_src, 1);
        if (ct_nw_proto_p && !md->ct_orig_tuple_ipv6) {
            *ct_nw_proto_p = md->ct_orig_tuple.ipv4.ipv4_proto;
            if (*ct_nw_proto_p) {
                miniflow_push_words(mf, ct_nw_src,
                                    &md->ct_orig_tuple.ipv4.ipv4_src, 1);
                ct_tp_src = md->ct_orig_tuple.ipv4.src_port;
                ct_tp_dst = md->ct_orig_tuple.ipv4.dst_port;
            }
        }

        miniflow_push_be32(mf, ipv6_label, 0); /* Padding for IPv4. */

        nw_tos = nh->ip_tos;
        nw_ttl = nh->ip_ttl;
        nw_proto = nh->ip_proto;
        nw_frag = ipv4_get_nw_frag(nh);
        data_pull(&data, &size, ip_len);
    } else if (dl_type == htons(ETH_TYPE_IPV6)) {
        const struct ovs_16aligned_ip6_hdr *nh = data;
        ovs_be32 tc_flow;
        uint16_t plen;

        if (OVS_UNLIKELY(!ipv6_sanity_check(nh, size))) {
            goto out;
        }
        data_pull(&data, &size, sizeof *nh);

        plen = ntohs(nh->ip6_plen);
        dp_packet_set_l2_pad_size(packet, size - plen);
        size = plen;   /* Never pull padding. */

        miniflow_push_words(mf, ipv6_src, &nh->ip6_src,
                            sizeof nh->ip6_src / 8);
        miniflow_push_words(mf, ipv6_dst, &nh->ip6_dst,
                            sizeof nh->ip6_dst / 8);
        if (ct_nw_proto_p && md->ct_orig_tuple_ipv6) {
            *ct_nw_proto_p = md->ct_orig_tuple.ipv6.ipv6_proto;
            if (*ct_nw_proto_p) {
                miniflow_push_words(mf, ct_ipv6_src,
                                    &md->ct_orig_tuple.ipv6.ipv6_src,
                                    2 *
                                    sizeof md->ct_orig_tuple.ipv6.ipv6_src / 8);
                ct_tp_src = md->ct_orig_tuple.ipv6.src_port;
                ct_tp_dst = md->ct_orig_tuple.ipv6.dst_port;
            }
        }

        tc_flow = get_16aligned_be32(&nh->ip6_flow);
        nw_tos = ntohl(tc_flow) >> 20;
        nw_ttl = nh->ip6_hlim;
        nw_proto = nh->ip6_nxt;

        const struct ovs_16aligned_ip6_frag *frag_hdr;
        if (!parse_ipv6_ext_hdrs__(&data, &size, &nw_proto, &nw_frag,
                                   &frag_hdr)) {
            goto out;
        }

        /* This needs to be after the parse_ipv6_ext_hdrs__() call because it
         * leaves the nw_frag word uninitialized. */
        ASSERT_SEQUENTIAL(ipv6_label, nw_frag);
        ovs_be32 label = tc_flow & htonl(IPV6_LABEL_MASK);
        miniflow_push_be32(mf, ipv6_label, label);
    } else {
        if (dl_type == htons(ETH_TYPE_ARP) ||
            dl_type == htons(ETH_TYPE_RARP)) {
            struct eth_addr arp_buf[2];
            const struct arp_eth_header *arp = (const struct arp_eth_header *)
                data_try_pull(&data, &size, ARP_ETH_HEADER_LEN);

            if (OVS_LIKELY(arp) && OVS_LIKELY(arp->ar_hrd == htons(1))
                && OVS_LIKELY(arp->ar_pro == htons(ETH_TYPE_IP))
                && OVS_LIKELY(arp->ar_hln == ETH_ADDR_LEN)
                && OVS_LIKELY(arp->ar_pln == 4)) {
                miniflow_push_be32(mf, nw_src,
                                   get_16aligned_be32(&arp->ar_spa));
                miniflow_push_be32(mf, nw_dst,
                                   get_16aligned_be32(&arp->ar_tpa));

                /* We only match on the lower 8 bits of the opcode. */
                if (OVS_LIKELY(ntohs(arp->ar_op) <= 0xff)) {
                    miniflow_push_be32(mf, ipv6_label, 0); /* Pad with ARP. */
                    miniflow_push_be32(mf, nw_frag, htonl(ntohs(arp->ar_op)));
                }

                /* Must be adjacent. */
                ASSERT_SEQUENTIAL(arp_sha, arp_tha);

                arp_buf[0] = arp->ar_sha;
                arp_buf[1] = arp->ar_tha;
                miniflow_push_macs(mf, arp_sha, arp_buf);
                miniflow_pad_to_64(mf, arp_tha);
            }
        } else if (dl_type == htons(ETH_TYPE_NSH)) {
            struct ovs_key_nsh nsh;

            if (OVS_LIKELY(parse_nsh(&data, &size, &nsh))) {
                miniflow_push_words(mf, nsh, &nsh,
                                    sizeof(struct ovs_key_nsh) /
                                    sizeof(uint64_t));
            }
        }
        goto out;
    }

    packet->l4_ofs = (char *)data - frame;
    miniflow_push_be32(mf, nw_frag,
                       bytes_to_be32(nw_frag, nw_tos, nw_ttl, nw_proto));

    if (OVS_LIKELY(!(nw_frag & FLOW_NW_FRAG_LATER))) {
        if (OVS_LIKELY(nw_proto == IPPROTO_TCP)) {
            if (OVS_LIKELY(size >= TCP_HEADER_LEN)) {
                const struct tcp_header *tcp = data;

                miniflow_push_be32(mf, arp_tha.ea[2], 0);
                miniflow_push_be32(mf, tcp_flags,
                                   TCP_FLAGS_BE32(tcp->tcp_ctl));
                miniflow_push_be16(mf, tp_src, tcp->tcp_src);
                miniflow_push_be16(mf, tp_dst, tcp->tcp_dst);
                miniflow_push_be16(mf, ct_tp_src, ct_tp_src);
                miniflow_push_be16(mf, ct_tp_dst, ct_tp_dst);
            }
        } else if (OVS_LIKELY(nw_proto == IPPROTO_UDP)) {
            if (OVS_LIKELY(size >= UDP_HEADER_LEN)) {
                const struct udp_header *udp = data;

                miniflow_push_be16(mf, tp_src, udp->udp_src);
                miniflow_push_be16(mf, tp_dst, udp->udp_dst);
                miniflow_push_be16(mf, ct_tp_src, ct_tp_src);
                miniflow_push_be16(mf, ct_tp_dst, ct_tp_dst);
            }
        } else if (OVS_LIKELY(nw_proto == IPPROTO_SCTP)) {
            if (OVS_LIKELY(size >= SCTP_HEADER_LEN)) {
                const struct sctp_header *sctp = data;

                miniflow_push_be16(mf, tp_src, sctp->sctp_src);
                miniflow_push_be16(mf, tp_dst, sctp->sctp_dst);
                miniflow_push_be16(mf, ct_tp_src, ct_tp_src);
                miniflow_push_be16(mf, ct_tp_dst, ct_tp_dst);
            }
        } else if (OVS_LIKELY(nw_proto == IPPROTO_ICMP)) {
            if (OVS_LIKELY(size >= ICMP_HEADER_LEN)) {
                const struct icmp_header *icmp = data;

                miniflow_push_be16(mf, tp_src, htons(icmp->icmp_type));
                miniflow_push_be16(mf, tp_dst, htons(icmp->icmp_code));
                miniflow_push_be16(mf, ct_tp_src, ct_tp_src);
                miniflow_push_be16(mf, ct_tp_dst, ct_tp_dst);
            }
        } else if (OVS_LIKELY(nw_proto == IPPROTO_IGMP)) {
            if (OVS_LIKELY(size >= IGMP_HEADER_LEN)) {
                const struct igmp_header *igmp = data;

                miniflow_push_be16(mf, tp_src, htons(igmp->igmp_type));
                miniflow_push_be16(mf, tp_dst, htons(igmp->igmp_code));
                miniflow_push_be16(mf, ct_tp_src, ct_tp_src);
                miniflow_push_be16(mf, ct_tp_dst, ct_tp_dst);
                miniflow_push_be32(mf, igmp_group_ip4,
                                   get_16aligned_be32(&igmp->group));
                miniflow_pad_to_64(mf, igmp_group_ip4);
            }
        } else if (OVS_LIKELY(nw_proto == IPPROTO_ICMPV6)) {
            if (OVS_LIKELY(size >= sizeof(struct icmp6_data_header))) {
                const struct in6_addr *nd_target;
                struct eth_addr arp_buf[2];
                /* This will populate whether we received Option 1
                 * or Option 2. */
                uint8_t opt_type;
                /* This holds the ND Reserved field. */
                ovs_be32 rso_flags;
                const struct icmp6_data_header *icmp6;

                icmp6 = data_pull(&data, &size, sizeof *icmp6);
                if (parse_icmpv6(&data, &size, icmp6,
                                 &rso_flags, &nd_target, arp_buf, &opt_type)) {
                    if (nd_target) {
                        miniflow_push_words(mf, nd_target, nd_target,
                                            sizeof *nd_target / sizeof(uint64_t));
                    }
                    miniflow_push_macs(mf, arp_sha, arp_buf);
                    /* Populate options field and set the padding
                     * accordingly. */
                    if (opt_type != 0) {
                        miniflow_push_be16(mf, tcp_flags, htons(opt_type));
                        /* Pad to align with 64 bits.
                         * This will zero out the pad3 field. */
                        miniflow_pad_to_64(mf, tcp_flags);
                    } else {
                        /* Pad to align with 64 bits.
                         * This will zero out the tcp_flags & pad3 field. */
                        miniflow_pad_to_64(mf, arp_tha);
                    }
                    miniflow_push_be16(mf, tp_src,
                                       htons(icmp6->icmp6_base.icmp6_type));
                    miniflow_push_be16(mf, tp_dst,
                                       htons(icmp6->icmp6_base.icmp6_code));
                    miniflow_pad_to_64(mf, tp_dst);
                    /* Fill ND reserved field. */
                    miniflow_push_be32(mf, igmp_group_ip4, rso_flags);
                    miniflow_pad_to_64(mf, igmp_group_ip4);
                } else {
                    /* ICMPv6 but not ND. */
                    miniflow_push_be16(mf, tp_src,
                                       htons(icmp6->icmp6_base.icmp6_type));
                    miniflow_push_be16(mf, tp_dst,
                                       htons(icmp6->icmp6_base.icmp6_code));
                    miniflow_push_be16(mf, ct_tp_src, ct_tp_src);
                    miniflow_push_be16(mf, ct_tp_dst, ct_tp_dst);
                }
            }
        }
    }
 out:
    dst->map = mf.map;
}


flow_wildcards_or(struct flow_wildcards *dst,
                  const struct flow_wildcards *src1,
                  const struct flow_wildcards *src2)
{
    uint64_t *dst_u64 = (uint64_t *) &dst->masks;
    const uint64_t *src1_u64 = (const uint64_t *) &src1->masks;
    const uint64_t *src2_u64 = (const uint64_t *) &src2->masks;
    size_t i;

    for (i = 0; i < FLOW_U64S; i++) {
        dst_u64[i] = src1_u64[i] | src2_u64[i];
    }
}


dp_packet_ol_flags_ptr(const struct dp_packet *b)
{
    return CONST_CAST(uint64_t *, &b->mbuf.ol_flags);
}


flow_unwildcard_tp_ports(const struct flow *flow, struct flow_wildcards *wc)
{
    if (flow->nw_proto != IPPROTO_ICMP) {
        memset(&wc->masks.tp_src, 0xff, sizeof wc->masks.tp_src);
        memset(&wc->masks.tp_dst, 0xff, sizeof wc->masks.tp_dst);
    } else {
        wc->masks.tp_src = htons(0xff);
        wc->masks.tp_dst = htons(0xff);
    }
}


flow_hash_5tuple(const struct flow *flow, uint32_t basis)
{
    BUILD_ASSERT_DECL(FLOW_WC_SEQ == 42);
    uint32_t hash = basis;

    if (flow) {

        if (flow->dl_type == htons(ETH_TYPE_IPV6)) {
            const uint64_t *flow_u64 = (const uint64_t *)flow;
            int ofs = offsetof(struct flow, ipv6_src) / 8;
            int end = ofs + 2 * sizeof flow->ipv6_src / 8;

            for (;ofs < end; ofs++) {
                hash = hash_add64(hash, flow_u64[ofs]);
            }
        } else if (flow->dl_type == htons(ETH_TYPE_IP)
                   || flow->dl_type == htons(ETH_TYPE_ARP)) {
            hash = hash_add(hash, (OVS_FORCE uint32_t) flow->nw_src);
            hash = hash_add(hash, (OVS_FORCE uint32_t) flow->nw_dst);
        } else {
            goto out;
        }

        hash = hash_add(hash, flow->nw_proto);
        if (flow->nw_proto != IPPROTO_TCP && flow->nw_proto != IPPROTO_UDP
            && flow->nw_proto != IPPROTO_SCTP && flow->nw_proto != IPPROTO_ICMP
            && flow->nw_proto != IPPROTO_ICMPV6) {
            goto out;
        }

        /* Add both ports at once. */
        hash = hash_add(hash,
                        ((const uint32_t *)flow)[offsetof(struct flow, tp_src)
                                                 / sizeof(uint32_t)]);
    }
out:
    return hash_finish(hash, 42); /* Arbitrary number. */
}


dp_packet_get_cutlen(const struct dp_packet *b)
{
    /* Always in valid range if user uses dp_packet_set_cutlen. */
    return b->cutlen;
}


dp_packet_delete_batch(struct dp_packet_batch *batch, bool should_steal)
{
    if (should_steal) {
        struct dp_packet *packet;

        DP_PACKET_BATCH_FOR_EACH (i, packet, batch) {
            dp_packet_delete(packet);
        }
        dp_packet_batch_init(batch);
    }
}


process_one(struct conntrack *ct, struct dp_packet *pkt,
            struct conn_lookup_ctx *ctx, uint16_t zone,
            bool force, bool commit, long long now, const uint32_t *setmark,
            const struct ovs_key_ct_labels *setlabel,
            const struct nat_action_info_t *nat_action_info,
            ovs_be16 tp_src, ovs_be16 tp_dst, const char *helper,
            uint32_t tp_id)
{
    /* Reset ct_state whenever entering a new zone. */
    if (pkt->md.ct_state && pkt->md.ct_zone != zone) {
        pkt->md.ct_state = 0;
    }

    bool create_new_conn = false;
    conn_key_lookup(ct, &ctx->key, ctx->hash, now, &ctx->conn, &ctx->reply);
    struct conn *conn = ctx->conn;

    /* Delete found entry if in wrong direction. 'force' implies commit. */
    if (OVS_UNLIKELY(force && ctx->reply && conn)) {
        ovs_mutex_lock(&ct->ct_lock);
        if (conn_lookup(ct, &conn->key, now, NULL, NULL)) {
            conn_clean(ct, conn);
        }
        ovs_mutex_unlock(&ct->ct_lock);
        conn = NULL;
    }

    if (OVS_LIKELY(conn)) {
        if (conn->conn_type == CT_CONN_TYPE_UN_NAT) {

            ctx->reply = true;
            struct conn *rev_conn = conn;  /* Save for debugging. */
            uint32_t hash = conn_key_hash(&conn->rev_key, ct->hash_basis);
            conn_key_lookup(ct, &ctx->key, hash, now, &conn, &ctx->reply);

            if (!conn) {
                pkt->md.ct_state |= CS_INVALID;
                write_ct_md(pkt, zone, NULL, NULL, NULL);
                char *log_msg = xasprintf("Missing parent conn %p", rev_conn);
                ct_print_conn_info(rev_conn, log_msg, VLL_INFO, true, true);
                free(log_msg);
                return;
            }
        }
    }

    enum ct_alg_ctl_type ct_alg_ctl = get_alg_ctl_type(pkt, tp_src, tp_dst,
                                                       helper);

    if (OVS_LIKELY(conn)) {
        if (OVS_LIKELY(!conn_update_state_alg(ct, pkt, ctx, conn,
                                              nat_action_info,
                                              ct_alg_ctl, now,
                                              &create_new_conn))) {
            create_new_conn = conn_update_state(ct, pkt, ctx, conn, now);
        }
        if (nat_action_info && !create_new_conn) {
            handle_nat(pkt, conn, zone, ctx->reply, ctx->icmp_related);
        }

    } else if (check_orig_tuple(ct, pkt, ctx, now, &conn, nat_action_info)) {
        create_new_conn = conn_update_state(ct, pkt, ctx, conn, now);
    } else {
        if (ctx->icmp_related) {
            /* An icmp related conn should always be found; no new
               connection is created based on an icmp related packet. */
            pkt->md.ct_state = CS_INVALID;
        } else {
            create_new_conn = true;
        }
    }

    const struct alg_exp_node *alg_exp = NULL;
    struct alg_exp_node alg_exp_entry;

    if (OVS_UNLIKELY(create_new_conn)) {

        ovs_rwlock_rdlock(&ct->resources_lock);
        alg_exp = expectation_lookup(&ct->alg_expectations, &ctx->key,
                                     ct->hash_basis,
                                     alg_src_ip_wc(ct_alg_ctl));
        if (alg_exp) {
            memcpy(&alg_exp_entry, alg_exp, sizeof alg_exp_entry);
            alg_exp = &alg_exp_entry;
        }
        ovs_rwlock_unlock(&ct->resources_lock);

        ovs_mutex_lock(&ct->ct_lock);
        if (!conn_lookup(ct, &ctx->key, now, NULL, NULL)) {
            conn = conn_not_found(ct, pkt, ctx, commit, now, nat_action_info,
                                  helper, alg_exp, ct_alg_ctl, tp_id);
        }
        ovs_mutex_unlock(&ct->ct_lock);
    }

    write_ct_md(pkt, zone, conn, &ctx->key, alg_exp);

    if (conn && setmark) {
        set_mark(pkt, conn, setmark[0], setmark[1]);
    }

    if (conn && setlabel) {
        set_label(pkt, conn, &setlabel[0], &setlabel[1]);
    }

    handle_alg_ctl(ct, ctx, pkt, ct_alg_ctl, conn, now, !!nat_action_info);

    set_cached_conn(nat_action_info, ctx, conn, pkt);
}


flow_push_vlan_uninit(struct flow *flow, struct flow_wildcards *wc)
{
    if (wc) {
        int n = flow_count_vlan_headers(flow);
        if (n) {
            memset(wc->masks.vlans, 0xff, sizeof(union flow_vlan_hdr) * n);
        }
    }
    memmove(&flow->vlans[1], &flow->vlans[0],
            sizeof(union flow_vlan_hdr) * (FLOW_MAX_VLAN_HEADERS - 1));
    memset(&flow->vlans[0], 0, sizeof(union flow_vlan_hdr));
}


nat_packet(struct dp_packet *pkt, const struct conn *conn, bool related)
{
    if (conn->nat_info->nat_action & NAT_ACTION_SRC) {
        pkt->md.ct_state |= CS_SRC_NAT;
        if (conn->key.dl_type == htons(ETH_TYPE_IP)) {
            struct ip_header *nh = dp_packet_l3(pkt);
            packet_set_ipv4_addr(pkt, &nh->ip_src,
                                 conn->rev_key.dst.addr.ipv4);
        } else {
            struct ovs_16aligned_ip6_hdr *nh6 = dp_packet_l3(pkt);
            packet_set_ipv6_addr(pkt, conn->key.nw_proto,
                                 nh6->ip6_src.be32,
                                 &conn->rev_key.dst.addr.ipv6, true);
        }
        if (!related) {
            pat_packet(pkt, conn);
        }
    } else if (conn->nat_info->nat_action & NAT_ACTION_DST) {
        pkt->md.ct_state |= CS_DST_NAT;
        if (conn->key.dl_type == htons(ETH_TYPE_IP)) {
            struct ip_header *nh = dp_packet_l3(pkt);
            packet_set_ipv4_addr(pkt, &nh->ip_dst,
                                 conn->rev_key.src.addr.ipv4);
        } else {
            struct ovs_16aligned_ip6_hdr *nh6 = dp_packet_l3(pkt);
            packet_set_ipv6_addr(pkt, conn->key.nw_proto,
                                 nh6->ip6_dst.be32,
                                 &conn->rev_key.src.addr.ipv6, true);
        }
        if (!related) {
            pat_packet(pkt, conn);
        }
    }
}


reverse_pat_packet(struct dp_packet *pkt, const struct conn *conn)
{
    if (conn->nat_info->nat_action & NAT_ACTION_SRC) {
        if (conn->key.nw_proto == IPPROTO_TCP) {
            struct tcp_header *th_in = dp_packet_l4(pkt);
            packet_set_tcp_port(pkt, conn->key.src.port,
                                th_in->tcp_dst);
        } else if (conn->key.nw_proto == IPPROTO_UDP) {
            struct udp_header *uh_in = dp_packet_l4(pkt);
            packet_set_udp_port(pkt, conn->key.src.port,
                                uh_in->udp_dst);
        }
    } else if (conn->nat_info->nat_action & NAT_ACTION_DST) {
        if (conn->key.nw_proto == IPPROTO_TCP) {
            struct tcp_header *th_in = dp_packet_l4(pkt);
            packet_set_tcp_port(pkt, th_in->tcp_src,
                                conn->key.dst.port);
        } else if (conn->key.nw_proto == IPPROTO_UDP) {
            struct udp_header *uh_in = dp_packet_l4(pkt);
            packet_set_udp_port(pkt, uh_in->udp_src,
                                conn->key.dst.port);
        }
    }
}


dp_packet_get_rss_hash(const struct dp_packet *p)
{
    return *dp_packet_rss_ptr(p);
}


set_label(struct dp_packet *pkt, struct conn *conn,
          const struct ovs_key_ct_labels *val,
          const struct ovs_key_ct_labels *mask)
{
    ovs_mutex_lock(&conn->lock);
    if (conn->alg_related) {
        pkt->md.ct_label = conn->label;
    } else {
        ovs_u128 v, m;

        memcpy(&v, val, sizeof v);
        memcpy(&m, mask, sizeof m);

        pkt->md.ct_label.u64.lo = v.u64.lo
                              | (pkt->md.ct_label.u64.lo & ~(m.u64.lo));
        pkt->md.ct_label.u64.hi = v.u64.hi
                              | (pkt->md.ct_label.u64.hi & ~(m.u64.hi));
        conn->label = pkt->md.ct_label;
    }
    ovs_mutex_unlock(&conn->lock);
}


dp_packet_reset_offload(struct dp_packet *p)
{
    *dp_packet_ol_flags_ptr(p) &= ~DP_PACKET_OL_SUPPORTED_MASK;
}


    OVS_REQUIRES(ct->ct_lock)
{
    uint32_t hash = zone_key_hash(zone, ct->hash_basis);
    struct zone_limit *zl;
    HMAP_FOR_EACH_IN_BUCKET (zl, node, hash, &ct->zone_limits) {
        if (zl->czl.zone == zone) {
            return zl;
        }
    }
    return NULL;
}


expectation_lookup(struct hmap *alg_expectations, const struct conn_key *key,
                   uint32_t basis, bool src_ip_wc)
{
    struct conn_key check_key;
    memcpy(&check_key, key, sizeof check_key);
    check_key.src.port = ALG_WC_SRC_PORT;

    if (src_ip_wc) {
        memset(&check_key.src.addr, 0, sizeof check_key.src.addr);
    }

    struct alg_exp_node *alg_exp_node;

    HMAP_FOR_EACH_WITH_HASH (alg_exp_node, node,
                             conn_key_hash(&check_key, basis),
                             alg_expectations) {
        if (!conn_key_cmp(&alg_exp_node->key, &check_key)) {
            return alg_exp_node;
        }
    }
    return NULL;
}


expectation_ref_create(struct hindex *alg_expectation_refs,
                       struct alg_exp_node *alg_exp_node,
                       uint32_t basis)
{
    if (!expectation_ref_lookup_unique(alg_expectation_refs,
                                       &alg_exp_node->parent_key,
                                       &alg_exp_node->key, basis)) {
        hindex_insert(alg_expectation_refs, &alg_exp_node->node_ref,
                      conn_key_hash(&alg_exp_node->parent_key, basis));
    }
}


miniflow_map_init(struct miniflow *flow, const struct flow *src)
{
    /* Initialize map, counting the number of nonzero elements. */
    flowmap_init(&flow->map);
    for (size_t i = 0; i < FLOW_U64S; i++) {
        if (flow_u64_value(src, i)) {
            flowmap_set(&flow->map, i, 1);
        }
    }
}


dp_packet_try_pull(struct dp_packet *b, size_t size)
{
    return dp_packet_size(b) - dp_packet_l2_pad_size(b) >= size
        ? dp_packet_pull(b, size) : NULL;
}


conn_lookup(struct conntrack *ct, const struct conn_key *key,
            long long now, struct conn **conn_out, bool *reply)
{
    uint32_t hash = conn_key_hash(key, ct->hash_basis);
    return conn_key_lookup(ct, key, hash, now, conn_out, reply);
}


conntrack_init(void)
{
    static struct ovsthread_once setup_l4_once = OVSTHREAD_ONCE_INITIALIZER;
    struct conntrack *ct = xzalloc(sizeof *ct);

    ovs_rwlock_init(&ct->resources_lock);
    ovs_rwlock_wrlock(&ct->resources_lock);
    hmap_init(&ct->alg_expectations);
    hindex_init(&ct->alg_expectation_refs);
    ovs_rwlock_unlock(&ct->resources_lock);

    ovs_mutex_init_adaptive(&ct->ct_lock);
    ovs_mutex_lock(&ct->ct_lock);
    cmap_init(&ct->conns);
    for (unsigned i = 0; i < ARRAY_SIZE(ct->exp_lists); i++) {
        ovs_list_init(&ct->exp_lists[i]);
    }
    hmap_init(&ct->zone_limits);
    ct->zone_limit_seq = 0;
    timeout_policy_init(ct);
    ovs_mutex_unlock(&ct->ct_lock);

    ct->hash_basis = random_uint32();
    atomic_count_init(&ct->n_conn, 0);
    atomic_init(&ct->n_conn_limit, DEFAULT_N_CONN_LIMIT);
    atomic_init(&ct->tcp_seq_chk, true);
    latch_init(&ct->clean_thread_exit);
    ct->clean_thread = ovs_thread_create("ct_clean", clean_thread_main, ct);
    ct->ipf = ipf_init();

    /* Initialize the l4 protocols. */
    if (ovsthread_once_start(&setup_l4_once)) {
        for (int i = 0; i < ARRAY_SIZE(l4_protos); i++) {
            l4_protos[i] = &ct_proto_other;
        }
        /* IPPROTO_UDP uses ct_proto_other, so no need to initialize it. */
        l4_protos[IPPROTO_TCP] = &ct_proto_tcp;
        l4_protos[IPPROTO_ICMP] = &ct_proto_icmp4;
        l4_protos[IPPROTO_ICMPV6] = &ct_proto_icmp6;

        ovsthread_once_done(&setup_l4_once);
    }
    return ct;
}


dp_packet_l4_checksum_bad(const struct dp_packet *p)
{
    return (*dp_packet_ol_flags_ptr(p) & DP_PACKET_OL_RX_L4_CKSUM_MASK) ==
            DP_PACKET_OL_RX_L4_CKSUM_BAD;
}


format_flags(struct ds *ds, const char *(*bit_to_string)(uint32_t),
             uint32_t flags, char del)
{
    uint32_t bad = 0;

    if (!flags) {
        ds_put_char(ds, '0');
        return;
    }
    while (flags) {
        uint32_t bit = rightmost_1bit(flags);
        const char *s;

        s = bit_to_string(bit);
        if (s) {
            ds_put_format(ds, "%s%c", s, del);
        } else {
            bad |= bit;
        }

        flags &= ~bit;
    }

    if (bad) {
        ds_put_format(ds, "0x%"PRIx32"%c", bad, del);
    }
    ds_chomp(ds, del);
}


process_one(struct conntrack *ct, struct dp_packet *pkt,
            struct conn_lookup_ctx *ctx, uint16_t zone,
            bool force, bool commit, long long now, const uint32_t *setmark,
            const struct ovs_key_ct_labels *setlabel,
            const struct nat_action_info_t *nat_action_info,
            ovs_be16 tp_src, ovs_be16 tp_dst, const char *helper,
            uint32_t tp_id)
{
    /* Reset ct_state whenever entering a new zone. */
    if (pkt->md.ct_state && pkt->md.ct_zone != zone) {
        pkt->md.ct_state = 0;
    }

    bool create_new_conn = false;
    conn_key_lookup(ct, &ctx->key, ctx->hash, now, &ctx->conn, &ctx->reply);
    struct conn *conn = ctx->conn;

    /* Delete found entry if in wrong direction. 'force' implies commit. */
    if (OVS_UNLIKELY(force && ctx->reply && conn)) {
        ovs_mutex_lock(&ct->ct_lock);
        if (conn_lookup(ct, &conn->key, now, NULL, NULL)) {
            conn_clean(ct, conn);
        }
        ovs_mutex_unlock(&ct->ct_lock);
        conn = NULL;
    }

    if (OVS_LIKELY(conn)) {
        if (conn->conn_type == CT_CONN_TYPE_UN_NAT) {

            ctx->reply = true;
            struct conn *rev_conn = conn;  /* Save for debugging. */
            uint32_t hash = conn_key_hash(&conn->rev_key, ct->hash_basis);
            conn_key_lookup(ct, &ctx->key, hash, now, &conn, &ctx->reply);

            if (!conn) {
                pkt->md.ct_state |= CS_INVALID;
                write_ct_md(pkt, zone, NULL, NULL, NULL);
                char *log_msg = xasprintf("Missing master conn %p", rev_conn);
                ct_print_conn_info(rev_conn, log_msg, VLL_INFO, true, true);
                free(log_msg);
                return;
            }
        }
    }

    enum ct_alg_ctl_type ct_alg_ctl = get_alg_ctl_type(pkt, tp_src, tp_dst,
                                                       helper);

    if (OVS_LIKELY(conn)) {
        if (OVS_LIKELY(!conn_update_state_alg(ct, pkt, ctx, conn,
                                              nat_action_info,
                                              ct_alg_ctl, now,
                                              &create_new_conn))) {
            create_new_conn = conn_update_state(ct, pkt, ctx, conn, now);
        }
        if (nat_action_info && !create_new_conn) {
            handle_nat(pkt, conn, zone, ctx->reply, ctx->icmp_related);
        }

    } else if (check_orig_tuple(ct, pkt, ctx, now, &conn, nat_action_info)) {
        create_new_conn = conn_update_state(ct, pkt, ctx, conn, now);
    } else {
        if (ctx->icmp_related) {
            /* An icmp related conn should always be found; no new
               connection is created based on an icmp related packet. */
            pkt->md.ct_state = CS_INVALID;
        } else {
            create_new_conn = true;
        }
    }

    const struct alg_exp_node *alg_exp = NULL;
    struct alg_exp_node alg_exp_entry;

    if (OVS_UNLIKELY(create_new_conn)) {

        ovs_rwlock_rdlock(&ct->resources_lock);
        alg_exp = expectation_lookup(&ct->alg_expectations, &ctx->key,
                                     ct->hash_basis,
                                     alg_src_ip_wc(ct_alg_ctl));
        if (alg_exp) {
            memcpy(&alg_exp_entry, alg_exp, sizeof alg_exp_entry);
            alg_exp = &alg_exp_entry;
        }
        ovs_rwlock_unlock(&ct->resources_lock);

        ovs_mutex_lock(&ct->ct_lock);
        if (!conn_lookup(ct, &ctx->key, now, NULL, NULL)) {
            conn = conn_not_found(ct, pkt, ctx, commit, now, nat_action_info,
                                  helper, alg_exp, ct_alg_ctl, tp_id);
        }
        ovs_mutex_unlock(&ct->ct_lock);
    }

    write_ct_md(pkt, zone, conn, &ctx->key, alg_exp);

    if (conn && setmark) {
        set_mark(pkt, conn, setmark[0], setmark[1]);
    }

    if (conn && setlabel) {
        set_label(pkt, conn, &setlabel[0], &setlabel[1]);
    }

    handle_alg_ctl(ct, ctx, pkt, ct_alg_ctl, conn, now, !!nat_action_info);

    set_cached_conn(nat_action_info, ctx, conn, pkt);
}


expectation_create(struct conntrack *ct, ovs_be16 dst_port,
                   const struct conn *master_conn, bool reply, bool src_ip_wc,
                   bool skip_nat)
{
    union ct_addr src_addr;
    union ct_addr dst_addr;
    union ct_addr alg_nat_repl_addr;
    struct alg_exp_node *alg_exp_node = xzalloc(sizeof *alg_exp_node);

    if (reply) {
        src_addr = master_conn->key.src.addr;
        dst_addr = master_conn->key.dst.addr;
        alg_exp_node->nat_rpl_dst = true;
        if (skip_nat) {
            alg_nat_repl_addr = dst_addr;
        } else if (master_conn->nat_info &&
                   master_conn->nat_info->nat_action & NAT_ACTION_DST) {
            alg_nat_repl_addr = master_conn->rev_key.src.addr;
            alg_exp_node->nat_rpl_dst = false;
        } else {
            alg_nat_repl_addr = master_conn->rev_key.dst.addr;
        }
    } else {
        src_addr = master_conn->rev_key.src.addr;
        dst_addr = master_conn->rev_key.dst.addr;
        alg_exp_node->nat_rpl_dst = false;
        if (skip_nat) {
            alg_nat_repl_addr = src_addr;
        } else if (master_conn->nat_info &&
                   master_conn->nat_info->nat_action & NAT_ACTION_DST) {
            alg_nat_repl_addr = master_conn->key.dst.addr;
            alg_exp_node->nat_rpl_dst = true;
        } else {
            alg_nat_repl_addr = master_conn->key.src.addr;
        }
    }
    if (src_ip_wc) {
        memset(&src_addr, 0, sizeof src_addr);
    }

    alg_exp_node->key.dl_type = master_conn->key.dl_type;
    alg_exp_node->key.nw_proto = master_conn->key.nw_proto;
    alg_exp_node->key.zone = master_conn->key.zone;
    alg_exp_node->key.src.addr = src_addr;
    alg_exp_node->key.dst.addr = dst_addr;
    alg_exp_node->key.src.port = ALG_WC_SRC_PORT;
    alg_exp_node->key.dst.port = dst_port;
    alg_exp_node->master_mark = master_conn->mark;
    alg_exp_node->master_label = master_conn->label;
    memcpy(&alg_exp_node->master_key, &master_conn->key,
           sizeof alg_exp_node->master_key);
    /* Take the write lock here because it is almost 100%
     * likely that the lookup will fail and
     * expectation_create() will be called below. */
    ovs_rwlock_wrlock(&ct->resources_lock);
    struct alg_exp_node *alg_exp = expectation_lookup(
        &ct->alg_expectations, &alg_exp_node->key, ct->hash_basis, src_ip_wc);
    if (alg_exp) {
        free(alg_exp_node);
        ovs_rwlock_unlock(&ct->resources_lock);
        return;
    }

    alg_exp_node->alg_nat_repl_addr = alg_nat_repl_addr;
    hmap_insert(&ct->alg_expectations, &alg_exp_node->node,
                conn_key_hash(&alg_exp_node->key, ct->hash_basis));
    expectation_ref_create(&ct->alg_expectation_refs, alg_exp_node,
                           ct->hash_basis);
    ovs_rwlock_unlock(&ct->resources_lock);
}


dp_packet_hwol_l4_is_sctp(struct dp_packet *b)
{
    return (b->mbuf.ol_flags & PKT_TX_L4_MASK) == PKT_TX_SCTP_CKSUM;
}


dp_packet_hwol_set_csum_tcp(struct dp_packet *b)
{
    b->mbuf.ol_flags |= PKT_TX_TCP_CKSUM;
}


dp_packet_hwol_set_csum_sctp(struct dp_packet *b)
{
    b->mbuf.ol_flags |= PKT_TX_SCTP_CKSUM;
}


flow_wc_map(const struct flow *flow, struct flowmap *map)
{
    /* Update this function whenever struct flow changes. */
    BUILD_ASSERT_DECL(FLOW_WC_SEQ == 41);

    flowmap_init(map);

    if (flow_tnl_dst_is_set(&flow->tunnel)) {
        FLOWMAP_SET__(map, tunnel, offsetof(struct flow_tnl, metadata));
        if (!(flow->tunnel.flags & FLOW_TNL_F_UDPIF)) {
            if (flow->tunnel.metadata.present.map) {
                FLOWMAP_SET(map, tunnel.metadata);
            }
        } else {
            FLOWMAP_SET(map, tunnel.metadata.present.len);
            FLOWMAP_SET__(map, tunnel.metadata.opts.gnv,
                          flow->tunnel.metadata.present.len);
        }
    }

    /* Metadata fields that can appear on packet input. */
    FLOWMAP_SET(map, skb_priority);
    FLOWMAP_SET(map, pkt_mark);
    FLOWMAP_SET(map, recirc_id);
    FLOWMAP_SET(map, dp_hash);
    FLOWMAP_SET(map, in_port);
    FLOWMAP_SET(map, dl_dst);
    FLOWMAP_SET(map, dl_src);
    FLOWMAP_SET(map, dl_type);
    FLOWMAP_SET(map, vlans);
    FLOWMAP_SET(map, ct_state);
    FLOWMAP_SET(map, ct_zone);
    FLOWMAP_SET(map, ct_mark);
    FLOWMAP_SET(map, ct_label);
    FLOWMAP_SET(map, packet_type);

    /* Ethertype-dependent fields. */
    if (OVS_LIKELY(flow->dl_type == htons(ETH_TYPE_IP))) {
        FLOWMAP_SET(map, nw_src);
        FLOWMAP_SET(map, nw_dst);
        FLOWMAP_SET(map, nw_proto);
        FLOWMAP_SET(map, nw_frag);
        FLOWMAP_SET(map, nw_tos);
        FLOWMAP_SET(map, nw_ttl);
        FLOWMAP_SET(map, tp_src);
        FLOWMAP_SET(map, tp_dst);
        FLOWMAP_SET(map, ct_nw_proto);
        FLOWMAP_SET(map, ct_nw_src);
        FLOWMAP_SET(map, ct_nw_dst);
        FLOWMAP_SET(map, ct_tp_src);
        FLOWMAP_SET(map, ct_tp_dst);

        if (OVS_UNLIKELY(flow->nw_proto == IPPROTO_IGMP)) {
            FLOWMAP_SET(map, igmp_group_ip4);
        } else {
            FLOWMAP_SET(map, tcp_flags);
        }
    } else if (flow->dl_type == htons(ETH_TYPE_IPV6)) {
        FLOWMAP_SET(map, ipv6_src);
        FLOWMAP_SET(map, ipv6_dst);
        FLOWMAP_SET(map, ipv6_label);
        FLOWMAP_SET(map, nw_proto);
        FLOWMAP_SET(map, nw_frag);
        FLOWMAP_SET(map, nw_tos);
        FLOWMAP_SET(map, nw_ttl);
        FLOWMAP_SET(map, tp_src);
        FLOWMAP_SET(map, tp_dst);

        if (OVS_UNLIKELY(is_nd(flow, NULL))) {
            FLOWMAP_SET(map, nd_target);
            FLOWMAP_SET(map, arp_sha);
            FLOWMAP_SET(map, arp_tha);
            FLOWMAP_SET(map, tcp_flags);
            FLOWMAP_SET(map, igmp_group_ip4);
        } else {
            FLOWMAP_SET(map, ct_nw_proto);
            FLOWMAP_SET(map, ct_ipv6_src);
            FLOWMAP_SET(map, ct_ipv6_dst);
            FLOWMAP_SET(map, ct_tp_src);
            FLOWMAP_SET(map, ct_tp_dst);
            FLOWMAP_SET(map, tcp_flags);
        }
    } else if (eth_type_mpls(flow->dl_type)) {
        FLOWMAP_SET(map, mpls_lse);
    } else if (flow->dl_type == htons(ETH_TYPE_ARP) ||
               flow->dl_type == htons(ETH_TYPE_RARP)) {
        FLOWMAP_SET(map, nw_src);
        FLOWMAP_SET(map, nw_dst);
        FLOWMAP_SET(map, nw_proto);
        FLOWMAP_SET(map, arp_sha);
        FLOWMAP_SET(map, arp_tha);
    } else if (flow->dl_type == htons(ETH_TYPE_NSH)) {
        FLOWMAP_SET(map, nsh.flags);
        FLOWMAP_SET(map, nsh.mdtype);
        FLOWMAP_SET(map, nsh.np);
        FLOWMAP_SET(map, nsh.path_hdr);
        FLOWMAP_SET(map, nsh.context);
    }
}


handle_alg_ctl(struct conntrack *ct, const struct conn_lookup_ctx *ctx,
               struct dp_packet *pkt, enum ct_alg_ctl_type ct_alg_ctl,
               const struct conn *conn, long long now, bool nat,
               const struct conn *conn_for_expectation)
{
    /* ALG control packet handling with expectation creation. */
    if (OVS_UNLIKELY(alg_helpers[ct_alg_ctl] && conn && conn->alg)) {
        alg_helpers[ct_alg_ctl](ct, ctx, pkt, conn_for_expectation, now,
                                CT_FTP_CTL_INTEREST, nat);
    }
}


nat_conn_keys_lookup(struct hmap *nat_conn_keys,
                     const struct conn_key *key,
                     uint32_t basis)
{
    struct nat_conn_key_node *nat_conn_key_node;

    HMAP_FOR_EACH_WITH_HASH (nat_conn_key_node, node,
                             conn_key_hash(key, basis), nat_conn_keys) {
        if (!conn_key_cmp(&nat_conn_key_node->key, key)) {
            return nat_conn_key_node;
        }
    }
    return NULL;
}


dp_packet_ip_checksum_valid(struct dp_packet *p)
{
    return (p->mbuf.ol_flags & PKT_RX_IP_CKSUM_MASK) ==
            PKT_RX_IP_CKSUM_GOOD;
}


extract_l4_icmp6(struct conn_key *key, const void *data, size_t size,
                 bool *related)
{
    const struct icmp6_header *icmp6 = data;

    /* All the messages that we support need at least 4 bytes after
     * the header */
    if (size < sizeof *icmp6 + 4) {
        return false;
    }

    switch (icmp6->icmp6_type) {
    case ICMP6_ECHO_REQUEST:
    case ICMP6_ECHO_REPLY:
        if (icmp6->icmp6_code != 0) {
            return false;
        }
        /* Separate ICMP connection: identified using id */
        key->src.icmp_id = key->dst.icmp_id = *(ovs_be16 *) (icmp6 + 1);
        key->src.icmp_type = icmp6->icmp6_type;
        key->dst.icmp_type = reverse_icmp6_type(icmp6->icmp6_type);
        break;
    case ICMP6_DST_UNREACH:
    case ICMP6_PACKET_TOO_BIG:
    case ICMP6_TIME_EXCEEDED:
    case ICMP6_PARAM_PROB: {
        /* ICMP packet part of another connection. We should
         * extract the key from embedded packet header */
        struct conn_key inner_key;
        const char *l3 = (const char *) icmp6 + 8;
        const char *tail = (const char *) data + size;
        const char *l4 = NULL;

        if (!related) {
            return false;
        }

        memset(&inner_key, 0, sizeof inner_key);
        inner_key.dl_type = htons(ETH_TYPE_IPV6);
        bool ok = extract_l3_ipv6(&inner_key, l3, tail - l3, &l4);
        if (!ok) {
            return false;
        }

        /* pf doesn't do this, but it seems a good idea */
        if (!ipv6_addr_equals(&inner_key.src.addr.ipv6_aligned,
                              &key->dst.addr.ipv6_aligned)) {
            return false;
        }

        key->src = inner_key.src;
        key->dst = inner_key.dst;
        key->nw_proto = inner_key.nw_proto;

        ok = extract_l4(key, l4, tail - l4, NULL, l3, false, NULL);
        if (ok) {
            conn_key_reverse(key);
            *related = true;
        }
        return ok;
    }
    default:
        return false;
    }

    return true;
}


dp_packet_mbuf_rss_flag_reset(struct dp_packet *p)
{
    p->mbuf.ol_flags &= ~PKT_RX_RSS_HASH;
}


dp_packet_mbuf_rss_flag_reset(struct dp_packet *p OVS_UNUSED)
{
}


conntrack_flush(struct conntrack *ct, const uint16_t *zone)
{
    for (unsigned i = 0; i < CONNTRACK_BUCKETS; i++) {
        struct conntrack_bucket *ctb = &ct->buckets[i];
        ovs_mutex_lock(&ctb->cleanup_mutex);
        ct_lock_lock(&ctb->lock);
        for (unsigned j = 0; j < N_CT_TM; j++) {
            struct conn *conn, *next;
            LIST_FOR_EACH_SAFE (conn, next, exp_node, &ctb->exp_lists[j]) {
                if (!zone || *zone == conn->key.zone) {
                    conn_clean(ct, conn, ctb);
                }
            }
        }
        ct_lock_unlock(&ctb->lock);
        ovs_mutex_unlock(&ctb->cleanup_mutex);
    }

    return 0;
}


un_nat_packet(struct dp_packet *pkt, const struct conn *conn,
              bool related)
{
    if (conn->nat_info->nat_action & NAT_ACTION_SRC) {
        pkt->md.ct_state |= CS_DST_NAT;
        if (conn->key.dl_type == htons(ETH_TYPE_IP)) {
            struct ip_header *nh = dp_packet_l3(pkt);
            packet_set_ipv4_addr(pkt, &nh->ip_dst,
                                 conn->key.src.addr.ipv4_aligned);
        } else {
            struct ovs_16aligned_ip6_hdr *nh6 = dp_packet_l3(pkt);
            packet_set_ipv6_addr(pkt, conn->key.nw_proto,
                                 nh6->ip6_dst.be32,
                                 &conn->key.src.addr.ipv6_aligned, true);
        }

        if (OVS_UNLIKELY(related)) {
            reverse_nat_packet(pkt, conn);
        } else {
            un_pat_packet(pkt, conn);
        }
    } else if (conn->nat_info->nat_action & NAT_ACTION_DST) {
        pkt->md.ct_state |= CS_SRC_NAT;
        if (conn->key.dl_type == htons(ETH_TYPE_IP)) {
            struct ip_header *nh = dp_packet_l3(pkt);
            packet_set_ipv4_addr(pkt, &nh->ip_src,
                                 conn->key.dst.addr.ipv4_aligned);
        } else {
            struct ovs_16aligned_ip6_hdr *nh6 = dp_packet_l3(pkt);
            packet_set_ipv6_addr(pkt, conn->key.nw_proto,
                                 nh6->ip6_src.be32,
                                 &conn->key.dst.addr.ipv6_aligned, true);
        }

        if (OVS_UNLIKELY(related)) {
            reverse_nat_packet(pkt, conn);
        } else {
            un_pat_packet(pkt, conn);
        }
    }
}


flow_mask_hash_fields(const struct flow *flow, struct flow_wildcards *wc,
                      enum nx_hash_fields fields)
{
    int i;
    switch (fields) {
    case NX_HASH_FIELDS_ETH_SRC:
        memset(&wc->masks.dl_src, 0xff, sizeof wc->masks.dl_src);
        break;

    case NX_HASH_FIELDS_SYMMETRIC_L4:
        memset(&wc->masks.dl_src, 0xff, sizeof wc->masks.dl_src);
        memset(&wc->masks.dl_dst, 0xff, sizeof wc->masks.dl_dst);
        if (flow->dl_type == htons(ETH_TYPE_IP)) {
            memset(&wc->masks.nw_src, 0xff, sizeof wc->masks.nw_src);
            memset(&wc->masks.nw_dst, 0xff, sizeof wc->masks.nw_dst);
        } else if (flow->dl_type == htons(ETH_TYPE_IPV6)) {
            memset(&wc->masks.ipv6_src, 0xff, sizeof wc->masks.ipv6_src);
            memset(&wc->masks.ipv6_dst, 0xff, sizeof wc->masks.ipv6_dst);
        }
        if (is_ip_any(flow)) {
            memset(&wc->masks.nw_proto, 0xff, sizeof wc->masks.nw_proto);
            flow_unwildcard_tp_ports(flow, wc);
        }
        for (i = 0; i < FLOW_MAX_VLAN_HEADERS; i++) {
            wc->masks.vlans[i].tci |= htons(VLAN_VID_MASK | VLAN_CFI);
        }
        break;

    case NX_HASH_FIELDS_SYMMETRIC_L3L4_UDP:
        if (is_ip_any(flow) && flow->nw_proto == IPPROTO_UDP) {
            memset(&wc->masks.tp_src, 0xff, sizeof wc->masks.tp_src);
            memset(&wc->masks.tp_dst, 0xff, sizeof wc->masks.tp_dst);
        }
        /* fall through */
    case NX_HASH_FIELDS_SYMMETRIC_L3L4:
        if (flow->dl_type == htons(ETH_TYPE_IP)) {
            memset(&wc->masks.nw_src, 0xff, sizeof wc->masks.nw_src);
            memset(&wc->masks.nw_dst, 0xff, sizeof wc->masks.nw_dst);
        } else if (flow->dl_type == htons(ETH_TYPE_IPV6)) {
            memset(&wc->masks.ipv6_src, 0xff, sizeof wc->masks.ipv6_src);
            memset(&wc->masks.ipv6_dst, 0xff, sizeof wc->masks.ipv6_dst);
        } else {
            break; /* non-IP flow */
        }

        memset(&wc->masks.nw_proto, 0xff, sizeof wc->masks.nw_proto);
        if (flow->nw_proto == IPPROTO_TCP || flow->nw_proto == IPPROTO_SCTP) {
            memset(&wc->masks.tp_src, 0xff, sizeof wc->masks.tp_src);
            memset(&wc->masks.tp_dst, 0xff, sizeof wc->masks.tp_dst);
        }
        break;

    case NX_HASH_FIELDS_NW_SRC:
        if (flow->dl_type == htons(ETH_TYPE_IP)) {
            memset(&wc->masks.nw_src, 0xff, sizeof wc->masks.nw_src);
        } else if (flow->dl_type == htons(ETH_TYPE_IPV6)) {
            memset(&wc->masks.ipv6_src, 0xff, sizeof wc->masks.ipv6_src);
        }
        break;

    case NX_HASH_FIELDS_NW_DST:
        if (flow->dl_type == htons(ETH_TYPE_IP)) {
            memset(&wc->masks.nw_dst, 0xff, sizeof wc->masks.nw_dst);
        } else if (flow->dl_type == htons(ETH_TYPE_IPV6)) {
            memset(&wc->masks.ipv6_dst, 0xff, sizeof wc->masks.ipv6_dst);
        }
        break;

    default:
        OVS_NOT_REACHED();
    }
}


parse_nsh(const void **datap, size_t *sizep, struct ovs_key_nsh *key)
{
    const struct nsh_hdr *nsh = (const struct nsh_hdr *) *datap;
    uint8_t version, length, flags, ttl;

    /* Check if it is long enough for NSH header, doesn't support
     * MD type 2 yet
     */
    if (OVS_UNLIKELY(*sizep < NSH_BASE_HDR_LEN)) {
        return false;
    }

    version = nsh_get_ver(nsh);
    flags = nsh_get_flags(nsh);
    length = nsh_hdr_len(nsh);
    ttl = nsh_get_ttl(nsh);

    if (OVS_UNLIKELY(length > *sizep || version != 0)) {
        return false;
    }

    key->flags = flags;
    key->ttl = ttl;
    key->mdtype = nsh->md_type;
    key->np = nsh->next_proto;
    key->path_hdr = nsh_get_path_hdr(nsh);

    switch (key->mdtype) {
        case NSH_M_TYPE1:
            if (length != NSH_M_TYPE1_LEN) {
                return false;
            }
            for (size_t i = 0; i < 4; i++) {
                key->context[i] = get_16aligned_be32(&nsh->md1.context[i]);
            }
            break;
        case NSH_M_TYPE2:
            /* Don't support MD type 2 metedata parsing yet */
            if (length < NSH_BASE_HDR_LEN) {
                return false;
            }

            memset(key->context, 0, sizeof(key->context));
            break;
        default:
            /* We don't parse other context headers yet. */
            break;
    }

    data_pull(datap, sizep, length);

    return true;
}


dp_packet_ip_checksum_bad(struct dp_packet *p OVS_UNUSED)
{
#ifdef DPDK_NETDEV
    return (p->mbuf.ol_flags & PKT_RX_IP_CKSUM_MASK) ==
            PKT_RX_IP_CKSUM_BAD;
#else
    return false;
#endif
}


extract_l4_udp(struct conn_key *key, const void *data, size_t size)
{
    if (OVS_UNLIKELY(size < UDP_HEADER_LEN)) {
        return false;
    }

    const struct udp_header *udp = data;
    key->src.port = udp->udp_src;
    key->dst.port = udp->udp_dst;

    /* Port 0 is invalid */
    return key->src.port && key->dst.port;
}


dp_packet_batch_init_packet_fields(struct dp_packet_batch *batch)
{
    struct dp_packet *packet;

    DP_PACKET_BATCH_FOR_EACH (packet, batch) {
        dp_packet_reset_cutlen(packet);
        packet->packet_type = htonl(PT_ETH);
    }
}


dp_packet_batch_clone(struct dp_packet_batch *dst,
                      struct dp_packet_batch *src)
{
    struct dp_packet *packet;

    dp_packet_batch_init(dst);
    DP_PACKET_BATCH_FOR_EACH (packet, src) {
        dp_packet_batch_add(dst, dp_packet_clone(packet));
    }
    dst->trunc = src->trunc;
}


conn_not_found(struct conntrack *ct, struct dp_packet *pkt,
               struct conn_lookup_ctx *ctx, bool commit, long long now,
               const struct nat_action_info_t *nat_action_info,
               struct conn *conn_for_un_nat_copy,
               const char *helper,
               const struct alg_exp_node *alg_exp)
{
    unsigned bucket = hash_to_bucket(ctx->hash);
    struct conn *nc = NULL;

    if (!valid_new(pkt, &ctx->key)) {
        pkt->md.ct_state = CS_INVALID;
        return nc;
    }
    pkt->md.ct_state = CS_NEW;
    if (alg_exp) {
        pkt->md.ct_state |= CS_RELATED;
    }

    if (commit) {
        unsigned int n_conn_limit;

        atomic_read_relaxed(&ct->n_conn_limit, &n_conn_limit);

        if (atomic_count_get(&ct->n_conn) >= n_conn_limit) {
            COVERAGE_INC(conntrack_full);
            return nc;
        }

        nc = new_conn(&ct->buckets[bucket], pkt, &ctx->key, now);
        ctx->conn = nc;
        nc->rev_key = nc->key;
        conn_key_reverse(&nc->rev_key);

        if (helper) {
            nc->alg = xstrdup(helper);
        }

        if (alg_exp) {
            nc->alg_related = true;
            nc->mark = alg_exp->master_mark;
            nc->label = alg_exp->master_label;
            nc->master_key = alg_exp->master_key;
        }

        if (nat_action_info) {
            nc->nat_info = xmemdup(nat_action_info, sizeof *nc->nat_info);

            if (alg_exp) {
                if (alg_exp->passive_mode) {
                    nc->rev_key.dst.addr = alg_exp->alg_nat_repl_addr;
                    nc->nat_info->nat_action = NAT_ACTION_SRC;
                } else {
                    nc->rev_key.src.addr = alg_exp->alg_nat_repl_addr;
                    nc->nat_info->nat_action = NAT_ACTION_DST;
                }
                *conn_for_un_nat_copy = *nc;
                ct_rwlock_wrlock(&ct->resources_lock);
                bool new_insert = nat_conn_keys_insert(&ct->nat_conn_keys,
                                                       conn_for_un_nat_copy,
                                                       ct->hash_basis);
                ct_rwlock_unlock(&ct->resources_lock);
                if (!new_insert) {
                    char *log_msg = xasprintf("Pre-existing alg "
                                              "nat_conn_key");
                    ct_print_conn_info(conn_for_un_nat_copy, log_msg, VLL_INFO,
                                       true, false);
                    free(log_msg);
                }
            } else {
                *conn_for_un_nat_copy = *nc;
                ct_rwlock_wrlock(&ct->resources_lock);
                bool nat_res = nat_select_range_tuple(
                                   ct, nc, conn_for_un_nat_copy);

                if (!nat_res) {
                    goto nat_res_exhaustion;
                }

                /* Update nc with nat adjustments made to
                 * conn_for_un_nat_copy by nat_select_range_tuple(). */
                *nc = *conn_for_un_nat_copy;
                ct_rwlock_unlock(&ct->resources_lock);
            }
            conn_for_un_nat_copy->conn_type = CT_CONN_TYPE_UN_NAT;
            conn_for_un_nat_copy->nat_info = NULL;
            conn_for_un_nat_copy->alg = NULL;
            nat_packet(pkt, nc, ctx->icmp_related);
        }
        hmap_insert(&ct->buckets[bucket].connections, &nc->node, ctx->hash);
        atomic_count_inc(&ct->n_conn);
    }

    return nc;

    /* This would be a user error or a DOS attack.
     * A user error is prevented by allocating enough
     * combinations of NAT addresses when combined with
     * ephemeral ports.  A DOS attack should be protected
     * against with firewall rules or a separate firewall.
     * Also using zone partitioning can limit DoS impact. */
nat_res_exhaustion:
    ovs_list_remove(&nc->exp_node);
    delete_conn(nc);
    /* conn_for_un_nat_copy is a local variable in process_one; this
     * memset() serves to document that conn_for_un_nat_copy is from
     * this point on unused. */
    memset(conn_for_un_nat_copy, 0, sizeof *conn_for_un_nat_copy);
    ct_rwlock_unlock(&ct->resources_lock);
    static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 5);
    VLOG_WARN_RL(&rl, "Unable to NAT due to tuple space exhaustion - "
                 "if DoS attack, use firewalling and/or zone partitioning.");
    return NULL;
}


extract_l4_udp(struct conn_key *key, const void *data, size_t size,
               size_t *chk_len)
{
    const struct udp_header *udp = data;

    if (OVS_UNLIKELY(size < (chk_len ? *chk_len : UDP_HEADER_LEN))) {
        return false;
    }

    key->src.port = udp->udp_src;
    key->dst.port = udp->udp_dst;

    /* Port 0 is invalid */
    return key->src.port && key->dst.port;
}


extract_l4_icmp(struct conn_key *key, const void *data, size_t size,
                bool *related, size_t *chk_len)
{
    const struct icmp_header *icmp = data;

    if (OVS_UNLIKELY(size < (chk_len ? *chk_len : ICMP_HEADER_LEN))) {
        return false;
    }

    switch (icmp->icmp_type) {
    case ICMP4_ECHO_REQUEST:
    case ICMP4_ECHO_REPLY:
    case ICMP4_TIMESTAMP:
    case ICMP4_TIMESTAMPREPLY:
    case ICMP4_INFOREQUEST:
    case ICMP4_INFOREPLY:
        if (icmp->icmp_code != 0) {
            return false;
        }
        /* Separate ICMP connection: identified using id */
        key->src.icmp_id = key->dst.icmp_id = icmp->icmp_fields.echo.id;
        key->src.icmp_type = icmp->icmp_type;
        key->dst.icmp_type = reverse_icmp_type(icmp->icmp_type);
        break;
    case ICMP4_DST_UNREACH:
    case ICMP4_TIME_EXCEEDED:
    case ICMP4_PARAM_PROB:
    case ICMP4_SOURCEQUENCH:
    case ICMP4_REDIRECT: {
        /* ICMP packet part of another connection. We should
         * extract the key from embedded packet header */
        struct conn_key inner_key;
        const char *l3 = (const char *) (icmp + 1);
        const char *tail = (const char *) data + size;
        const char *l4;
        bool ok;

        if (!related) {
            return false;
        }

        memset(&inner_key, 0, sizeof inner_key);
        inner_key.dl_type = htons(ETH_TYPE_IP);
        ok = extract_l3_ipv4(&inner_key, l3, tail - l3, &l4, false);
        if (!ok) {
            return false;
        }

        if (inner_key.src.addr.ipv4_aligned != key->dst.addr.ipv4_aligned) {
            return false;
        }

        key->src = inner_key.src;
        key->dst = inner_key.dst;
        key->nw_proto = inner_key.nw_proto;
        size_t check_len = ICMP_ERROR_DATA_L4_LEN;

        ok = extract_l4(key, l4, tail - l4, NULL, l3, false, &check_len);
        if (ok) {
            conn_key_reverse(key);
            *related = true;
        }
        return ok;
    }
    default:
        return false;
    }

    return true;
}


create_un_nat_conn(struct conntrack *ct, struct conn *conn_for_un_nat_copy,
                   long long now, bool alg_un_nat)
{
    struct conn *nc = xmemdup(conn_for_un_nat_copy, sizeof *nc);
    nc->key = conn_for_un_nat_copy->rev_key;
    nc->rev_key = conn_for_un_nat_copy->key;
    uint32_t un_nat_hash = conn_key_hash(&nc->key, ct->hash_basis);
    unsigned un_nat_conn_bucket = hash_to_bucket(un_nat_hash);
    ct_lock_lock(&ct->buckets[un_nat_conn_bucket].lock);
    struct conn *rev_conn = conn_lookup(ct, &nc->key, now);

    if (alg_un_nat) {
        if (!rev_conn) {
            hmap_insert(&ct->buckets[un_nat_conn_bucket].connections,
                        &nc->node, un_nat_hash);
        } else {
            char *log_msg = xasprintf("Unusual condition for un_nat conn "
                                      "create for alg: rev_conn %p", rev_conn);
            ct_print_conn_info(nc, log_msg, VLL_INFO, true, false);
            free(log_msg);
            free(nc);
        }
    } else {
        ct_rwlock_rdlock(&ct->resources_lock);

        struct nat_conn_key_node *nat_conn_key_node =
            nat_conn_keys_lookup(&ct->nat_conn_keys, &nc->key, ct->hash_basis);
        if (nat_conn_key_node && !conn_key_cmp(&nat_conn_key_node->value,
            &nc->rev_key) && !rev_conn) {

            hmap_insert(&ct->buckets[un_nat_conn_bucket].connections,
                        &nc->node, un_nat_hash);
        } else {
            char *log_msg = xasprintf("Unusual condition for un_nat conn "
                                      "create: nat_conn_key_node/rev_conn "
                                      "%p/%p", nat_conn_key_node, rev_conn);
            ct_print_conn_info(nc, log_msg, VLL_INFO, true, false);
            free(log_msg);
            free(nc);
        }
        ct_rwlock_unlock(&ct->resources_lock);
    }
    ct_lock_unlock(&ct->buckets[un_nat_conn_bucket].lock);
}


is_tftp_ctl(const struct dp_packet *pkt)
{
    uint8_t ip_proto = get_ip_proto(pkt);
    struct udp_header *uh = dp_packet_l4(pkt);

    /* CT_IPPORT_TFTP is used because IPPORT_TFTP in not defined in OSX,
     * at least in in.h. Since this value will never change, remove
     * the external dependency. */
#define CT_IPPORT_TFTP 69
    return (ip_proto == IPPROTO_UDP &&
            uh->udp_dst == htons(CT_IPPORT_TFTP));

}


ct_print_conn_info(struct conn *c, char *log_msg, enum vlog_level vll,
                   bool force, bool rl_on)
{
#define CT_VLOG(RL_ON, LEVEL, ...)                                          \
    do {                                                                    \
        if (RL_ON) {                                                        \
            static struct vlog_rate_limit rl_ = VLOG_RATE_LIMIT_INIT(5, 5); \
            vlog_rate_limit(&this_module, LEVEL, &rl_, __VA_ARGS__);        \
        } else {                                                            \
            vlog(&this_module, LEVEL, __VA_ARGS__);                         \
        }                                                                   \
    } while (0)

    if (OVS_UNLIKELY(force || vlog_is_enabled(&this_module, vll))) {
        if (c->key.dl_type == htons(ETH_TYPE_IP)) {
            CT_VLOG(rl_on, vll, "%s: src ip "IP_FMT" dst ip "IP_FMT" rev src "
                    "ip "IP_FMT" rev dst ip "IP_FMT" src/dst ports "
                    "%"PRIu16"/%"PRIu16" rev src/dst ports "
                    "%"PRIu16"/%"PRIu16" zone/rev zone "
                    "%"PRIu16"/%"PRIu16" nw_proto/rev nw_proto "
                    "%"PRIu8"/%"PRIu8, log_msg,
                    IP_ARGS(c->key.src.addr.ipv4_aligned),
                    IP_ARGS(c->key.dst.addr.ipv4_aligned),
                    IP_ARGS(c->rev_key.src.addr.ipv4_aligned),
                    IP_ARGS(c->rev_key.dst.addr.ipv4_aligned),
                    ntohs(c->key.src.port), ntohs(c->key.dst.port),
                    ntohs(c->rev_key.src.port), ntohs(c->rev_key.dst.port),
                    c->key.zone, c->rev_key.zone, c->key.nw_proto,
                    c->rev_key.nw_proto);
        } else {
            char ip6_s[INET6_ADDRSTRLEN];
            inet_ntop(AF_INET6, &c->key.src.addr.ipv6, ip6_s, sizeof ip6_s);
            char ip6_d[INET6_ADDRSTRLEN];
            inet_ntop(AF_INET6, &c->key.dst.addr.ipv6, ip6_d, sizeof ip6_d);
            char ip6_rs[INET6_ADDRSTRLEN];
            inet_ntop(AF_INET6, &c->rev_key.src.addr.ipv6, ip6_rs,
                      sizeof ip6_rs);
            char ip6_rd[INET6_ADDRSTRLEN];
            inet_ntop(AF_INET6, &c->rev_key.dst.addr.ipv6, ip6_rd,
                      sizeof ip6_rd);

            CT_VLOG(rl_on, vll, "%s: src ip %s dst ip %s rev src ip %s"
                    " rev dst ip %s src/dst ports %"PRIu16"/%"PRIu16
                    " rev src/dst ports %"PRIu16"/%"PRIu16" zone/rev zone "
                    "%"PRIu16"/%"PRIu16" nw_proto/rev nw_proto "
                    "%"PRIu8"/%"PRIu8, log_msg, ip6_s, ip6_d, ip6_rs,
                    ip6_rd, ntohs(c->key.src.port), ntohs(c->key.dst.port),
                    ntohs(c->rev_key.src.port), ntohs(c->rev_key.dst.port),
                    c->key.zone, c->rev_key.zone, c->key.nw_proto,
                    c->rev_key.nw_proto);
        }
    }
}


conntrack_flush(struct conntrack *ct, const uint16_t *zone)
{
    unsigned i;

    for (i = 0; i < CONNTRACK_BUCKETS; i++) {
        struct conn *conn, *next;

        ct_lock_lock(&ct->buckets[i].lock);
        HMAP_FOR_EACH_SAFE (conn, next, node, &ct->buckets[i].connections) {
            if ((!zone || *zone == conn->key.zone) &&
                (conn->conn_type == CT_CONN_TYPE_DEFAULT)) {
                conn_clean(ct, conn, &ct->buckets[i]);
            }
        }
        ct_lock_unlock(&ct->buckets[i].lock);
    }

    ct_rwlock_wrlock(&ct->resources_lock);
    struct alg_exp_node *alg_exp_node, *alg_exp_node_next;
    HMAP_FOR_EACH_SAFE (alg_exp_node, alg_exp_node_next,
                       node, &ct->alg_expectations) {
        if (!zone || *zone == alg_exp_node->key.zone) {
            ovs_list_remove(&alg_exp_node->exp_node);
            hmap_remove(&ct->alg_expectations, &alg_exp_node->node);
            free(alg_exp_node);
        }
    }
    ct_rwlock_unlock(&ct->resources_lock);
    return 0;
}


conn_seq_skew_set(struct conntrack *ct, const struct conn_key *key,
                  long long now, int seq_skew, bool seq_skew_dir)
{
    uint32_t hash = conn_key_hash(key, ct->hash_basis);
    unsigned bucket = hash_to_bucket(hash);
    ct_lock_lock(&ct->buckets[bucket].lock);
    struct conn *conn = conn_lookup(ct, key, now);
    if (conn && seq_skew) {
        conn->seq_skew = seq_skew;
        conn->seq_skew_dir = seq_skew_dir;
    }
    ct_lock_unlock(&ct->buckets[bucket].lock);
}


nat_conn_keys_remove(struct hmap *nat_conn_keys,
                     const struct conn_key *key,
                     uint32_t basis)
{
    struct nat_conn_key_node *nat_conn_key_node;
    uint32_t nat_conn_key_hash = conn_key_hash(key, basis);

    HMAP_FOR_EACH_WITH_HASH (nat_conn_key_node, node, nat_conn_key_hash,
                             nat_conn_keys) {
        if (!conn_key_cmp(&nat_conn_key_node->key, key)) {
            hmap_remove(nat_conn_keys, &nat_conn_key_node->node);
            free(nat_conn_key_node);
            return;
        }
    }
}


parse_vlan(const void **datap, size_t *sizep)
{
    const struct eth_header *eth = *datap;

    struct qtag_prefix {
        ovs_be16 eth_type;      /* ETH_TYPE_VLAN */
        ovs_be16 tci;
    };

    data_pull(datap, sizep, ETH_ADDR_LEN * 2);

    if (eth->eth_type == htons(ETH_TYPE_VLAN)) {
        if (OVS_LIKELY(*sizep
                       >= sizeof(struct qtag_prefix) + sizeof(ovs_be16))) {
            const struct qtag_prefix *qp = data_pull(datap, sizep, sizeof *qp);
            return qp->tci | htons(VLAN_CFI);
        }
    }
    return 0;
}


format_flags_masked(struct ds *ds, const char *name,
                    const char *(*bit_to_string)(uint32_t), uint32_t flags,
                    uint32_t mask, uint32_t max_mask)
{
    if (name) {
        ds_put_format(ds, "%s=", name);
    }

    if (mask == max_mask) {
        format_flags(ds, bit_to_string, flags, '|');
        return;
    }

    if (!mask) {
        ds_put_cstr(ds, "0/0");
        return;
    }

    while (mask) {
        uint32_t bit = rightmost_1bit(mask);
        const char *s = bit_to_string(bit);

        ds_put_format(ds, "%s%s", (flags & bit) ? "+" : "-",
                      s ? s : "[Unknown]");
        mask &= ~bit;
    }
}


miniflow_hash_5tuple(const struct miniflow *flow, uint32_t basis)
{
    uint32_t hash = basis;

    if (flow) {
        ovs_be16 dl_type = MINIFLOW_GET_BE16(flow, dl_type);

        hash = hash_add(hash, MINIFLOW_GET_U8(flow, nw_proto));

        /* Separate loops for better optimization. */
        if (dl_type == htons(ETH_TYPE_IPV6)) {
            struct flowmap map = FLOWMAP_EMPTY_INITIALIZER;
            uint64_t value;

            FLOWMAP_SET(&map, ipv6_src);
            FLOWMAP_SET(&map, ipv6_dst);

            MINIFLOW_FOR_EACH_IN_FLOWMAP(value, flow, map) {
                hash = hash_add64(hash, value);
            }
        } else {
            hash = hash_add(hash, MINIFLOW_GET_U32(flow, nw_src));
            hash = hash_add(hash, MINIFLOW_GET_U32(flow, nw_dst));
        }
        /* Add both ports at once. */
        hash = hash_add(hash, MINIFLOW_GET_U32(flow, tp_src));
        hash = hash_finish(hash, 42); /* Arbitrary number. */
    }
    return hash;
}


flow_wc_map(const struct flow *flow, struct flowmap *map)
{
    /* Update this function whenever struct flow changes. */
    BUILD_ASSERT_DECL(FLOW_WC_SEQ == 35);

    flowmap_init(map);

    if (flow_tnl_dst_is_set(&flow->tunnel)) {
        FLOWMAP_SET__(map, tunnel, offsetof(struct flow_tnl, metadata));
        if (!(flow->tunnel.flags & FLOW_TNL_F_UDPIF)) {
            if (flow->tunnel.metadata.present.map) {
                FLOWMAP_SET(map, tunnel.metadata);
            }
        } else {
            FLOWMAP_SET(map, tunnel.metadata.present.len);
            FLOWMAP_SET__(map, tunnel.metadata.opts.gnv,
                          flow->tunnel.metadata.present.len);
        }
    }

    /* Metadata fields that can appear on packet input. */
    FLOWMAP_SET(map, skb_priority);
    FLOWMAP_SET(map, pkt_mark);
    FLOWMAP_SET(map, recirc_id);
    FLOWMAP_SET(map, dp_hash);
    FLOWMAP_SET(map, in_port);
    FLOWMAP_SET(map, dl_dst);
    FLOWMAP_SET(map, dl_src);
    FLOWMAP_SET(map, dl_type);
    FLOWMAP_SET(map, vlan_tci);
    FLOWMAP_SET(map, ct_state);
    FLOWMAP_SET(map, ct_zone);
    FLOWMAP_SET(map, ct_mark);
    FLOWMAP_SET(map, ct_label);

    /* Ethertype-dependent fields. */
    if (OVS_LIKELY(flow->dl_type == htons(ETH_TYPE_IP))) {
        FLOWMAP_SET(map, nw_src);
        FLOWMAP_SET(map, nw_dst);
        FLOWMAP_SET(map, nw_proto);
        FLOWMAP_SET(map, nw_frag);
        FLOWMAP_SET(map, nw_tos);
        FLOWMAP_SET(map, nw_ttl);
        FLOWMAP_SET(map, tp_src);
        FLOWMAP_SET(map, tp_dst);

        if (OVS_UNLIKELY(flow->nw_proto == IPPROTO_IGMP)) {
            FLOWMAP_SET(map, igmp_group_ip4);
        } else {
            FLOWMAP_SET(map, tcp_flags);
        }
    } else if (flow->dl_type == htons(ETH_TYPE_IPV6)) {
        FLOWMAP_SET(map, ipv6_src);
        FLOWMAP_SET(map, ipv6_dst);
        FLOWMAP_SET(map, ipv6_label);
        FLOWMAP_SET(map, nw_proto);
        FLOWMAP_SET(map, nw_frag);
        FLOWMAP_SET(map, nw_tos);
        FLOWMAP_SET(map, nw_ttl);
        FLOWMAP_SET(map, tp_src);
        FLOWMAP_SET(map, tp_dst);

        if (OVS_UNLIKELY(flow->nw_proto == IPPROTO_ICMPV6)) {
            FLOWMAP_SET(map, nd_target);
            FLOWMAP_SET(map, arp_sha);
            FLOWMAP_SET(map, arp_tha);
        } else {
            FLOWMAP_SET(map, tcp_flags);
        }
    } else if (eth_type_mpls(flow->dl_type)) {
        FLOWMAP_SET(map, mpls_lse);
    } else if (flow->dl_type == htons(ETH_TYPE_ARP) ||
               flow->dl_type == htons(ETH_TYPE_RARP)) {
        FLOWMAP_SET(map, nw_src);
        FLOWMAP_SET(map, nw_dst);
        FLOWMAP_SET(map, nw_proto);
        FLOWMAP_SET(map, arp_sha);
        FLOWMAP_SET(map, arp_tha);
    }
}


dp_packet_l4_size(const struct dp_packet *b)
{
    return b->l4_ofs != UINT16_MAX
        ? (const char *)dp_packet_tail(b) - (const char *)dp_packet_l4(b)
        - dp_packet_l2_pad_size(b)
        : 0;
}


static uint32_t esp_get_stc(ESPState *s)
{
    uint32_t dmalen;

    dmalen = s->wregs[ESP_TCLO];
    dmalen |= s->wregs[ESP_TCMID] << 8;
    dmalen |= s->wregs[ESP_TCHI] << 16;

    return dmalen;
}


static void sysbus_esp_init(Object *obj)
{
    SysBusESPState *sysbus = SYSBUS_ESP(obj);

    object_initialize_child(obj, "esp", &sysbus->esp, TYPE_ESP);
}


static bool esp_is_before_version_5(void *opaque, int version_id)
{
    ESPState *s = ESP(opaque);

    version_id = MIN(version_id, s->mig_version_id);
    return version_id < 5;
}


void esp_reg_write(ESPState *s, uint32_t saddr, uint64_t val)
{
    trace_esp_mem_writeb(saddr, s->wregs[saddr], val);
    switch (saddr) {
    case ESP_TCHI:
        s->tchi_written = true;
        /* fall through */
    case ESP_TCLO:
    case ESP_TCMID:
        s->rregs[ESP_RSTAT] &= ~STAT_TC;
        break;
    case ESP_FIFO:
        if (s->do_cmd) {
            esp_fifo_push(&s->cmdfifo, val);
        } else {
            esp_fifo_push(&s->fifo, val);
        }

        /* Non-DMA transfers raise an interrupt after every byte */
        if (s->rregs[ESP_CMD] == CMD_TI) {
            s->rregs[ESP_RINTR] |= INTR_FC | INTR_BS;
            esp_raise_irq(s);
        }
        break;
    case ESP_CMD:
        s->rregs[saddr] = val;
        if (val & CMD_DMA) {
            s->dma = 1;
            /* Reload DMA counter.  */
            if (esp_get_stc(s) == 0) {
                esp_set_tc(s, 0x10000);
            } else {
                esp_set_tc(s, esp_get_stc(s));
            }
        } else {
            s->dma = 0;
        }
        switch (val & CMD_CMD) {
        case CMD_NOP:
            trace_esp_mem_writeb_cmd_nop(val);
            break;
        case CMD_FLUSH:
            trace_esp_mem_writeb_cmd_flush(val);
            fifo8_reset(&s->fifo);
            break;
        case CMD_RESET:
            trace_esp_mem_writeb_cmd_reset(val);
            esp_soft_reset(s);
            break;
        case CMD_BUSRESET:
            trace_esp_mem_writeb_cmd_bus_reset(val);
            if (!(s->wregs[ESP_CFG1] & CFG1_RESREPT)) {
                s->rregs[ESP_RINTR] |= INTR_RST;
                esp_raise_irq(s);
            }
            break;
        case CMD_TI:
            trace_esp_mem_writeb_cmd_ti(val);
            handle_ti(s);
            break;
        case CMD_ICCS:
            trace_esp_mem_writeb_cmd_iccs(val);
            write_response(s);
            s->rregs[ESP_RINTR] |= INTR_FC;
            s->rregs[ESP_RSTAT] |= STAT_MI;
            break;
        case CMD_MSGACC:
            trace_esp_mem_writeb_cmd_msgacc(val);
            s->rregs[ESP_RINTR] |= INTR_DC;
            s->rregs[ESP_RSEQ] = 0;
            s->rregs[ESP_RFLAGS] = 0;
            esp_raise_irq(s);
            break;
        case CMD_PAD:
            trace_esp_mem_writeb_cmd_pad(val);
            s->rregs[ESP_RSTAT] = STAT_TC;
            s->rregs[ESP_RINTR] |= INTR_FC;
            s->rregs[ESP_RSEQ] = 0;
            break;
        case CMD_SATN:
            trace_esp_mem_writeb_cmd_satn(val);
            break;
        case CMD_RSTATN:
            trace_esp_mem_writeb_cmd_rstatn(val);
            break;
        case CMD_SEL:
            trace_esp_mem_writeb_cmd_sel(val);
            handle_s_without_atn(s);
            break;
        case CMD_SELATN:
            trace_esp_mem_writeb_cmd_selatn(val);
            handle_satn(s);
            break;
        case CMD_SELATNS:
            trace_esp_mem_writeb_cmd_selatns(val);
            handle_satn_stop(s);
            break;
        case CMD_ENSEL:
            trace_esp_mem_writeb_cmd_ensel(val);
            s->rregs[ESP_RINTR] = 0;
            break;
        case CMD_DISSEL:
            trace_esp_mem_writeb_cmd_dissel(val);
            s->rregs[ESP_RINTR] = 0;
            esp_raise_irq(s);
            break;
        default:
            trace_esp_error_unhandled_command(val);
            break;
        }
        break;
    case ESP_WBUSID ... ESP_WSYNO:
        break;
    case ESP_CFG1:
    case ESP_CFG2: case ESP_CFG3:
    case ESP_RES3: case ESP_RES4:
        s->rregs[saddr] = val;
        break;
    case ESP_WCCF ... ESP_WTEST:
        break;
    default:
        trace_esp_error_invalid_write(val, saddr);
        return;
    }
    s->wregs[saddr] = val;
}


static void do_busid_cmd(ESPState *s, uint8_t busid)
{
    uint32_t cmdlen;
    int32_t datalen;
    int lun;
    SCSIDevice *current_lun;
    uint8_t buf[ESP_CMDFIFO_SZ];

    trace_esp_do_busid_cmd(busid);
    lun = busid & 7;
    cmdlen = fifo8_num_used(&s->cmdfifo);
    if (!cmdlen || !s->current_dev) {
        return;
    }
    esp_fifo_pop_buf(&s->cmdfifo, buf, cmdlen);

    current_lun = scsi_device_find(&s->bus, 0, s->current_dev->id, lun);
    s->current_req = scsi_req_new(current_lun, 0, lun, buf, s);
    datalen = scsi_req_enqueue(s->current_req);
    s->ti_size = datalen;
    fifo8_reset(&s->cmdfifo);
    if (datalen != 0) {
        s->rregs[ESP_RSTAT] = STAT_TC;
        s->rregs[ESP_RSEQ] = SEQ_CD;
        s->ti_cmd = 0;
        esp_set_tc(s, 0);
        if (datalen > 0) {
            /*
             * Switch to DATA IN phase but wait until initial data xfer is
             * complete before raising the command completion interrupt
             */
            s->data_in_ready = false;
            s->rregs[ESP_RSTAT] |= STAT_DI;
        } else {
            s->rregs[ESP_RSTAT] |= STAT_DO;
            s->rregs[ESP_RINTR] |= INTR_BS | INTR_FC;
            esp_raise_irq(s);
            esp_lower_drq(s);
        }
        scsi_req_continue(s->current_req);
        return;
    }
}


static void handle_s_without_atn(ESPState *s)
{
    int32_t cmdlen;

    if (s->dma && !s->dma_enabled) {
        s->dma_cb = handle_s_without_atn;
        return;
    }
    s->pdma_cb = s_without_satn_pdma_cb;
    cmdlen = get_cmd(s, ESP_CMDFIFO_SZ);
    if (cmdlen > 0) {
        s->cmdfifo_cdb_offset = 0;
        s->do_cmd = 0;
        do_busid_cmd(s, 0);
    } else if (cmdlen == 0) {
        s->do_cmd = 1;
        /* Target present, but no cmd yet - switch to command phase */
        s->rregs[ESP_RSEQ] = SEQ_CD;
        s->rregs[ESP_RSTAT] = STAT_CD;
    }
}


FLAC__bool FLAC__bitreader_skip_bits_no_crc(FLAC__BitReader *br, uint32_t bits)
{
	/*
	 * OPT: a faster implementation is possible but probably not that useful
	 * since this is only called a couple of times in the metadata readers.
	 */
	FLAC__ASSERT(0 != br);
	FLAC__ASSERT(0 != br->buffer);

	if(bits > 0) {
		const uint32_t n = br->consumed_bits & 7;
		uint32_t m;
		FLAC__uint32 x;

		if(n != 0) {
			m = flac_min(8-n, bits);
			if(!FLAC__bitreader_read_raw_uint32(br, &x, m))
				return false;
			bits -= m;
		}
		m = bits / 8;
		if(m > 0) {
			if(!FLAC__bitreader_skip_byte_block_aligned_no_crc(br, m))
				return false;
			bits %= 8;
		}
		if(bits > 0) {
			if(!FLAC__bitreader_read_raw_uint32(br, &x, bits))
				return false;
		}
	}

	return true;
}


FLAC__bool FLAC__bitreader_init(FLAC__BitReader *br, FLAC__BitReaderReadCallback rcb, void *cd)
{
	FLAC__ASSERT(0 != br);

	br->words = br->bytes = 0;
	br->consumed_words = br->consumed_bits = 0;
	br->capacity = FLAC__BITREADER_DEFAULT_CAPACITY;
	br->buffer = malloc(sizeof(brword) * br->capacity);
	if(br->buffer == 0)
		return false;
	br->read_callback = rcb;
	br->client_data = cd;

	return true;
}


inline FLAC__bool FLAC__bitreader_read_uint32_little_endian(FLAC__BitReader *br, FLAC__uint32 *val)
{
	FLAC__uint32 x8, x32 = 0;

	/* this doesn't need to be that fast as currently it is only used for vorbis comments */

	if(!FLAC__bitreader_read_raw_uint32(br, &x32, 8))
		return false;

	if(!FLAC__bitreader_read_raw_uint32(br, &x8, 8))
		return false;
	x32 |= (x8 << 8);

	if(!FLAC__bitreader_read_raw_uint32(br, &x8, 8))
		return false;
	x32 |= (x8 << 16);

	if(!FLAC__bitreader_read_raw_uint32(br, &x8, 8))
		return false;
	x32 |= (x8 << 24);

	*val = x32;
	return true;
}


srs_set_separator(srs_t *srs, char value)
{
	if (strchr(srs_separators, value) == NULL)
		return SRS_ESEPARATORINVALID;
	srs->separator = value;
	return SRS_SUCCESS;
}


srs_forward(srs_t *srs, char *buf, unsigned buflen,
				const char *sender, const char *alias)
{
	char	*senduser;
	char	*sendhost;
	char	*tmp;
	unsigned		 len;

	if (srs->noforward)
		return SRS_ENOTREWRITTEN;

	/* This is allowed to be a plain domain */
	while ((tmp = strchr(alias, '@')) != NULL)
		alias = tmp + 1;

	tmp = strchr(sender, '@');
	if (tmp == NULL)
		return SRS_ENOSENDERATSIGN;
	sendhost = tmp + 1;

	len = strlen(sender);

	if (! srs->alwaysrewrite) {
		if (strcasecmp(sendhost, alias) == 0) {
			if (strlen(sender) >= buflen)
				return SRS_EBUFTOOSMALL;
			strcpy(buf, sender);
			return SRS_SUCCESS;
		}
	}

	/* Reconstruct the whole show into our alloca() buffer. */
	senduser = alloca(len + 1);
	strcpy(senduser, sender);
	tmp = (senduser + (tmp - sender));
	sendhost = tmp + 1;
	*tmp = '\0';

	return srs_compile_guarded(srs, buf, buflen,
					sendhost, senduser, alias);
}


set_vertex_data(ImageRenderData *rd, const ImageRef *ref, const ImageRect *dest_rect) {
#define R(n, a, b) rd->vertices[n*4] = ref->src_rect.a; rd->vertices[n*4 + 1] = ref->src_rect.b; rd->vertices[n*4 + 2] = dest_rect->a; rd->vertices[n*4 + 3] = dest_rect->b;
        R(0, right, top); R(1, right, bottom); R(2, left, bottom); R(3, left, top);
#undef R
}


ref_outside_region(const ImageRef *ref, index_type margin_top, index_type margin_bottom) {
    return ref->start_row + ref->effective_num_rows <= margin_top || ref->start_row > (int32_t)margin_bottom;
}


inflate_png(GraphicsManager UNUSED *self, Image *img, uint8_t *buf, size_t bufsz) {
    png_read_data d = {.err_handler=png_error_handler};
    inflate_png_inner(&d, buf, bufsz);
    if (d.ok) {
        free_load_data(&img->load_data);
        img->load_data.buf = d.decompressed;
        img->load_data.buf_capacity = d.sz;
        img->load_data.buf_used = d.sz;
        img->load_data.data_sz = d.sz;
        img->width = d.width; img->height = d.height;
    }
    else free(d.decompressed);
    free(d.row_pointers);
    return d.ok;
}


free_load_data(LoadData *ld) {
    free(ld->buf); ld->buf_used = 0; ld->buf_capacity = 0;
    ld->buf = NULL;

    if (ld->mapped_file) munmap(ld->mapped_file, ld->mapped_file_sz);
    ld->mapped_file = NULL; ld->mapped_file_sz = 0;
}


new(PyTypeObject UNUSED *type, PyObject UNUSED *args, PyObject UNUSED *kwds) {
    PyObject *ans = (PyObject*)grman_alloc();
    if (ans == NULL) PyErr_NoMemory();
    return ans;
}


table_create(struct smtpd *conf, const char *backend, const char *name,
    const char *config)
{
	struct table		*t;
	struct table_backend	*tb;
	char			 path[LINE_MAX];
	size_t			 n;
	struct stat		 sb;

	if (name && table_find(conf, name))
		fatalx("table_create: table \"%s\" already defined", name);

	if ((tb = table_backend_lookup(backend)) == NULL) {
		if ((size_t)snprintf(path, sizeof(path), PATH_LIBEXEC"/table-%s",
			backend) >= sizeof(path)) {
			fatalx("table_create: path too long \""
			    PATH_LIBEXEC"/table-%s\"", backend);
		}
		if (stat(path, &sb) == 0) {
			tb = table_backend_lookup("proc");
			(void)strlcpy(path, backend, sizeof(path));
			if (config) {
				(void)strlcat(path, ":", sizeof(path));
				if (strlcat(path, config, sizeof(path))
				    >= sizeof(path))
					fatalx("table_create: config file path too long");
			}
			config = path;
		}
	}

	if (tb == NULL)
		fatalx("table_create: backend \"%s\" does not exist", backend);

	t = xcalloc(1, sizeof(*t));
	t->t_backend = tb;

	if (config) {
		if (strlcpy(t->t_config, config, sizeof t->t_config)
		    >= sizeof t->t_config)
			fatalx("table_create: table config \"%s\" too large",
			    t->t_config);
	}

	if (strcmp(tb->name, "static") != 0)
		t->t_type = T_DYNAMIC;

	if (name == NULL)
		(void)snprintf(t->t_name, sizeof(t->t_name), "<dynamic:%u>",
		    last_table_id++);
	else {
		n = strlcpy(t->t_name, name, sizeof(t->t_name));
		if (n >= sizeof(t->t_name))
			fatalx("table_create: table name too long");
	}

	dict_set(conf->sc_tables_dict, t->t_name, t);

	return (t);
}


lka_report_smtp_tx_commit(const char *direction, struct timeval *tv, uint64_t reqid, uint32_t msgid, size_t msgsz)
{
	report_smtp_broadcast(reqid, direction, tv, "tx-commit", "%08x|%zd\n",
	    msgid, msgsz);
}


filter_check_fcrdns(struct filter *filter, int fcrdns)
{
	int	ret = 0;

	if (!filter->config->fcrdns)
		return 0;

	ret = fcrdns == 1;
	return filter->config->not_fcrdns < 0 ? !ret : ret;
}


filter_result_disconnect(uint64_t reqid, const char *message)
{
	m_create(p_pony, IMSG_FILTER_SMTP_PROTOCOL, 0, 0, -1);
	m_add_id(p_pony, reqid);
	m_add_int(p_pony, FILTER_DISCONNECT);
	m_add_string(p_pony, message);
	m_close(p_pony);
}


lka_report_smtp_link_connect(const char *direction, struct timeval *tv, uint64_t reqid, const char *rdns,
    int fcrdns,
    const struct sockaddr_storage *ss_src,
    const struct sockaddr_storage *ss_dest)
{
	struct filter_session *fs;
	char	src[NI_MAXHOST + 5];
	char	dest[NI_MAXHOST + 5];
	uint16_t	src_port = 0;
	uint16_t	dest_port = 0;
	const char     *fcrdns_str;

	if (ss_src->ss_family == AF_INET)
		src_port = ntohs(((const struct sockaddr_in *)ss_src)->sin_port);
	else if (ss_src->ss_family == AF_INET6)
		src_port = ntohs(((const struct sockaddr_in6 *)ss_src)->sin6_port);

	if (ss_dest->ss_family == AF_INET)
		dest_port = ntohs(((const struct sockaddr_in *)ss_dest)->sin_port);
	else if (ss_dest->ss_family == AF_INET6)
		dest_port = ntohs(((const struct sockaddr_in6 *)ss_dest)->sin6_port);

	if (strcmp(ss_to_text(ss_src), "local") == 0) {
		(void)snprintf(src, sizeof src, "unix:%s", SMTPD_SOCKET);
		(void)snprintf(dest, sizeof dest, "unix:%s", SMTPD_SOCKET);
	} else {
		(void)snprintf(src, sizeof src, "%s:%d", ss_to_text(ss_src), src_port);
		(void)snprintf(dest, sizeof dest, "%s:%d", ss_to_text(ss_dest), dest_port);
	}

	switch (fcrdns) {
	case 1:
		fcrdns_str = "pass";
		break;
	case 0:
		fcrdns_str = "fail";
		break;
	default:
		fcrdns_str = "error";
		break;
	}

	fs = tree_xget(&sessions, reqid);
	fs->rdns = xstrdup(rdns);
	fs->fcrdns = fcrdns;
	fs->ss_src = *ss_src;
	fs->ss_dest = *ss_dest;

	report_smtp_broadcast(reqid, direction, tv, "link-connect",
	    "%s|%s|%s|%s\n", rdns, fcrdns_str, src, dest);
}


lka_report_smtp_tx_reset(const char *direction, struct timeval *tv, uint64_t reqid, uint32_t msgid)
{
	report_smtp_broadcast(reqid, direction, tv, "tx-reset", "%08x\n",
	    msgid);
}


filter_builtins_notimpl(struct filter_session *fs, struct filter *filter, uint64_t reqid, const char *param)
{
	return 0;
}


int WavpackPackInit (WavpackContext *wpc)
{
    if (wpc->metabytes > 16384)             // 16384 bytes still leaves plenty of room for audio
        write_metadata_block (wpc);         //  in this block (otherwise write a special one)

    // The default block size is a compromise. Longer blocks provide better encoding efficiency,
    // but longer blocks adversely affect memory requirements and seeking performance. For WavPack
    // version 5.0, the default block sizes have been reduced by half from the previous version,
    // but the difference in encoding efficiency will generally be less than 0.1 percent.

    if (wpc->dsd_multiplier) {
        wpc->block_samples = (wpc->config.sample_rate % 7) ? 48000 : 44100;

        if (wpc->config.flags & CONFIG_HIGH_FLAG)
            wpc->block_samples /= 2;

        if (wpc->config.num_channels == 1)
            wpc->block_samples *= 2;

        while (wpc->block_samples > 12000 && (int64_t) wpc->block_samples * wpc->config.num_channels > 300000)
            wpc->block_samples /= 2;
    }
    else {
        int divisor = (wpc->config.flags & CONFIG_HIGH_FLAG) ? 2 : 4;

        while (wpc->config.sample_rate % divisor)
            divisor--;

        wpc->block_samples = wpc->config.sample_rate / divisor;

        while (wpc->block_samples > 12000 && (int64_t) wpc->block_samples * wpc->config.num_channels > 75000)
            wpc->block_samples /= 2;

        while ((int64_t) wpc->block_samples * wpc->config.num_channels < 20000)
            wpc->block_samples *= 2;
    }

    if (wpc->config.block_samples) {
        if ((wpc->config.flags & CONFIG_MERGE_BLOCKS) &&
            wpc->block_samples > (uint32_t) wpc->config.block_samples) {
                wpc->block_boundary = wpc->config.block_samples;
                wpc->block_samples /= wpc->config.block_samples;
                wpc->block_samples *= wpc->config.block_samples;
        }
        else
            wpc->block_samples = wpc->config.block_samples;
    }

    wpc->ave_block_samples = wpc->block_samples;
    wpc->max_samples = wpc->block_samples + (wpc->block_samples >> 1);

    for (wpc->current_stream = 0; wpc->current_stream < wpc->num_streams; wpc->current_stream++) {
        WavpackStream *wps = wpc->streams [wpc->current_stream];

        wps->sample_buffer = malloc (wpc->max_samples * (wps->wphdr.flags & MONO_FLAG ? 4 : 8));

#ifdef ENABLE_DSD
        if (wps->wphdr.flags & DSD_FLAG)
            pack_dsd_init (wpc);
        else
#endif
            pack_init (wpc);
    }

    return TRUE;
}


static int create_riff_header (WavpackContext *wpc, int64_t total_samples, void *outbuffer)
{
    int do_rf64 = 0, write_junk = 1;
    ChunkHeader ds64hdr, datahdr, fmthdr;
    char *outptr = outbuffer;
    RiffChunkHeader riffhdr;
    DS64Chunk ds64_chunk;
    JunkChunk junkchunk;
    WaveHeader wavhdr;

    int64_t total_data_bytes, total_riff_bytes;
    int32_t channel_mask = wpc->config.channel_mask;
    int32_t sample_rate = wpc->config.sample_rate;
    int bytes_per_sample = wpc->config.bytes_per_sample;
    int bits_per_sample = wpc->config.bits_per_sample;
    int format = (wpc->config.float_norm_exp) ? 3 : 1;
    int num_channels = wpc->config.num_channels;
    int wavhdrsize = 16;

    wpc->riff_header_created = TRUE;

    if (format == 3 && wpc->config.float_norm_exp != 127) {
        strcpy (wpc->error_message, "can't create valid RIFF wav header for non-normalized floating data!");
        return FALSE;
    }

    if (total_samples == -1)
        total_samples = 0x7ffff000 / (bytes_per_sample * num_channels);

    total_data_bytes = total_samples * bytes_per_sample * num_channels;

    if (total_data_bytes > 0xff000000) {
        write_junk = 0;
        do_rf64 = 1;
    }

    CLEAR (wavhdr);

    wavhdr.FormatTag = format;
    wavhdr.NumChannels = num_channels;
    wavhdr.SampleRate = sample_rate;
    wavhdr.BytesPerSecond = sample_rate * num_channels * bytes_per_sample;
    wavhdr.BlockAlign = bytes_per_sample * num_channels;
    wavhdr.BitsPerSample = bits_per_sample;

    if (num_channels > 2 || channel_mask != 0x5 - num_channels) {
        wavhdrsize = sizeof (wavhdr);
        wavhdr.cbSize = 22;
        wavhdr.ValidBitsPerSample = bits_per_sample;
        wavhdr.SubFormat = format;
        wavhdr.ChannelMask = channel_mask;
        wavhdr.FormatTag = 0xfffe;
        wavhdr.BitsPerSample = bytes_per_sample * 8;
        wavhdr.GUID [4] = 0x10;
        wavhdr.GUID [6] = 0x80;
        wavhdr.GUID [9] = 0xaa;
        wavhdr.GUID [11] = 0x38;
        wavhdr.GUID [12] = 0x9b;
        wavhdr.GUID [13] = 0x71;
    }

    memcpy (riffhdr.ckID, do_rf64 ? "RF64" : "RIFF", sizeof (riffhdr.ckID));
    memcpy (riffhdr.formType, "WAVE", sizeof (riffhdr.formType));
    total_riff_bytes = sizeof (riffhdr) + wavhdrsize + sizeof (datahdr) + total_data_bytes + wpc->riff_trailer_bytes;
    if (do_rf64) total_riff_bytes += sizeof (ds64hdr) + sizeof (ds64_chunk);
    if (write_junk) total_riff_bytes += sizeof (junkchunk);
    memcpy (fmthdr.ckID, "fmt ", sizeof (fmthdr.ckID));
    memcpy (datahdr.ckID, "data", sizeof (datahdr.ckID));
    fmthdr.ckSize = wavhdrsize;

    if (write_junk) {
        CLEAR (junkchunk);
        memcpy (junkchunk.ckID, "junk", sizeof (junkchunk.ckID));
        junkchunk.ckSize = sizeof (junkchunk) - 8;
        WavpackNativeToLittleEndian (&junkchunk, ChunkHeaderFormat);
    }

    if (do_rf64) {
        memcpy (ds64hdr.ckID, "ds64", sizeof (ds64hdr.ckID));
        ds64hdr.ckSize = sizeof (ds64_chunk);
        CLEAR (ds64_chunk);
        ds64_chunk.riffSize64 = total_riff_bytes;
        ds64_chunk.dataSize64 = total_data_bytes;
        ds64_chunk.sampleCount64 = total_samples;
        riffhdr.ckSize = (uint32_t) -1;
        datahdr.ckSize = (uint32_t) -1;
        WavpackNativeToLittleEndian (&ds64hdr, ChunkHeaderFormat);
        WavpackNativeToLittleEndian (&ds64_chunk, DS64ChunkFormat);
    }
    else {
        riffhdr.ckSize = (uint32_t) total_riff_bytes;
        datahdr.ckSize = (uint32_t) total_data_bytes;
    }

    WavpackNativeToLittleEndian (&riffhdr, ChunkHeaderFormat);
    WavpackNativeToLittleEndian (&fmthdr, ChunkHeaderFormat);
    WavpackNativeToLittleEndian (&wavhdr, WaveHeaderFormat);
    WavpackNativeToLittleEndian (&datahdr, ChunkHeaderFormat);

    // write the RIFF chunks up to just before the data starts

    outptr = (char *) memcpy (outptr, &riffhdr, sizeof (riffhdr)) + sizeof (riffhdr);

    if (do_rf64) {
        outptr = (char *) memcpy (outptr, &ds64hdr, sizeof (ds64hdr)) + sizeof (ds64hdr);
        outptr = (char *) memcpy (outptr, &ds64_chunk, sizeof (ds64_chunk)) + sizeof (ds64_chunk);
    }

    if (write_junk)
        outptr = (char *) memcpy (outptr, &junkchunk, sizeof (junkchunk)) + sizeof (junkchunk);

    outptr = (char *) memcpy (outptr, &fmthdr, sizeof (fmthdr)) + sizeof (fmthdr);
    outptr = (char *) memcpy (outptr, &wavhdr, wavhdrsize) + wavhdrsize;
    outptr = (char *) memcpy (outptr, &datahdr, sizeof (datahdr)) + sizeof (datahdr);

    return (int)(outptr - (char *) outbuffer);
}


static VALUE from_document(int argc, VALUE *argv, VALUE klass)
{
  VALUE document;
  VALUE parse_options;
  xmlDocPtr doc;
  xmlRelaxNGParserCtxtPtr ctx;
  xmlRelaxNGPtr schema;
  VALUE errors;
  VALUE rb_schema;
  int scanned_args = 0;

  scanned_args = rb_scan_args(argc, argv, "11", &document, &parse_options);

  Data_Get_Struct(document, xmlDoc, doc);
  doc = doc->doc; /* In case someone passes us a node. ugh. */

  if (scanned_args == 1) {
    parse_options = rb_const_get(rb_const_get(mNokogiriXml, rb_intern("ParseOptions")), rb_intern("DEFAULT_SCHEMA"));
  }

  ctx = xmlRelaxNGNewDocParserCtxt(doc);

  errors = rb_ary_new();
  xmlSetStructuredErrorFunc((void *)errors, Nokogiri_error_array_pusher);

#ifdef HAVE_XMLRELAXNGSETPARSERSTRUCTUREDERRORS
  xmlRelaxNGSetParserStructuredErrors(
    ctx,
    Nokogiri_error_array_pusher,
    (void *)errors
  );
#endif

  schema = xmlRelaxNGParse(ctx);

  xmlSetStructuredErrorFunc(NULL, NULL);
  xmlRelaxNGFreeParserCtxt(ctx);

  if(NULL == schema) {
    xmlErrorPtr error = xmlGetLastError();
    if(error)
      Nokogiri_error_raise(NULL, error);
    else
      rb_raise(rb_eRuntimeError, "Could not parse document");

    return Qnil;
  }

  rb_schema = Data_Wrap_Struct(klass, 0, dealloc, schema);
  rb_iv_set(rb_schema, "@errors", errors);
  rb_iv_set(rb_schema, "@parse_options", parse_options);

  return rb_schema;
}


static VALUE validate_document(VALUE self, VALUE document)
{
  xmlDocPtr doc;
  xmlRelaxNGPtr schema;
  VALUE errors;
  xmlRelaxNGValidCtxtPtr valid_ctxt;

  Data_Get_Struct(self, xmlRelaxNG, schema);
  Data_Get_Struct(document, xmlDoc, doc);

  errors = rb_ary_new();

  valid_ctxt = xmlRelaxNGNewValidCtxt(schema);

  if(NULL == valid_ctxt) {
    /* we have a problem */
    rb_raise(rb_eRuntimeError, "Could not create a validation context");
  }

#ifdef HAVE_XMLRELAXNGSETVALIDSTRUCTUREDERRORS
  xmlRelaxNGSetValidStructuredErrors(
    valid_ctxt,
    Nokogiri_error_array_pusher,
    (void *)errors
  );
#endif

  xmlRelaxNGValidateDoc(valid_ctxt, doc);

  xmlRelaxNGFreeValidCtxt(valid_ctxt);

  return errors;
}


static BlockAIOCB *iscsi_aio_ioctl(BlockDriverState *bs,
        unsigned long int req, void *buf,
        BlockCompletionFunc *cb, void *opaque)
{
    IscsiLun *iscsilun = bs->opaque;
    struct iscsi_context *iscsi = iscsilun->iscsi;
    struct iscsi_data data;
    IscsiAIOCB *acb;

    acb = qemu_aio_get(&iscsi_aiocb_info, bs, cb, opaque);

    acb->iscsilun = iscsilun;
    acb->bh          = NULL;
    acb->status      = -EINPROGRESS;
    acb->ioh         = buf;
    acb->cancelled   = false;

    if (req != SG_IO) {
        iscsi_ioctl_handle_emulated(acb, req, buf);
        return &acb->common;
    }

    if (acb->ioh->cmd_len > SCSI_CDB_MAX_SIZE) {
        error_report("iSCSI: ioctl error CDB exceeds max size (%d > %d)",
                     acb->ioh->cmd_len, SCSI_CDB_MAX_SIZE);
        qemu_aio_unref(acb);
        return NULL;
    }

    acb->task = malloc(sizeof(struct scsi_task));
    if (acb->task == NULL) {
        error_report("iSCSI: Failed to allocate task for scsi command. %s",
                     iscsi_get_error(iscsi));
        qemu_aio_unref(acb);
        return NULL;
    }
    memset(acb->task, 0, sizeof(struct scsi_task));

    switch (acb->ioh->dxfer_direction) {
    case SG_DXFER_TO_DEV:
        acb->task->xfer_dir = SCSI_XFER_WRITE;
        break;
    case SG_DXFER_FROM_DEV:
        acb->task->xfer_dir = SCSI_XFER_READ;
        break;
    default:
        acb->task->xfer_dir = SCSI_XFER_NONE;
        break;
    }

    acb->task->cdb_size = acb->ioh->cmd_len;
    memcpy(&acb->task->cdb[0], acb->ioh->cmdp, acb->ioh->cmd_len);
    acb->task->expxferlen = acb->ioh->dxfer_len;

    data.size = 0;
    qemu_mutex_lock(&iscsilun->mutex);
    if (acb->task->xfer_dir == SCSI_XFER_WRITE) {
        if (acb->ioh->iovec_count == 0) {
            data.data = acb->ioh->dxferp;
            data.size = acb->ioh->dxfer_len;
        } else {
            scsi_task_set_iov_out(acb->task,
                                 (struct scsi_iovec *) acb->ioh->dxferp,
                                 acb->ioh->iovec_count);
        }
    }

    if (iscsi_scsi_command_async(iscsi, iscsilun->lun, acb->task,
                                 iscsi_aio_ioctl_cb,
                                 (data.size > 0) ? &data : NULL,
                                 acb) != 0) {
        qemu_mutex_unlock(&iscsilun->mutex);
        scsi_free_scsi_task(acb->task);
        qemu_aio_unref(acb);
        return NULL;
    }

    /* tell libiscsi to read straight into the buffer we got from ioctl */
    if (acb->task->xfer_dir == SCSI_XFER_READ) {
        if (acb->ioh->iovec_count == 0) {
            scsi_task_add_data_in_buffer(acb->task,
                                         acb->ioh->dxfer_len,
                                         acb->ioh->dxferp);
        } else {
            scsi_task_set_iov_in(acb->task,
                                 (struct scsi_iovec *) acb->ioh->dxferp,
                                 acb->ioh->iovec_count);
        }
    }

    iscsi_set_events(iscsilun);
    qemu_mutex_unlock(&iscsilun->mutex);

    return &acb->common;
}


static void iscsi_retry_timer_expired(void *opaque)
{
    struct IscsiTask *iTask = opaque;
    iTask->complete = 1;
    if (iTask->co) {
        aio_co_wake(iTask->co);
    }
}


static void iscsi_xcopy_data(struct iscsi_data *data,
                             IscsiLun *src, int64_t src_lba,
                             IscsiLun *dst, int64_t dst_lba,
                             uint16_t num_blocks)
{
    uint8_t *buf;
    const int src_offset = XCOPY_DESC_OFFSET;
    const int dst_offset = XCOPY_DESC_OFFSET + IDENT_DESCR_TGT_DESCR_SIZE;
    const int seg_offset = dst_offset + IDENT_DESCR_TGT_DESCR_SIZE;

    data->size = XCOPY_DESC_OFFSET +
                 IDENT_DESCR_TGT_DESCR_SIZE * 2 +
                 XCOPY_BLK2BLK_SEG_DESC_SIZE;
    data->data = g_malloc0(data->size);
    buf = data->data;

    /* Initialise the parameter list header */
    iscsi_xcopy_populate_header(buf, 1, 0, 2 /* LIST_ID_USAGE_DISCARD */,
                                0, 2 * IDENT_DESCR_TGT_DESCR_SIZE,
                                XCOPY_BLK2BLK_SEG_DESC_SIZE,
                                0);

    /* Initialise CSCD list with one src + one dst descriptor */
    iscsi_populate_target_desc(&buf[src_offset], src);
    iscsi_populate_target_desc(&buf[dst_offset], dst);

    /* Initialise one segment descriptor */
    iscsi_xcopy_populate_desc(&buf[seg_offset], 0, 0, 0, 1, num_blocks,
                              src_lba, dst_lba);
}