"pahole" tells me here the change does affect the layout quite a bit. Maybe we can use different types for the archs. E.g. something like
<pre>
#ifdef __tile__
typedef unsigned int FlowRefcntInt
#else
typedef unsigned short FlowRefcntInt
#endif
</pre>
Original:
<pre>
struct Flow_ {
        FlowAddress                src;                  /*     0    16 */
        FlowAddress                dst;                  /*    16    16 */
        union {
                Port               sp;                   /*           2 */
                uint8_t            type;                 /*           1 */
        };                                               /*    32     2 */
        union {
                Port               dp;                   /*           2 */
                uint8_t            code;                 /*           1 */
        };                                               /*    34     2 */
        uint8_t                    proto;                /*    36     1 */
        uint8_t                    recursion_level;      /*    37     1 */
        uint16_t                   vlan_id[2];           /*    38     4 */
        short unsigned int         use_cnt_sc_atomic__;  /*    42     2 */
        int                        autofp_tmqh_flow_qid_sc_atomic__; /*    44     4 */
        uint32_t                   probing_parser_toserver_al_proto_masks; /*    48     4 */
        uint32_t                   probing_parser_toclient_al_proto_masks; /*    52     4 */
        uint32_t                   flags;                /*    56     4 */
        int32_t                    lastts_sec;           /*    60     4 */
        /* --- cacheline 1 boundary (64 bytes) --- */
        pthread_mutex_t            m;                    /*    64    40 */
        void *                     protoctx;             /*   104     8 */
        uint8_t                    protomap;             /*   112     1 */
        uint8_t                    pad0;                 /*   113     1 */
        uint16_t                   alproto;              /*   114     2 */
        uint32_t                   de_ctx_id;            /*   116     4 */
        void *                     alparser;             /*   120     8 */
        /* --- cacheline 2 boundary (128 bytes) --- */
        void *                     alstate;              /*   128     8 */
        struct DetectEngineState_ * de_state;            /*   136     8 */
        struct SigGroupHead_ *     sgh_toclient;         /*   144     8 */
        struct SigGroupHead_ *     sgh_toserver;         /*   152     8 */
        GenericVar *               flowvar;              /*   160     8 */
        pthread_mutex_t            de_state_m;           /*   168    40 */
        /* --- cacheline 3 boundary (192 bytes) was 16 bytes ago --- */
        struct Flow_ *             hnext;                /*   208     8 */
        struct Flow_ *             hprev;                /*   216     8 */
        struct FlowBucket_ *       fb;                   /*   224     8 */
        struct Flow_ *             lnext;                /*   232     8 */
        struct Flow_ *             lprev;                /*   240     8 */
        struct timeval             startts;              /*   248    16 */
        /* --- cacheline 4 boundary (256 bytes) was 8 bytes ago --- */
        uint32_t                   todstpktcnt;          /*   264     4 */
        uint32_t                   tosrcpktcnt;          /*   268     4 */
        uint64_t                   bytecnt;              /*   272     8 */

        /* size: 280, cachelines: 5, members: 35 */
        /* last cacheline: 24 bytes */
};
</pre>

This patch:
<pre>
struct Flow_ {
        FlowAddress                src;                  /*     0    16 */
        FlowAddress                dst;                  /*    16    16 */
        union {
                Port               sp;                   /*           2 */
                uint8_t            type;                 /*           1 */
        };                                               /*    32     2 */
        union {
                Port               dp;                   /*           2 */
                uint8_t            code;                 /*           1 */
        };                                               /*    34     2 */
        uint8_t                    proto;                /*    36     1 */
        uint8_t                    recursion_level;      /*    37     1 */
        uint16_t                   vlan_id[2];           /*    38     4 */

        /* XXX 2 bytes hole, try to pack */

        unsigned int               use_cnt_sc_atomic__;  /*    44     4 */
        int                        autofp_tmqh_flow_qid_sc_atomic__; /*    48     4 */
        uint32_t                   probing_parser_toserver_al_proto_masks; /*    52     4 */
        uint32_t                   probing_parser_toclient_al_proto_masks; /*    56     4 */
        uint32_t                   flags;                /*    60     4 */
        /* --- cacheline 1 boundary (64 bytes) --- */
        int32_t                    lastts_sec;           /*    64     4 */

        /* XXX 4 bytes hole, try to pack */

        pthread_mutex_t            m;                    /*    72    40 */
        void *                     protoctx;             /*   112     8 */
        uint8_t                    protomap;             /*   120     1 */
        uint8_t                    pad0;                 /*   121     1 */
        uint16_t                   alproto;              /*   122     2 */
        uint32_t                   de_ctx_id;            /*   124     4 */
        /* --- cacheline 2 boundary (128 bytes) --- */
        void *                     alparser;             /*   128     8 */
        void *                     alstate;              /*   136     8 */
        struct DetectEngineState_ * de_state;            /*   144     8 */
        struct SigGroupHead_ *     sgh_toclient;         /*   152     8 */
        struct SigGroupHead_ *     sgh_toserver;         /*   160     8 */
        GenericVar *               flowvar;              /*   168     8 */
        pthread_mutex_t            de_state_m;           /*   176    40 */
        /* --- cacheline 3 boundary (192 bytes) was 24 bytes ago --- */
        struct Flow_ *             hnext;                /*   216     8 */
        struct Flow_ *             hprev;                /*   224     8 */
        struct FlowBucket_ *       fb;                   /*   232     8 */
        struct Flow_ *             lnext;                /*   240     8 */
        struct Flow_ *             lprev;                /*   248     8 */
        /* --- cacheline 4 boundary (256 bytes) --- */
        struct timeval             startts;              /*   256    16 */
        uint32_t                   todstpktcnt;          /*   272     4 */
        uint32_t                   tosrcpktcnt;          /*   276     4 */
        uint64_t                   bytecnt;              /*   280     8 */

        /* size: 288, cachelines: 5, members: 35 */
        /* sum members: 282, holes: 2, sum holes: 6 */
        /* last cacheline: 32 bytes */
};
</pre>