Repository: JuliaLang/julia
File: src/subtype.c (C)

Code before:
            vb.constraintkind = vb.concrete ? 1 : 2;
        else if (u->var->lb != jl_bottom_type)
            vb.constraintkind = 2;
        else if (vb.occurs_cov && noinv)
            vb.constraintkind = 1;
        int reintersection = constraint1 != vb.constraintkind || vb.concrete;
        if (reintersection) {
            if (constraint1 == 1) {
                vb.lb = vb.var->lb;
                vb.ub = vb.var->ub;
            }
            restore_env(e, &se, vb.constraintkind == 1 ? 1 : 0);
            vb.occurs_cov = vb.occurs_inv = 0;
            res = intersect_unionall_(t, u, e, R, param, &vb);
        }
    }
    else if (res == jl_bottom_type && vb.constraintkind == 1 &&
             vb.occurs_cov > 1) {
        // vb.constraintkind == 1 implies occurs_inv == 0, together with occurs_cov > 1
        // means vb is diagonal
        // res == jl_bottom_type may be a false negative if vb.ub contains different
        // Type{X} variables, which may have a concrete common super type (queried via w)
        jl_value_t *w = widen_Type_union_pointwise(vb.ub);
        if (w != vb.ub) {
            vb.ub = w;
            if (jl_subtype(vb.ub, u->var->ub)) {
                vb.lb = u->var->lb;
                vb.concrete = 0;
                vb.widen_to_kind = 1;
                vb.occurs_cov = vb.occurs_inv = 0;
                restore_env(e, &se, 1);
                res = intersect_unionall_(t, u, e, R, param, &vb);
                if (res != jl_bottom_type) {
                    vb.lb = u->var->lb;
                    vb.constraintkind = 0;
                    vb.concrete = 0;
                    vb.widen_to_kind = 0;
                    vb.occurs_cov = vb.occurs_inv = 0;
                    restore_env(e, &se, 0);
                    res = intersect_unionall_(t, u, e, R, param, &vb);
                }
            }
        }
    }
    free_env(&se);
    JL_GC_POP();
    return res;
}

static jl_value_t *intersect_invariant(jl_value_t *x, jl_value_t *y, jl_stenv_t *e);


Review comment (suggestion):
I'm suspicious of a call to `jl_subtype` happening from this deep inside the subtyping algorithm, ignoring all the current status of the algorithm

Code after:
    else if (jl_is_uniontype(v)) {
        return always_occurs_cov(((jl_uniontype_t*)v)->a, var, param) &&
               always_occurs_cov(((jl_uniontype_t*)v)->b, var, param);
    }
    else if (jl_is_unionall(v)) {
        jl_unionall_t *ua = (jl_unionall_t*)v;
        return ua->var != var && (
            always_occurs_cov(ua->var->ub, var, 0) ||
            always_occurs_cov(ua->body, var, param));
    }
    else if (jl_is_vararg(v)) {
        jl_vararg_t *vm = (jl_vararg_t*)v;
        return vm->T && always_occurs_cov(vm->T, var, param);
    }
    else if (jl_is_datatype(v)) {
        int nparam = jl_is_tuple_type(v) ? 1 : param;
        for (size_t i = 0; i < jl_nparams(v); i++) {
            if (always_occurs_cov(jl_tparam(v, i), var, nparam))
                return 1;
        }
    }
    return 0;
}

static jl_value_t *intersect_unionall(jl_value_t *t, jl_unionall_t *u, jl_stenv_t *e, int8_t R, int param)
{
    jl_value_t *res = NULL;
    jl_savedenv_t se;
    jl_varbinding_t vb = { u->var, u->var->lb, u->var->ub, R, 0, 0, 0, 0, 0, 0, 0, 0, 0,
                           e->invdepth, NULL, e->vars };
    JL_GC_PUSH4(&res, &vb.lb, &vb.ub, &vb.innervars);
    save_env(e, &se, 1);
    int noinv = !var_occurs_invariant(u->body, u->var);
    if (is_leaf_typevar(u->var) && noinv && always_occurs_cov(u->body, u->var, param))
        vb.constraintkind = 1;
    res = intersect_unionall_(t, u, e, R, param, &vb);
    vb.intersected = 1;
    if (vb.limited) {
        // if the environment got too big, avoid tree recursion and propagate the flag
        if (e->vars)
            e->vars->limited = 1;
    }
    else if (res != jl_bottom_type) {
        int constraint1 = vb.constraintkind;
        if (vb.concrete || vb.occurs_inv>1 || (vb.occurs_inv && vb.occurs_cov))
            vb.constraintkind = vb.concrete ? 1 : 2;
        else if (u->var->lb != jl_bottom_type)
            vb.constraintkind = 2;
        else if (vb.occurs_cov && noinv)
            vb.constraintkind = 1;
        int reintersection = constraint1 != vb.constraintkind || vb.concrete;