Commit message: Fully incorporate the code from Python 3.7.2 (#78)

This is a full port, following the recipe in update_process.md. I've also tried to keep the recipe up to date and improved the automation (see tools/script). I haven't cleaned up the commits. As of #77 there are a few tests that sanity-check this (though it's far from a full test suite), and they're run by Travis-CI and AppVeyor.
CVE: CVE-2019-19275
CWE: CWE-125
Vulnerability description: typed_ast 1.3.0 and 1.3.1 has an ast_for_arguments out-of-bounds read. An attacker with the ability to cause a Python interpreter to parse Python source (but not necessarily execute it) may be able to crash the interpreter process. This could be a concern, for example, in a web-based service that parses (but does not execute) Python code. (This issue also affected certain Python 3.8.0-alpha prereleases.)

Project: typed_ast
File: Python-ast.c

ast2obj_withitem(void* _o)
{
    withitem_ty o = (withitem_ty)_o;
    PyObject *result = NULL, *value = NULL;
    if (!o) {
        Py_RETURN_NONE;
    }

    result = PyType_GenericNew(withitem_type, NULL, NULL);
    if (!result) return NULL;
    value = ast2obj_expr(o->context_expr);
    if (!value) goto failed;
    if (_PyObject_SetAttrId(result, &PyId_context_expr, value) == -1)
        goto failed;
    Py_DECREF(value);
    value = ast2obj_expr(o->optional_vars);
    if (!value) goto failed;
    if (_PyObject_SetAttrId(result, &PyId_optional_vars, value) == -1)
        goto failed;
    Py_DECREF(value);
    return result;
failed:
    Py_XDECREF(value);
    Py_XDECREF(result);
    return NULL;
}

Project: typed_ast

BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, int col_offset,
      PyArena *arena)
{
    expr_ty p;
    if (!left) {
        PyErr_SetString(PyExc_ValueError,
                        "field left is required for BinOp");
        return NULL;
    }
    if (!op) {
        PyErr_SetString(PyExc_ValueError,
                        "field op is required for BinOp");
        return NULL;
    }
    if (!right) {
        PyErr_SetString(PyExc_ValueError,
                        "field right is required for BinOp");
        return NULL;
    }
    p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = BinOp_kind;
    p->v.BinOp.left = left;
    p->v.BinOp.op = op;
    p->v.BinOp.right = right;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast

AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno, int
          col_offset, PyArena *arena)
{
    stmt_ty p;
    if (!target) {
        PyErr_SetString(PyExc_ValueError,
                        "field target is required for AugAssign");
        return NULL;
    }
    if (!op) {
        PyErr_SetString(PyExc_ValueError,
                        "field op is required for AugAssign");
        return NULL;
    }
    if (!value) {
        PyErr_SetString(PyExc_ValueError,
                        "field value is required for AugAssign");
        return NULL;
    }
    p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = AugAssign_kind;
    p->v.AugAssign.target = target;
    p->v.AugAssign.op = op;
    p->v.AugAssign.value = value;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast

UnaryOp(unaryop_ty op, expr_ty operand, int lineno, int col_offset, PyArena
        *arena)
{
    expr_ty p;
    if (!op) {
        PyErr_SetString(PyExc_ValueError,
                        "field op is required for UnaryOp");
        return NULL;
    }
    if (!operand) {
        PyErr_SetString(PyExc_ValueError,
                        "field operand is required for UnaryOp");
        return NULL;
    }
    p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = UnaryOp_kind;
    p->v.UnaryOp.op = op;
    p->v.UnaryOp.operand = operand;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast

Global(asdl_seq * names, int lineno, int col_offset, PyArena *arena)
{
    stmt_ty p;
    p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = Global_kind;
    p->v.Global.names = names;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast
File: Python-ast.c

PyObject* ast2obj_operator(operator_ty o)
{
    switch(o) {
        case Add:
            Py_INCREF(Add_singleton);
            return Add_singleton;
        case Sub:
            Py_INCREF(Sub_singleton);
            return Sub_singleton;
        case Mult:
            Py_INCREF(Mult_singleton);
            return Mult_singleton;
        case MatMult:
            Py_INCREF(MatMult_singleton);
            return MatMult_singleton;
        case Div:
            Py_INCREF(Div_singleton);
            return Div_singleton;
        case Mod:
            Py_INCREF(Mod_singleton);
            return Mod_singleton;
        case Pow:
            Py_INCREF(Pow_singleton);
            return Pow_singleton;
        case LShift:
            Py_INCREF(LShift_singleton);
            return LShift_singleton;
        case RShift:
            Py_INCREF(RShift_singleton);
            return RShift_singleton;
        case BitOr:
            Py_INCREF(BitOr_singleton);
            return BitOr_singleton;
        case BitXor:
            Py_INCREF(BitXor_singleton);
            return BitXor_singleton;
        case BitAnd:
            Py_INCREF(BitAnd_singleton);
            return BitAnd_singleton;
        case FloorDiv:
            Py_INCREF(FloorDiv_singleton);
            return FloorDiv_singleton;
        default:
            /* should never happen, but just in case ... */
            PyErr_Format(PyExc_SystemError, "unknown operator found");
            return NULL;
    }
}

Project: typed_ast

JoinedStr(asdl_seq * values, int lineno, int col_offset, PyArena *arena)
{
    expr_ty p;
    p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = JoinedStr_kind;
    p->v.JoinedStr.values = values;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast
File: Python-ast.c

ast2obj_mod(void* _o)
{
    mod_ty o = (mod_ty)_o;
    PyObject *result = NULL, *value = NULL;
    if (!o) {
        Py_RETURN_NONE;
    }

    switch (o->kind) {
    case Module_kind:
        result = PyType_GenericNew(Module_type, NULL, NULL);
        if (!result) goto failed;
        value = ast2obj_list(o->v.Module.body, ast2obj_stmt);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_body, value) == -1)
            goto failed;
        Py_DECREF(value);
        value = ast2obj_list(o->v.Module.type_ignores, ast2obj_type_ignore);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_type_ignores, value) == -1)
            goto failed;
        Py_DECREF(value);
        break;
    case Interactive_kind:
        result = PyType_GenericNew(Interactive_type, NULL, NULL);
        if (!result) goto failed;
        value = ast2obj_list(o->v.Interactive.body, ast2obj_stmt);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_body, value) == -1)
            goto failed;
        Py_DECREF(value);
        break;
    case Expression_kind:
        result = PyType_GenericNew(Expression_type, NULL, NULL);
        if (!result) goto failed;
        value = ast2obj_expr(o->v.Expression.body);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_body, value) == -1)
            goto failed;
        Py_DECREF(value);
        break;
    case FunctionType_kind:
        result = PyType_GenericNew(FunctionType_type, NULL, NULL);
        if (!result) goto failed;
        value = ast2obj_list(o->v.FunctionType.argtypes, ast2obj_expr);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_argtypes, value) == -1)
            goto failed;
        Py_DECREF(value);
        value = ast2obj_expr(o->v.FunctionType.returns);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_returns, value) == -1)
            goto failed;
        Py_DECREF(value);
        break;
    case Suite_kind:
        result = PyType_GenericNew(Suite_type, NULL, NULL);
        if (!result) goto failed;
        value = ast2obj_list(o->v.Suite.body, ast2obj_stmt);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_body, value) == -1)
            goto failed;
        Py_DECREF(value);
        break;
    }
    return result;
failed:
    Py_XDECREF(value);
    Py_XDECREF(result);
    return NULL;
}

Project: typed_ast

obj2ast_expr_context(PyObject* obj, expr_context_ty* out, PyArena* arena)
{
    int isinstance;

    isinstance = PyObject_IsInstance(obj, (PyObject *)Load_type);
    if (isinstance == -1) {
        return 1;
    }
    if (isinstance) {
        *out = Load;
        return 0;
    }
    isinstance = PyObject_IsInstance(obj, (PyObject *)Store_type);
    if (isinstance == -1) {
        return 1;
    }
    if (isinstance) {
        *out = Store;
        return 0;
    }
    isinstance = PyObject_IsInstance(obj, (PyObject *)Del_type);
    if (isinstance == -1) {
        return 1;
    }
    if (isinstance) {
        *out = Del;
        return 0;
    }
    isinstance = PyObject_IsInstance(obj, (PyObject *)AugLoad_type);
    if (isinstance == -1) {
        return 1;
    }
    if (isinstance) {
        *out = AugLoad;
        return 0;
    }
    isinstance = PyObject_IsInstance(obj, (PyObject *)AugStore_type);
    if (isinstance == -1) {
        return 1;
    }
    if (isinstance) {
        *out = AugStore;
        return 0;
    }
    isinstance = PyObject_IsInstance(obj, (PyObject *)Param_type);
    if (isinstance == -1) {
        return 1;
    }
    if (isinstance) {
        *out = Param;
        return 0;
    }

    PyErr_Format(PyExc_TypeError, "expected some sort of expr_context, but got %R", obj);
    return 1;
}

Project: typed_ast
File: Python-ast.c

ast_clear(AST_object *self)
{
    Py_CLEAR(self->dict);
    return 0;
}

Project: typed_ast

get_operator(const node *n)
{
    switch (TYPE(n)) {
        case VBAR:
            return BitOr;
        case CIRCUMFLEX:
            return BitXor;
        case AMPER:
            return BitAnd;
        case LEFTSHIFT:
            return LShift;
        case RIGHTSHIFT:
            return RShift;
        case PLUS:
            return Add;
        case MINUS:
            return Sub;
        case STAR:
            return Mult;
        case SLASH:
            return Div;
        case DOUBLESLASH:
            return FloorDiv;
        case PERCENT:
            return Mod;
        default:
            return (operator_ty)0;
    }
}

Project: typed_ast

ast_for_for_stmt(struct compiling *c, const node *n0, bool is_async)
{
    const node * const n = is_async ? CHILD(n0, 1) : n0;
    asdl_seq *_target, *seq = NULL, *suite_seq;
    expr_ty expression;
    expr_ty target, first;
    const node *node_target;
    int has_type_comment;
    string type_comment;

    if (is_async && c->c_feature_version < 5) {
        ast_error(c, n,
                "Async for loops are only supported in Python 3.5 and greater");
        return NULL;
    }

    /* for_stmt: 'for' exprlist 'in' testlist ':' [TYPE_COMMENT] suite ['else' ':' suite] */
    REQ(n, for_stmt);

    has_type_comment = TYPE(CHILD(n, 5)) == TYPE_COMMENT;

    if (NCH(n) == 9 + has_type_comment) {
        seq = ast_for_suite(c, CHILD(n, 8 + has_type_comment));
        if (!seq)
            return NULL;
    }

    node_target = CHILD(n, 1);
    _target = ast_for_exprlist(c, node_target, Store);
    if (!_target)
        return NULL;
    /* Check the # of children rather than the length of _target, since
       for x, in ... has 1 element in _target, but still requires a Tuple. */
    first = (expr_ty)asdl_seq_GET(_target, 0);
    if (NCH(node_target) == 1)
        target = first;
    else
        target = Tuple(_target, Store, first->lineno, first->col_offset, c->c_arena);

    expression = ast_for_testlist(c, CHILD(n, 3));
    if (!expression)
        return NULL;
    suite_seq = ast_for_suite(c, CHILD(n, 5 + has_type_comment));
    if (!suite_seq)
        return NULL;

    if (has_type_comment)
        type_comment = NEW_TYPE_COMMENT(CHILD(n, 5));
    else
        type_comment = NULL;

    if (is_async)
        return AsyncFor(target, expression, suite_seq, seq, type_comment,
                        LINENO(n0), n0->n_col_offset,
                        c->c_arena);
    else
        return For(target, expression, suite_seq, seq, type_comment,
                   LINENO(n), n->n_col_offset,
                   c->c_arena);
}

Project: typed_ast
File: tokenizer.c

tok_backup(struct tok_state *tok, int c)
{
    if (c != EOF) {
        if (--tok->cur < tok->buf)
            Py_FatalError("tok_backup: beginning of buffer");
        if (*tok->cur != c)
            *tok->cur = c;
    }
}

Project: typed_ast

seq_for_testlist(struct compiling *c, const node *n)
{
    /* testlist: test (',' test)* [','] */
    asdl_seq *seq;
    expr_ty expression;
    int i;
    assert(TYPE(n) == testlist ||
           TYPE(n) == listmaker ||
           TYPE(n) == testlist_comp ||
           TYPE(n) == testlist_safe ||
           TYPE(n) == testlist1);

    seq = asdl_seq_new((NCH(n) + 1) / 2, c->c_arena);
    if (!seq)
        return NULL;

    for (i = 0; i < NCH(n); i += 2) {
        assert(TYPE(CHILD(n, i)) == test || TYPE(CHILD(n, i)) == old_test);

        expression = ast_for_expr(c, CHILD(n, i));
        if (!expression)
            return NULL;

        assert(i / 2 < seq->size);
        asdl_seq_SET(seq, i / 2, expression);
    }
    return seq;
}

Project: typed_ast

obj2ast_alias(PyObject* obj, alias_ty* out, PyArena* arena)
{
    PyObject* tmp = NULL;
    identifier name;
    identifier asname;

    if (lookup_attr_id(obj, &PyId_name, &tmp) < 0) {
        return 1;
    }
    if (tmp == NULL) {
        PyErr_SetString(PyExc_TypeError, "required field \"name\" missing from alias");
        return 1;
    }
    else {
        int res;
        res = obj2ast_identifier(tmp, &name, arena);
        if (res != 0) goto failed;
        Py_CLEAR(tmp);
    }
    if (lookup_attr_id(obj, &PyId_asname, &tmp) < 0) {
        return 1;
    }
    if (tmp == NULL || tmp == Py_None) {
        Py_CLEAR(tmp);
        asname = NULL;
    }
    else {
        int res;
        res = obj2ast_identifier(tmp, &asname, arena);
        if (res != 0) goto failed;
        Py_CLEAR(tmp);
    }
    *out = alias(name, asname, arena);
    return 0;
failed:
    Py_XDECREF(tmp);
    return 1;
}

Project: typed_ast

ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr)
{
    /* trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
       subscriptlist: subscript (',' subscript)* [',']
       subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop]
     */
    REQ(n, trailer);
    if (TYPE(CHILD(n, 0)) == LPAR) {
        if (NCH(n) == 2)
            return Call(left_expr, NULL, NULL, NULL, NULL, LINENO(n),
                        n->n_col_offset, c->c_arena);
        else
            return ast_for_call(c, CHILD(n, 1), left_expr);
    }
    else if (TYPE(CHILD(n, 0)) == DOT ) {
        PyObject *attr_id = NEW_IDENTIFIER(CHILD(n, 1));
        if (!attr_id)
            return NULL;
        return Attribute(left_expr, attr_id, Load,
                         LINENO(n), n->n_col_offset, c->c_arena);
    }
    else {
        REQ(CHILD(n, 0), LSQB);
        REQ(CHILD(n, 2), RSQB);
        n = CHILD(n, 1);
        if (NCH(n) == 1) {
            slice_ty slc = ast_for_slice(c, CHILD(n, 0));
            if (!slc)
                return NULL;
            return Subscript(left_expr, slc, Load, LINENO(n), n->n_col_offset,
                             c->c_arena);
        }
        else {
            /* The grammar is ambiguous here. The ambiguity is resolved
               by treating the sequence as a tuple literal if there are
               no slice features.
            */
            int j;
            slice_ty slc;
            expr_ty e;
            bool simple = true;
            asdl_seq *slices, *elts;
            slices = asdl_seq_new((NCH(n) + 1) / 2, c->c_arena);
            if (!slices)
                return NULL;
            for (j = 0; j < NCH(n); j += 2) {
                slc = ast_for_slice(c, CHILD(n, j));
                if (!slc)
                    return NULL;
                if (slc->kind != Index_kind)
                    simple = false;
                asdl_seq_SET(slices, j / 2, slc);
            }
            if (!simple) {
                return Subscript(left_expr, ExtSlice(slices, c->c_arena),
                                 Load, LINENO(n), n->n_col_offset, c->c_arena);
            }
            /* extract Index values and put them in a Tuple */
            elts = asdl_seq_new(asdl_seq_LEN(slices), c->c_arena);
            if (!elts)
                return NULL;
            for (j = 0; j < asdl_seq_LEN(slices); ++j) {
                slc = (slice_ty)asdl_seq_GET(slices, j);
                assert(slc->kind == Index_kind  && slc->v.Index.value);
                asdl_seq_SET(elts, j, slc->v.Index.value);
            }
            e = Tuple(elts, Load, LINENO(n), n->n_col_offset, c->c_arena);
            if (!e)
                return NULL;
            return Subscript(left_expr, Index(e, c->c_arena),
                             Load, LINENO(n), n->n_col_offset, c->c_arena);
        }
    }
}

Project: typed_ast

err_input(perrdetail *err)
{
    PyObject *v, *w, *errtype, *errtext;
    PyObject *msg_obj = NULL;
    char *msg = NULL;
    int offset = err->offset;

    errtype = PyExc_SyntaxError;
    switch (err->error) {
    case E_ERROR:
        return;
    case E_SYNTAX:
        errtype = PyExc_IndentationError;
        if (err->expected == INDENT)
            msg = "expected an indented block";
        else if (err->token == INDENT)
            msg = "unexpected indent";
        else if (err->token == DEDENT)
            msg = "unexpected unindent";
        else {
            errtype = PyExc_SyntaxError;
            if (err->token == TYPE_COMMENT)
              msg = "misplaced type annotation";
            else
            msg = "invalid syntax";
        }
        break;
    case E_TOKEN:
        msg = "invalid token";
        break;
    case E_EOFS:
        msg = "EOF while scanning triple-quoted string literal";
        break;
    case E_EOLS:
        msg = "EOL while scanning string literal";
        break;
    case E_INTR:
        if (!PyErr_Occurred())
            PyErr_SetNone(PyExc_KeyboardInterrupt);
        goto cleanup;
    case E_NOMEM:
        PyErr_NoMemory();
        goto cleanup;
    case E_EOF:
        msg = "unexpected EOF while parsing";
        break;
    case E_TABSPACE:
        errtype = PyExc_TabError;
        msg = "inconsistent use of tabs and spaces in indentation";
        break;
    case E_OVERFLOW:
        msg = "expression too long";
        break;
    case E_DEDENT:
        errtype = PyExc_IndentationError;
        msg = "unindent does not match any outer indentation level";
        break;
    case E_TOODEEP:
        errtype = PyExc_IndentationError;
        msg = "too many levels of indentation";
        break;
    case E_DECODE: {
        PyObject *type, *value, *tb;
        PyErr_Fetch(&type, &value, &tb);
        msg = "unknown decode error";
        if (value != NULL)
            msg_obj = PyObject_Str(value);
        Py_XDECREF(type);
        Py_XDECREF(value);
        Py_XDECREF(tb);
        break;
    }
    case E_LINECONT:
        msg = "unexpected character after line continuation character";
        break;

    case E_IDENTIFIER:
        msg = "invalid character in identifier";
        break;
    case E_BADSINGLE:
        msg = "multiple statements found while compiling a single statement";
        break;
    default:
        fprintf(stderr, "error=%d\n", err->error);
        msg = "unknown parsing error";
        break;
    }
    /* err->text may not be UTF-8 in case of decoding errors.
       Explicitly convert to an object. */
    if (!err->text) {
        errtext = Py_None;
        Py_INCREF(Py_None);
    } else {
        errtext = PyUnicode_DecodeUTF8(err->text, err->offset,
                                       "replace");
        if (errtext != NULL) {
            Py_ssize_t len = strlen(err->text);
            offset = (int)PyUnicode_GET_LENGTH(errtext);
            if (len != err->offset) {
                Py_DECREF(errtext);
                errtext = PyUnicode_DecodeUTF8(err->text, len,
                                               "replace");
            }
        }
    }
    v = Py_BuildValue("(OiiN)", err->filename,
                      err->lineno, offset, errtext);
    if (v != NULL) {
        if (msg_obj)
            w = Py_BuildValue("(OO)", msg_obj, v);
        else
            w = Py_BuildValue("(sO)", msg, v);
    } else
        w = NULL;
    Py_XDECREF(v);
    PyErr_SetObject(errtype, w);
    Py_XDECREF(w);
cleanup:
    Py_XDECREF(msg_obj);
    if (err->text != NULL) {
        PyObject_FREE(err->text);
        err->text = NULL;
    }
}

Project: typed_ast
File: Python-ast.c

static int obj2ast_int(PyObject* obj, int* out, PyArena* arena)
{
    int i;
    if (!PyLong_Check(obj)) {
        PyErr_Format(PyExc_ValueError, "invalid integer value: %R", obj);
        return 1;
    }

    i = _PyLong_AsInt(obj);
    if (i == -1 && PyErr_Occurred())
        return 1;
    *out = i;
    return 0;
}

Project: typed_ast

obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena)
{
    int isinstance;

    PyObject *tmp = NULL;

    if (obj == Py_None) {
        *out = NULL;
        return 0;
    }
    isinstance = PyObject_IsInstance(obj, (PyObject*)Module_type);
    if (isinstance == -1) {
        return 1;
    }
    if (isinstance) {
        asdl_seq* body;
        asdl_seq* type_ignores;

        if (lookup_attr_id(obj, &PyId_body, &tmp) < 0) {
            return 1;
        }
        if (tmp == NULL) {
            PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from Module");
            return 1;
        }
        else {
            int res;
            Py_ssize_t len;
            Py_ssize_t i;
            if (!PyList_Check(tmp)) {
                PyErr_Format(PyExc_TypeError, "Module field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name);
                goto failed;
            }
            len = PyList_GET_SIZE(tmp);
            body = _Ta3_asdl_seq_new(len, arena);
            if (body == NULL) goto failed;
            for (i = 0; i < len; i++) {
                stmt_ty val;
                res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena);
                if (res != 0) goto failed;
                if (len != PyList_GET_SIZE(tmp)) {
                    PyErr_SetString(PyExc_RuntimeError, "Module field \"body\" changed size during iteration");
                    goto failed;
                }
                asdl_seq_SET(body, i, val);
            }
            Py_CLEAR(tmp);
        }
        if (lookup_attr_id(obj, &PyId_type_ignores, &tmp) < 0) {
            return 1;
        }
        if (tmp == NULL) {
            PyErr_SetString(PyExc_TypeError, "required field \"type_ignores\" missing from Module");
            return 1;
        }
        else {
            int res;
            Py_ssize_t len;
            Py_ssize_t i;
            if (!PyList_Check(tmp)) {
                PyErr_Format(PyExc_TypeError, "Module field \"type_ignores\" must be a list, not a %.200s", tmp->ob_type->tp_name);
                goto failed;
            }
            len = PyList_GET_SIZE(tmp);
            type_ignores = _Ta3_asdl_seq_new(len, arena);
            if (type_ignores == NULL) goto failed;
            for (i = 0; i < len; i++) {
                type_ignore_ty val;
                res = obj2ast_type_ignore(PyList_GET_ITEM(tmp, i), &val, arena);
                if (res != 0) goto failed;
                if (len != PyList_GET_SIZE(tmp)) {
                    PyErr_SetString(PyExc_RuntimeError, "Module field \"type_ignores\" changed size during iteration");
                    goto failed;
                }
                asdl_seq_SET(type_ignores, i, val);
            }
            Py_CLEAR(tmp);
        }
        *out = Module(body, type_ignores, arena);
        if (*out == NULL) goto failed;
        return 0;
    }
    isinstance = PyObject_IsInstance(obj, (PyObject*)Interactive_type);
    if (isinstance == -1) {
        return 1;
    }
    if (isinstance) {
        asdl_seq* body;

        if (lookup_attr_id(obj, &PyId_body, &tmp) < 0) {
            return 1;
        }
        if (tmp == NULL) {
            PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from Interactive");
            return 1;
        }
        else {
            int res;
            Py_ssize_t len;
            Py_ssize_t i;
            if (!PyList_Check(tmp)) {
                PyErr_Format(PyExc_TypeError, "Interactive field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name);
                goto failed;
            }
            len = PyList_GET_SIZE(tmp);
            body = _Ta3_asdl_seq_new(len, arena);
            if (body == NULL) goto failed;
            for (i = 0; i < len; i++) {
                stmt_ty val;
                res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena);
                if (res != 0) goto failed;
                if (len != PyList_GET_SIZE(tmp)) {
                    PyErr_SetString(PyExc_RuntimeError, "Interactive field \"body\" changed size during iteration");
                    goto failed;
                }
                asdl_seq_SET(body, i, val);
            }
            Py_CLEAR(tmp);
        }
        *out = Interactive(body, arena);
        if (*out == NULL) goto failed;
        return 0;
    }
    isinstance = PyObject_IsInstance(obj, (PyObject*)Expression_type);
    if (isinstance == -1) {
        return 1;
    }
    if (isinstance) {
        expr_ty body;

        if (lookup_attr_id(obj, &PyId_body, &tmp) < 0) {
            return 1;
        }
        if (tmp == NULL) {
            PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from Expression");
            return 1;
        }
        else {
            int res;
            res = obj2ast_expr(tmp, &body, arena);
            if (res != 0) goto failed;
            Py_CLEAR(tmp);
        }
        *out = Expression(body, arena);
        if (*out == NULL) goto failed;
        return 0;
    }
    isinstance = PyObject_IsInstance(obj, (PyObject*)FunctionType_type);
    if (isinstance == -1) {
        return 1;
    }
    if (isinstance) {
        asdl_seq* argtypes;
        expr_ty returns;

        if (lookup_attr_id(obj, &PyId_argtypes, &tmp) < 0) {
            return 1;
        }
        if (tmp == NULL) {
            PyErr_SetString(PyExc_TypeError, "required field \"argtypes\" missing from FunctionType");
            return 1;
        }
        else {
            int res;
            Py_ssize_t len;
            Py_ssize_t i;
            if (!PyList_Check(tmp)) {
                PyErr_Format(PyExc_TypeError, "FunctionType field \"argtypes\" must be a list, not a %.200s", tmp->ob_type->tp_name);
                goto failed;
            }
            len = PyList_GET_SIZE(tmp);
            argtypes = _Ta3_asdl_seq_new(len, arena);
            if (argtypes == NULL) goto failed;
            for (i = 0; i < len; i++) {
                expr_ty val;
                res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &val, arena);
                if (res != 0) goto failed;
                if (len != PyList_GET_SIZE(tmp)) {
                    PyErr_SetString(PyExc_RuntimeError, "FunctionType field \"argtypes\" changed size during iteration");
                    goto failed;
                }
                asdl_seq_SET(argtypes, i, val);
            }
            Py_CLEAR(tmp);
        }
        if (lookup_attr_id(obj, &PyId_returns, &tmp) < 0) {
            return 1;
        }
        if (tmp == NULL) {
            PyErr_SetString(PyExc_TypeError, "required field \"returns\" missing from FunctionType");
            return 1;
        }
        else {
            int res;
            res = obj2ast_expr(tmp, &returns, arena);
            if (res != 0) goto failed;
            Py_CLEAR(tmp);
        }
        *out = FunctionType(argtypes, returns, arena);
        if (*out == NULL) goto failed;
        return 0;
    }
    isinstance = PyObject_IsInstance(obj, (PyObject*)Suite_type);
    if (isinstance == -1) {
        return 1;
    }
    if (isinstance) {
        asdl_seq* body;

        if (lookup_attr_id(obj, &PyId_body, &tmp) < 0) {
            return 1;
        }
        if (tmp == NULL) {
            PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from Suite");
            return 1;
        }
        else {
            int res;
            Py_ssize_t len;
            Py_ssize_t i;
            if (!PyList_Check(tmp)) {
                PyErr_Format(PyExc_TypeError, "Suite field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name);
                goto failed;
            }
            len = PyList_GET_SIZE(tmp);
            body = _Ta3_asdl_seq_new(len, arena);
            if (body == NULL) goto failed;
            for (i = 0; i < len; i++) {
                stmt_ty val;
                res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &val, arena);
                if (res != 0) goto failed;
                if (len != PyList_GET_SIZE(tmp)) {
                    PyErr_SetString(PyExc_RuntimeError, "Suite field \"body\" changed size during iteration");
                    goto failed;
                }
                asdl_seq_SET(body, i, val);
            }
            Py_CLEAR(tmp);
        }
        *out = Suite(body, arena);
        if (*out == NULL) goto failed;
        return 0;
    }

    PyErr_Format(PyExc_TypeError, "expected some sort of mod, but got %R", obj);
    failed:
    Py_XDECREF(tmp);
    return 1;
}

Project: typed_ast

ast_for_except_clause(struct compiling *c, const node *exc, node *body)
{
    /* except_clause: 'except' [test [(',' | 'as') test]] */
    REQ(exc, except_clause);
    REQ(body, suite);

    if (NCH(exc) == 1) {
        asdl_seq *suite_seq = ast_for_suite(c, body);
        if (!suite_seq)
            return NULL;

        return ExceptHandler(NULL, NULL, suite_seq, LINENO(exc),
                             exc->n_col_offset, c->c_arena);
    }
    else if (NCH(exc) == 2) {
        expr_ty expression;
        asdl_seq *suite_seq;

        expression = ast_for_expr(c, CHILD(exc, 1));
        if (!expression)
            return NULL;
        suite_seq = ast_for_suite(c, body);
        if (!suite_seq)
            return NULL;

        return ExceptHandler(expression, NULL, suite_seq, LINENO(exc),
                             exc->n_col_offset, c->c_arena);
    }
    else if (NCH(exc) == 4) {
        asdl_seq *suite_seq;
        expr_ty expression;
        expr_ty e = ast_for_expr(c, CHILD(exc, 3));
        if (!e)
            return NULL;
        if (!set_context(c, e, Store, CHILD(exc, 3)))
            return NULL;
        expression = ast_for_expr(c, CHILD(exc, 1));
        if (!expression)
            return NULL;
        suite_seq = ast_for_suite(c, body);
        if (!suite_seq)
            return NULL;

        return ExceptHandler(expression, e, suite_seq, LINENO(exc),
                             exc->n_col_offset, c->c_arena);
    }

    PyErr_Format(PyExc_SystemError,
                 "wrong number of children for 'except' clause: %d",
                 NCH(exc));
    return NULL;
}

Project: typed_ast

ast_for_augassign(struct compiling *c, const node *n)
{
    REQ(n, augassign);
    n = CHILD(n, 0);
    switch (STR(n)[0]) {
        case '+':
            return Add;
        case '-':
            return Sub;
        case '/':
            if (STR(n)[1] == '/')
                return FloorDiv;
            else
                return Div;
        case '%':
            return Mod;
        case '<':
            return LShift;
        case '>':
            return RShift;
        case '&':
            return BitAnd;
        case '^':
            return BitXor;
        case '|':
            return BitOr;
        case '*':
            if (STR(n)[1] == '*')
                return Pow;
            else
                return Mult;
        default:
            PyErr_Format(PyExc_SystemError, "invalid augassign: %s", STR(n));
            return (operator_ty)0;
    }
}

Project: typed_ast

Num(object n, int lineno, int col_offset, PyArena *arena)
{
    expr_ty p;
    if (!n) {
        PyErr_SetString(PyExc_ValueError,
                        "field n is required for Num");
        return NULL;
    }
    p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = Num_kind;
    p->v.Num.n = n;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast

ast3_parse_impl(PyObject *source,
                PyObject *filename,
                const char *mode,
                int feature_version)
{
    PyObject *source_copy;
    const char *str;
    int compile_mode = -1;
    PyCompilerFlags cf;
    int start[] = {Py_file_input, Py_eval_input, Py_single_input, Py_func_type_input};
    PyObject *result;

    cf.cf_flags = PyCF_ONLY_AST | PyCF_SOURCE_IS_UTF8;

    if (strcmp(mode, "exec") == 0)
        compile_mode = 0;
    else if (strcmp(mode, "eval") == 0)
        compile_mode = 1;
    else if (strcmp(mode, "single") == 0)
        compile_mode = 2;
    else if (strcmp(mode, "func_type") == 0)
        compile_mode = 3;
    else {
        PyErr_SetString(PyExc_ValueError,
                        "parse() mode must be 'exec', 'eval', 'single', for 'func_type'");
        goto error;
    }

    str = source_as_string(source, "parse", "string or bytes", &cf, &source_copy);
    if (str == NULL)
        goto error;

    result = string_object_to_py_ast(str, filename, start[compile_mode], &cf, feature_version);
    Py_XDECREF(source_copy);
    goto finally;

error:
    result = NULL;
finally:
    Py_DECREF(filename);
    return result;
}

Project: typed_ast

Ta3Parser_ParseFileFlags(FILE *fp, const char *filename, const char *enc,
                        grammar *g, int start,
                        const char *ps1, const char *ps2,
                        perrdetail *err_ret, int flags)
{
    int iflags = flags;
    return Ta3Parser_ParseFileFlagsEx(fp, filename, enc, g, start, ps1,
                                     ps2, err_ret, &iflags);
}

Project: typed_ast

ast2obj_excepthandler(void* _o)
{
    excepthandler_ty o = (excepthandler_ty)_o;
    PyObject *result = NULL, *value = NULL;
    if (!o) {
        Py_RETURN_NONE;
    }

    switch (o->kind) {
    case ExceptHandler_kind:
        result = PyType_GenericNew(ExceptHandler_type, NULL, NULL);
        if (!result) goto failed;
        value = ast2obj_expr(o->v.ExceptHandler.type);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_type, value) == -1)
            goto failed;
        Py_DECREF(value);
        value = ast2obj_identifier(o->v.ExceptHandler.name);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_name, value) == -1)
            goto failed;
        Py_DECREF(value);
        value = ast2obj_list(o->v.ExceptHandler.body, ast2obj_stmt);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_body, value) == -1)
            goto failed;
        Py_DECREF(value);
        break;
    }
    value = ast2obj_int(o->lineno);
    if (!value) goto failed;
    if (_PyObject_SetAttrId(result, &PyId_lineno, value) < 0)
        goto failed;
    Py_DECREF(value);
    value = ast2obj_int(o->col_offset);
    if (!value) goto failed;
    if (_PyObject_SetAttrId(result, &PyId_col_offset, value) < 0)
        goto failed;
    Py_DECREF(value);
    return result;
failed:
    Py_XDECREF(value);
    Py_XDECREF(result);
    return NULL;
}

Project: typed_ast

AsyncWith(asdl_seq * items, asdl_seq * body, string type_comment, int lineno,
          int col_offset, PyArena *arena)
{
    stmt_ty p;
    p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = AsyncWith_kind;
    p->v.AsyncWith.items = items;
    p->v.AsyncWith.body = body;
    p->v.AsyncWith.type_comment = type_comment;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast

ast2obj_expr(void* _o)
{
    expr_ty o = (expr_ty)_o;
    PyObject *result = NULL, *value = NULL;
    if (!o) {
        Py_RETURN_NONE;
    }

    switch (o->kind) {
    case BoolOp_kind:
        result = PyType_GenericNew(BoolOp_type, NULL, NULL);
        if (!result) goto failed;
        value = ast2obj_boolop(o->v.BoolOp.op);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_op, value) == -1)
            goto failed;
        Py_DECREF(value);
        value = ast2obj_list(o->v.BoolOp.values, ast2obj_expr);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_values, value) == -1)
            goto failed;
        Py_DECREF(value);
        break;
    case BinOp_kind:
        result = PyType_GenericNew(BinOp_type, NULL, NULL);
        if (!result) goto failed;
        value = ast2obj_expr(o->v.BinOp.left);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_left, value) == -1)
            goto failed;
        Py_DECREF(value);
        value = ast2obj_operator(o->v.BinOp.op);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_op, value) == -1)
            goto failed;
        Py_DECREF(value);
        value = ast2obj_expr(o->v.BinOp.right);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_right, value) == -1)
            goto failed;
        Py_DECREF(value);
        break;
    case UnaryOp_kind:
        result = PyType_GenericNew(UnaryOp_type, NULL, NULL);
        if (!result) goto failed;
        value = ast2obj_unaryop(o->v.UnaryOp.op);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_op, value) == -1)
            goto failed;
        Py_DECREF(value);
        value = ast2obj_expr(o->v.UnaryOp.operand);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_operand, value) == -1)
            goto failed;
        Py_DECREF(value);
        break;
    case Lambda_kind:
        result = PyType_GenericNew(Lambda_type, NULL, NULL);
        if (!result) goto failed;
        value = ast2obj_arguments(o->v.Lambda.args);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_args, value) == -1)
            goto failed;
        Py_DECREF(value);
        value = ast2obj_expr(o->v.Lambda.body);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_body, value) == -1)
            goto failed;
        Py_DECREF(value);
        break;
    case IfExp_kind:
        result = PyType_GenericNew(IfExp_type, NULL, NULL);
        if (!result) goto failed;
        value = ast2obj_expr(o->v.IfExp.test);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_test, value) == -1)
            goto failed;
        Py_DECREF(value);
        value = ast2obj_expr(o->v.IfExp.body);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_body, value) == -1)
            goto failed;
        Py_DECREF(value);
        value = ast2obj_expr(o->v.IfExp.orelse);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_orelse, value) == -1)
            goto failed;
        Py_DECREF(value);
        break;
    case Dict_kind:
        result = PyType_GenericNew(Dict_type, NULL, NULL);
        if (!result) goto failed;
        value = ast2obj_list(o->v.Dict.keys, ast2obj_expr);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_keys, value) == -1)
            goto failed;
        Py_DECREF(value);
        value = ast2obj_list(o->v.Dict.values, ast2obj_expr);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_values, value) == -1)
            goto failed;
        Py_DECREF(value);
        break;
    case Set_kind:
        result = PyType_GenericNew(Set_type, NULL, NULL);
        if (!result) goto failed;
        value = ast2obj_list(o->v.Set.elts, ast2obj_expr);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_elts, value) == -1)
            goto failed;
        Py_DECREF(value);
        break;
    case ListComp_kind:
        result = PyType_GenericNew(ListComp_type, NULL, NULL);
        if (!result) goto failed;
        value = ast2obj_expr(o->v.ListComp.elt);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_elt, value) == -1)
            goto failed;
        Py_DECREF(value);
        value = ast2obj_list(o->v.ListComp.generators, ast2obj_comprehension);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_generators, value) == -1)
            goto failed;
        Py_DECREF(value);
        break;
    case SetComp_kind:
        result = PyType_GenericNew(SetComp_type, NULL, NULL);
        if (!result) goto failed;
        value = ast2obj_expr(o->v.SetComp.elt);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_elt, value) == -1)
            goto failed;
        Py_DECREF(value);
        value = ast2obj_list(o->v.SetComp.generators, ast2obj_comprehension);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_generators, value) == -1)
            goto failed;
        Py_DECREF(value);
        break;
    case DictComp_kind:
        result = PyType_GenericNew(DictComp_type, NULL, NULL);
        if (!result) goto failed;
        value = ast2obj_expr(o->v.DictComp.key);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_key, value) == -1)
            goto failed;
        Py_DECREF(value);
        value = ast2obj_expr(o->v.DictComp.value);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
            goto failed;
        Py_DECREF(value);
        value = ast2obj_list(o->v.DictComp.generators, ast2obj_comprehension);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_generators, value) == -1)
            goto failed;
        Py_DECREF(value);
        break;
    case GeneratorExp_kind:
        result = PyType_GenericNew(GeneratorExp_type, NULL, NULL);
        if (!result) goto failed;
        value = ast2obj_expr(o->v.GeneratorExp.elt);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_elt, value) == -1)
            goto failed;
        Py_DECREF(value);
        value = ast2obj_list(o->v.GeneratorExp.generators,
                             ast2obj_comprehension);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_generators, value) == -1)
            goto failed;
        Py_DECREF(value);
        break;
    case Await_kind:
        result = PyType_GenericNew(Await_type, NULL, NULL);
        if (!result) goto failed;
        value = ast2obj_expr(o->v.Await.value);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
            goto failed;
        Py_DECREF(value);
        break;
    case Yield_kind:
        result = PyType_GenericNew(Yield_type, NULL, NULL);
        if (!result) goto failed;
        value = ast2obj_expr(o->v.Yield.value);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
            goto failed;
        Py_DECREF(value);
        break;
    case YieldFrom_kind:
        result = PyType_GenericNew(YieldFrom_type, NULL, NULL);
        if (!result) goto failed;
        value = ast2obj_expr(o->v.YieldFrom.value);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
            goto failed;
        Py_DECREF(value);
        break;
    case Compare_kind:
        result = PyType_GenericNew(Compare_type, NULL, NULL);
        if (!result) goto failed;
        value = ast2obj_expr(o->v.Compare.left);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_left, value) == -1)
            goto failed;
        Py_DECREF(value);
        {
            Py_ssize_t i, n = asdl_seq_LEN(o->v.Compare.ops);
            value = PyList_New(n);
            if (!value) goto failed;
            for(i = 0; i < n; i++)
                PyList_SET_ITEM(value, i, ast2obj_cmpop((cmpop_ty)asdl_seq_GET(o->v.Compare.ops, i)));
        }
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_ops, value) == -1)
            goto failed;
        Py_DECREF(value);
        value = ast2obj_list(o->v.Compare.comparators, ast2obj_expr);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_comparators, value) == -1)
            goto failed;
        Py_DECREF(value);
        break;
    case Call_kind:
        result = PyType_GenericNew(Call_type, NULL, NULL);
        if (!result) goto failed;
        value = ast2obj_expr(o->v.Call.func);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_func, value) == -1)
            goto failed;
        Py_DECREF(value);
        value = ast2obj_list(o->v.Call.args, ast2obj_expr);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_args, value) == -1)
            goto failed;
        Py_DECREF(value);
        value = ast2obj_list(o->v.Call.keywords, ast2obj_keyword);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_keywords, value) == -1)
            goto failed;
        Py_DECREF(value);
        break;
    case Num_kind:
        result = PyType_GenericNew(Num_type, NULL, NULL);
        if (!result) goto failed;
        value = ast2obj_object(o->v.Num.n);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_n, value) == -1)
            goto failed;
        Py_DECREF(value);
        break;
    case Str_kind:
        result = PyType_GenericNew(Str_type, NULL, NULL);
        if (!result) goto failed;
        value = ast2obj_string(o->v.Str.s);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_s, value) == -1)
            goto failed;
        Py_DECREF(value);
        value = ast2obj_string(o->v.Str.kind);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_kind, value) == -1)
            goto failed;
        Py_DECREF(value);
        break;
    case FormattedValue_kind:
        result = PyType_GenericNew(FormattedValue_type, NULL, NULL);
        if (!result) goto failed;
        value = ast2obj_expr(o->v.FormattedValue.value);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
            goto failed;
        Py_DECREF(value);
        value = ast2obj_int(o->v.FormattedValue.conversion);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_conversion, value) == -1)
            goto failed;
        Py_DECREF(value);
        value = ast2obj_expr(o->v.FormattedValue.format_spec);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_format_spec, value) == -1)
            goto failed;
        Py_DECREF(value);
        break;
    case JoinedStr_kind:
        result = PyType_GenericNew(JoinedStr_type, NULL, NULL);
        if (!result) goto failed;
        value = ast2obj_list(o->v.JoinedStr.values, ast2obj_expr);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_values, value) == -1)
            goto failed;
        Py_DECREF(value);
        break;
    case Bytes_kind:
        result = PyType_GenericNew(Bytes_type, NULL, NULL);
        if (!result) goto failed;
        value = ast2obj_bytes(o->v.Bytes.s);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_s, value) == -1)
            goto failed;
        Py_DECREF(value);
        break;
    case NameConstant_kind:
        result = PyType_GenericNew(NameConstant_type, NULL, NULL);
        if (!result) goto failed;
        value = ast2obj_singleton(o->v.NameConstant.value);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
            goto failed;
        Py_DECREF(value);
        break;
    case Ellipsis_kind:
        result = PyType_GenericNew(Ellipsis_type, NULL, NULL);
        if (!result) goto failed;
        break;
    case Constant_kind:
        result = PyType_GenericNew(Constant_type, NULL, NULL);
        if (!result) goto failed;
        value = ast2obj_constant(o->v.Constant.value);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
            goto failed;
        Py_DECREF(value);
        break;
    case Attribute_kind:
        result = PyType_GenericNew(Attribute_type, NULL, NULL);
        if (!result) goto failed;
        value = ast2obj_expr(o->v.Attribute.value);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
            goto failed;
        Py_DECREF(value);
        value = ast2obj_identifier(o->v.Attribute.attr);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_attr, value) == -1)
            goto failed;
        Py_DECREF(value);
        value = ast2obj_expr_context(o->v.Attribute.ctx);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_ctx, value) == -1)
            goto failed;
        Py_DECREF(value);
        break;
    case Subscript_kind:
        result = PyType_GenericNew(Subscript_type, NULL, NULL);
        if (!result) goto failed;
        value = ast2obj_expr(o->v.Subscript.value);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
            goto failed;
        Py_DECREF(value);
        value = ast2obj_slice(o->v.Subscript.slice);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_slice, value) == -1)
            goto failed;
        Py_DECREF(value);
        value = ast2obj_expr_context(o->v.Subscript.ctx);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_ctx, value) == -1)
            goto failed;
        Py_DECREF(value);
        break;
    case Starred_kind:
        result = PyType_GenericNew(Starred_type, NULL, NULL);
        if (!result) goto failed;
        value = ast2obj_expr(o->v.Starred.value);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
            goto failed;
        Py_DECREF(value);
        value = ast2obj_expr_context(o->v.Starred.ctx);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_ctx, value) == -1)
            goto failed;
        Py_DECREF(value);
        break;
    case Name_kind:
        result = PyType_GenericNew(Name_type, NULL, NULL);
        if (!result) goto failed;
        value = ast2obj_identifier(o->v.Name.id);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_id, value) == -1)
            goto failed;
        Py_DECREF(value);
        value = ast2obj_expr_context(o->v.Name.ctx);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_ctx, value) == -1)
            goto failed;
        Py_DECREF(value);
        break;
    case List_kind:
        result = PyType_GenericNew(List_type, NULL, NULL);
        if (!result) goto failed;
        value = ast2obj_list(o->v.List.elts, ast2obj_expr);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_elts, value) == -1)
            goto failed;
        Py_DECREF(value);
        value = ast2obj_expr_context(o->v.List.ctx);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_ctx, value) == -1)
            goto failed;
        Py_DECREF(value);
        break;
    case Tuple_kind:
        result = PyType_GenericNew(Tuple_type, NULL, NULL);
        if (!result) goto failed;
        value = ast2obj_list(o->v.Tuple.elts, ast2obj_expr);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_elts, value) == -1)
            goto failed;
        Py_DECREF(value);
        value = ast2obj_expr_context(o->v.Tuple.ctx);
        if (!value) goto failed;
        if (_PyObject_SetAttrId(result, &PyId_ctx, value) == -1)
            goto failed;
        Py_DECREF(value);
        break;
    }
    value = ast2obj_int(o->lineno);
    if (!value) goto failed;
    if (_PyObject_SetAttrId(result, &PyId_lineno, value) < 0)
        goto failed;
    Py_DECREF(value);
    value = ast2obj_int(o->col_offset);
    if (!value) goto failed;
    if (_PyObject_SetAttrId(result, &PyId_col_offset, value) < 0)
        goto failed;
    Py_DECREF(value);
    return result;
failed:
    Py_XDECREF(value);
    Py_XDECREF(result);
    return NULL;
}

Project: typed_ast

Delete(asdl_seq * targets, int lineno, int col_offset, PyArena *arena)
{
    stmt_ty p;
    p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = Delete_kind;
    p->v.Delete.targets = targets;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast

decode_unicode(struct compiling *c, const char *s, size_t len, int rawmode, const char *encoding)
{
    PyObject *v, *u;
    char *buf;
    char *p;
    const char *end;

    if (encoding == NULL) {
        u = NULL;
    } else {
        /* check for integer overflow */
        if (len > PY_SIZE_MAX / 6)
            return NULL;
        /* "ä" (2 bytes) may become "\U000000E4" (10 bytes), or 1:5
           "\ä" (3 bytes) may become "\u005c\U000000E4" (16 bytes), or ~1:6 */
        u = PyBytes_FromStringAndSize((char *)NULL, len * 6);
        if (u == NULL)
            return NULL;
        p = buf = PyBytes_AsString(u);
        end = s + len;
        while (s < end) {
            if (*s == '\\') {
                *p++ = *s++;
                if (*s & 0x80) {
                    strcpy(p, "u005c");
                    p += 5;
                }
            }
            if (*s & 0x80) { /* XXX inefficient */
                PyObject *w;
                int kind;
                void *data;
                Py_ssize_t len, i;
                w = decode_utf8(c, &s, end);
                if (w == NULL) {
                    Py_DECREF(u);
                    return NULL;
                }
                kind = PyUnicode_KIND(w);
                data = PyUnicode_DATA(w);
                len = PyUnicode_GET_LENGTH(w);
                for (i = 0; i < len; i++) {
                    Py_UCS4 chr = PyUnicode_READ(kind, data, i);
                    sprintf(p, "\\U%08x", chr);
                    p += 10;
                }
                /* Should be impossible to overflow */
                assert(p - buf <= Py_SIZE(u));
                Py_DECREF(w);
            } else {
                *p++ = *s++;
            }
        }
        len = p - buf;
        s = buf;
    }
    if (rawmode)
        v = PyUnicode_DecodeRawUnicodeEscape(s, len, NULL);
    else
        v = PyUnicode_DecodeUnicodeEscape(s, len, NULL);
    Py_XDECREF(u);
    return v;
}

Project: typed_ast
File: Python-ast.c

PyObject* ast2obj_cmpop(cmpop_ty o)
{
    switch(o) {
        case Eq:
            Py_INCREF(Eq_singleton);
            return Eq_singleton;
        case NotEq:
            Py_INCREF(NotEq_singleton);
            return NotEq_singleton;
        case Lt:
            Py_INCREF(Lt_singleton);
            return Lt_singleton;
        case LtE:
            Py_INCREF(LtE_singleton);
            return LtE_singleton;
        case Gt:
            Py_INCREF(Gt_singleton);
            return Gt_singleton;
        case GtE:
            Py_INCREF(GtE_singleton);
            return GtE_singleton;
        case Is:
            Py_INCREF(Is_singleton);
            return Is_singleton;
        case IsNot:
            Py_INCREF(IsNot_singleton);
            return IsNot_singleton;
        case In:
            Py_INCREF(In_singleton);
            return In_singleton;
        case NotIn:
            Py_INCREF(NotIn_singleton);
            return NotIn_singleton;
        default:
            /* should never happen, but just in case ... */
            PyErr_Format(PyExc_SystemError, "unknown cmpop found");
            return NULL;
    }
}

Project: typed_ast

Return(expr_ty value, int lineno, int col_offset, PyArena *arena)
{
    stmt_ty p;
    p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = Return_kind;
    p->v.Return.value = value;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast
File: Python-ast.c

ExtSlice(asdl_seq * dims, PyArena *arena)
{
    slice_ty p;
    p = (slice_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = ExtSlice_kind;
    p->v.ExtSlice.dims = dims;
    return p;
}

Project: typed_ast

Try(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse, asdl_seq *
    finalbody, int lineno, int col_offset, PyArena *arena)
{
    stmt_ty p;
    p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = Try_kind;
    p->v.Try.body = body;
    p->v.Try.handlers = handlers;
    p->v.Try.orelse = orelse;
    p->v.Try.finalbody = finalbody;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast
File: tokenizer.c

get_normal_name(const char *s)  /* for utf-8 and latin-1 */
{
    char buf[13];
    int i;
    for (i = 0; i < 12; i++) {
        int c = s[i];
        if (c == '\0')
            break;
        else if (c == '_')
            buf[i] = '-';
        else
            buf[i] = tolower(c);
    }
    buf[i] = '\0';
    if (strcmp(buf, "utf-8") == 0 ||
        strncmp(buf, "utf-8-", 6) == 0)
        return "utf-8";
    else if (strcmp(buf, "latin-1") == 0 ||
             strcmp(buf, "iso-8859-1") == 0 ||
             strcmp(buf, "iso-latin-1") == 0 ||
             strncmp(buf, "latin-1-", 8) == 0 ||
             strncmp(buf, "iso-8859-1-", 11) == 0 ||
             strncmp(buf, "iso-latin-1-", 12) == 0)
        return "iso-8859-1";
    else
        return s;
}

Project: typed_ast

ast_for_decorators(struct compiling *c, const node *n)
{
    asdl_seq* decorator_seq;
    expr_ty d;
    int i;

    REQ(n, decorators);
    decorator_seq = asdl_seq_new(NCH(n), c->c_arena);
    if (!decorator_seq)
        return NULL;

    for (i = 0; i < NCH(n); i++) {
        d = ast_for_decorator(c, CHILD(n, i));
        if (!d)
            return NULL;
        asdl_seq_SET(decorator_seq, i, d);
    }
    return decorator_seq;
}

Project: typed_ast

ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
{
    /* classdef: 'class' NAME ['(' testlist ')'] ':' suite */
    PyObject *classname;
    asdl_seq *bases, *s;

    REQ(n, classdef);

    if (!forbidden_check(c, n, STR(CHILD(n, 1))))
            return NULL;

    if (NCH(n) == 4) {
        s = ast_for_suite(c, CHILD(n, 3));
        if (!s)
            return NULL;
        classname = NEW_IDENTIFIER(CHILD(n, 1));
        if (!classname)
            return NULL;
        return ClassDef(classname, NULL, s, decorator_seq, LINENO(n),
                        n->n_col_offset, c->c_arena);
    }
    /* check for empty base list */
    if (TYPE(CHILD(n,3)) == RPAR) {
        s = ast_for_suite(c, CHILD(n,5));
        if (!s)
            return NULL;
        classname = NEW_IDENTIFIER(CHILD(n, 1));
        if (!classname)
            return NULL;
        return ClassDef(classname, NULL, s, decorator_seq, LINENO(n),
                        n->n_col_offset, c->c_arena);
    }

    /* else handle the base class list */
    bases = ast_for_class_bases(c, CHILD(n, 3));
    if (!bases)
        return NULL;

    s = ast_for_suite(c, CHILD(n, 6));
    if (!s)
        return NULL;
    classname = NEW_IDENTIFIER(CHILD(n, 1));
    if (!classname)
        return NULL;
    return ClassDef(classname, bases, s, decorator_seq,
                    LINENO(n), n->n_col_offset, c->c_arena);
}

Project: typed_ast

count_comp_fors(struct compiling *c, const node *n)
{
    int n_fors = 0;

  count_comp_for:
    n_fors++;
    REQ(n, comp_for);
    if (NCH(n) == 5)
        n = CHILD(n, 4);
    else
        return n_fors;
  count_comp_iter:
    REQ(n, comp_iter);
    n = CHILD(n, 0);
    if (TYPE(n) == comp_for)
        goto count_comp_for;
    else if (TYPE(n) == comp_if) {
        if (NCH(n) == 3) {
            n = CHILD(n, 2);
            goto count_comp_iter;
        }
        else
            return n_fors;
    }

    /* Should never be reached */
    PyErr_SetString(PyExc_SystemError,
                    "logic error in count_comp_fors");
    return -1;
}

Project: typed_ast
File: Python-ast.c

ast2obj_comprehension(void* _o)
{
    comprehension_ty o = (comprehension_ty)_o;
    PyObject *result = NULL, *value = NULL;
    if (!o) {
        Py_RETURN_NONE;
    }

    result = PyType_GenericNew(comprehension_type, NULL, NULL);
    if (!result) return NULL;
    value = ast2obj_expr(o->target);
    if (!value) goto failed;
    if (_PyObject_SetAttrId(result, &PyId_target, value) == -1)
        goto failed;
    Py_DECREF(value);
    value = ast2obj_expr(o->iter);
    if (!value) goto failed;
    if (_PyObject_SetAttrId(result, &PyId_iter, value) == -1)
        goto failed;
    Py_DECREF(value);
    value = ast2obj_list(o->ifs, ast2obj_expr);
    if (!value) goto failed;
    if (_PyObject_SetAttrId(result, &PyId_ifs, value) == -1)
        goto failed;
    Py_DECREF(value);
    value = ast2obj_int(o->is_async);
    if (!value) goto failed;
    if (_PyObject_SetAttrId(result, &PyId_is_async, value) == -1)
        goto failed;
    Py_DECREF(value);
    return result;
failed:
    Py_XDECREF(value);
    Py_XDECREF(result);
    return NULL;
}

Project: typed_ast

ast_for_expr(struct compiling *c, const node *n)
{
    /* handle the full range of simple expressions
       test: or_test ['if' or_test 'else' test] | lambdef
       or_test: and_test ('or' and_test)*
       and_test: not_test ('and' not_test)*
       not_test: 'not' not_test | comparison
       comparison: expr (comp_op expr)*
       expr: xor_expr ('|' xor_expr)*
       xor_expr: and_expr ('^' and_expr)*
       and_expr: shift_expr ('&' shift_expr)*
       shift_expr: arith_expr (('<<'|'>>') arith_expr)*
       arith_expr: term (('+'|'-') term)*
       term: factor (('*'|'/'|'%'|'//') factor)*
       factor: ('+'|'-'|'~') factor | power
       power: atom trailer* ('**' factor)*

       As well as modified versions that exist for backward compatibility,
       to explicitly allow:
       [ x for x in lambda: 0, lambda: 1 ]
       (which would be ambiguous without these extra rules)

       old_test: or_test | old_lambdef
       old_lambdef: 'lambda' [vararglist] ':' old_test

    */

    asdl_seq *seq;
    int i;

 loop:
    switch (TYPE(n)) {
        case test:
        case old_test:
            if (TYPE(CHILD(n, 0)) == lambdef ||
                TYPE(CHILD(n, 0)) == old_lambdef)
                return ast_for_lambdef(c, CHILD(n, 0));
            else if (NCH(n) > 1)
                return ast_for_ifexpr(c, n);
            /* Fallthrough */
        case or_test:
        case and_test:
            if (NCH(n) == 1) {
                n = CHILD(n, 0);
                goto loop;
            }
            seq = asdl_seq_new((NCH(n) + 1) / 2, c->c_arena);
            if (!seq)
                return NULL;
            for (i = 0; i < NCH(n); i += 2) {
                expr_ty e = ast_for_expr(c, CHILD(n, i));
                if (!e)
                    return NULL;
                asdl_seq_SET(seq, i / 2, e);
            }
            if (!strcmp(STR(CHILD(n, 1)), "and"))
                return BoolOp(And, seq, LINENO(n), n->n_col_offset,
                              c->c_arena);
            assert(!strcmp(STR(CHILD(n, 1)), "or"));
            return BoolOp(Or, seq, LINENO(n), n->n_col_offset, c->c_arena);
        case not_test:
            if (NCH(n) == 1) {
                n = CHILD(n, 0);
                goto loop;
            }
            else {
                expr_ty expression = ast_for_expr(c, CHILD(n, 1));
                if (!expression)
                    return NULL;

                return UnaryOp(Not, expression, LINENO(n), n->n_col_offset,
                               c->c_arena);
            }
        case comparison:
            if (NCH(n) == 1) {
                n = CHILD(n, 0);
                goto loop;
            }
            else {
                expr_ty expression;
                asdl_int_seq *ops;
                asdl_seq *cmps;
                ops = asdl_int_seq_new(NCH(n) / 2, c->c_arena);
                if (!ops)
                    return NULL;
                cmps = asdl_seq_new(NCH(n) / 2, c->c_arena);
                if (!cmps) {
                    return NULL;
                }
                for (i = 1; i < NCH(n); i += 2) {
                    cmpop_ty newoperator;

                    newoperator = ast_for_comp_op(c, CHILD(n, i));
                    if (!newoperator) {
                        return NULL;
                    }

                    expression = ast_for_expr(c, CHILD(n, i + 1));
                    if (!expression) {
                        return NULL;
                    }

                    asdl_seq_SET(ops, i / 2, newoperator);
                    asdl_seq_SET(cmps, i / 2, expression);
                }
                expression = ast_for_expr(c, CHILD(n, 0));
                if (!expression) {
                    return NULL;
                }

                return Compare(expression, ops, cmps, LINENO(n),
                               n->n_col_offset, c->c_arena);
            }
            break;

        /* The next five cases all handle BinOps.  The main body of code
           is the same in each case, but the switch turned inside out to
           reuse the code for each type of operator.
         */
        case expr:
        case xor_expr:
        case and_expr:
        case shift_expr:
        case arith_expr:
        case term:
            if (NCH(n) == 1) {
                n = CHILD(n, 0);
                goto loop;
            }
            return ast_for_binop(c, n);
        case yield_expr: {
            expr_ty exp = NULL;
            if (NCH(n) == 2) {
                exp = ast_for_testlist(c, CHILD(n, 1));
                if (!exp)
                    return NULL;
            }
            return Yield(exp, LINENO(n), n->n_col_offset, c->c_arena);
        }
        case factor:
            if (NCH(n) == 1) {
                n = CHILD(n, 0);
                goto loop;
            }
            return ast_for_factor(c, n);
        case power:
            return ast_for_power(c, n);
        default:
            PyErr_Format(PyExc_SystemError, "unhandled expr: %d", TYPE(n));
            return NULL;
    }
    /* should never get here unless if error is set */
    return NULL;
}

Project: typed_ast

Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset, PyArena
      *arena)
{
    expr_ty p;
    if (!ctx) {
        PyErr_SetString(PyExc_ValueError,
                        "field ctx is required for Tuple");
        return NULL;
    }
    p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = Tuple_kind;
    p->v.Tuple.elts = elts;
    p->v.Tuple.ctx = ctx;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast

ast_for_itercomp(struct compiling *c, const node *n, int type)
{
    expr_ty elt;
    asdl_seq *comps;

    assert(NCH(n) > 1);

    elt = ast_for_expr(c, CHILD(n, 0));
    if (!elt)
        return NULL;

    comps = ast_for_comprehension(c, CHILD(n, 1));
    if (!comps)
        return NULL;

    if (type == COMP_GENEXP)
        return GeneratorExp(elt, comps, LINENO(n), n->n_col_offset, c->c_arena);
    else if (type == COMP_SETCOMP)
        return SetComp(elt, comps, LINENO(n), n->n_col_offset, c->c_arena);
    else
        /* Should never happen */
        return NULL;
}

Project: typed_ast
File: Python-ast.c

ast2obj_arguments(void* _o)
{
    arguments_ty o = (arguments_ty)_o;
    PyObject *result = NULL, *value = NULL;
    if (!o) {
        Py_RETURN_NONE;
    }

    result = PyType_GenericNew(arguments_type, NULL, NULL);
    if (!result) return NULL;
    value = ast2obj_list(o->args, ast2obj_arg);
    if (!value) goto failed;
    if (_PyObject_SetAttrId(result, &PyId_args, value) == -1)
        goto failed;
    Py_DECREF(value);
    value = ast2obj_arg(o->vararg);
    if (!value) goto failed;
    if (_PyObject_SetAttrId(result, &PyId_vararg, value) == -1)
        goto failed;
    Py_DECREF(value);
    value = ast2obj_list(o->kwonlyargs, ast2obj_arg);
    if (!value) goto failed;
    if (_PyObject_SetAttrId(result, &PyId_kwonlyargs, value) == -1)
        goto failed;
    Py_DECREF(value);
    value = ast2obj_list(o->kw_defaults, ast2obj_expr);
    if (!value) goto failed;
    if (_PyObject_SetAttrId(result, &PyId_kw_defaults, value) == -1)
        goto failed;
    Py_DECREF(value);
    value = ast2obj_arg(o->kwarg);
    if (!value) goto failed;
    if (_PyObject_SetAttrId(result, &PyId_kwarg, value) == -1)
        goto failed;
    Py_DECREF(value);
    value = ast2obj_list(o->defaults, ast2obj_expr);
    if (!value) goto failed;
    if (_PyObject_SetAttrId(result, &PyId_defaults, value) == -1)
        goto failed;
    Py_DECREF(value);
    return result;
failed:
    Py_XDECREF(value);
    Py_XDECREF(result);
    return NULL;
}

Project: typed_ast

PyObject* Ta3AST_mod2obj(mod_ty t)
{
    if (!init_types())
        return NULL;
    return ast2obj_mod(t);
}

Project: typed_ast

Ta3Parser_ParseStringObject(const char *s, PyObject *filename,
                           grammar *g, int start,
                           perrdetail *err_ret, int *flags)
{
    struct tok_state *tok;
    int exec_input = start == file_input;

    if (initerr(err_ret, filename) < 0)
        return NULL;

    if (*flags & PyPARSE_IGNORE_COOKIE)
        tok = Ta3Tokenizer_FromUTF8(s, exec_input);
    else
        tok = Ta3Tokenizer_FromString(s, exec_input);
    if (tok == NULL) {
        err_ret->error = PyErr_Occurred() ? E_DECODE : E_NOMEM;
        return NULL;
    }

#ifndef PGEN
    Py_INCREF(err_ret->filename);
    tok->filename = err_ret->filename;
#endif
    if (*flags & PyPARSE_ASYNC_ALWAYS)
        tok->async_always = 1;
    return parsetok(tok, g, start, err_ret, flags);
}

Project: typed_ast
File: Python-ast.c

Module(asdl_seq * body, asdl_seq * type_ignores, PyArena *arena)
{
    mod_ty p;
    p = (mod_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = Module_kind;
    p->v.Module.body = body;
    p->v.Module.type_ignores = type_ignores;
    return p;
}

Project: typed_ast

YieldFrom(expr_ty value, int lineno, int col_offset, PyArena *arena)
{
    expr_ty p;
    if (!value) {
        PyErr_SetString(PyExc_ValueError,
                        "field value is required for YieldFrom");
        return NULL;
    }
    p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = YieldFrom_kind;
    p->v.YieldFrom.value = value;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast

Await(expr_ty value, int lineno, int col_offset, PyArena *arena)
{
    expr_ty p;
    if (!value) {
        PyErr_SetString(PyExc_ValueError,
                        "field value is required for Await");
        return NULL;
    }
    p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = Await_kind;
    p->v.Await.value = value;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast
File: Python-ast.c

obj2ast_unaryop(PyObject* obj, unaryop_ty* out, PyArena* arena)
{
    int isinstance;

    isinstance = PyObject_IsInstance(obj, (PyObject *)Invert_type);
    if (isinstance == -1) {
        return 1;
    }
    if (isinstance) {
        *out = Invert;
        return 0;
    }
    isinstance = PyObject_IsInstance(obj, (PyObject *)Not_type);
    if (isinstance == -1) {
        return 1;
    }
    if (isinstance) {
        *out = Not;
        return 0;
    }
    isinstance = PyObject_IsInstance(obj, (PyObject *)UAdd_type);
    if (isinstance == -1) {
        return 1;
    }
    if (isinstance) {
        *out = UAdd;
        return 0;
    }
    isinstance = PyObject_IsInstance(obj, (PyObject *)USub_type);
    if (isinstance == -1) {
        return 1;
    }
    if (isinstance) {
        *out = USub;
        return 0;
    }

    PyErr_Format(PyExc_TypeError, "expected some sort of unaryop, but got %R", obj);
    return 1;
}

Project: typed_ast

alias_for_import_name(struct compiling *c, const node *n, int store)
{
    /*
      import_as_name: NAME ['as' NAME]
      dotted_as_name: dotted_name ['as' NAME]
      dotted_name: NAME ('.' NAME)*
    */
    PyObject *str, *name;

 loop:
    switch (TYPE(n)) {
         case import_as_name: {
            node *name_node = CHILD(n, 0);
            str = NULL;
            if (NCH(n) == 3) {
                node *str_node = CHILD(n, 2);
                if (store && !forbidden_check(c, str_node, STR(str_node)))
                    return NULL;
                str = NEW_IDENTIFIER(str_node);
                if (!str)
                    return NULL;
            }
            else {
                if (!forbidden_check(c, name_node, STR(name_node)))
                    return NULL;
            }
            name = NEW_IDENTIFIER(name_node);
            if (!name)
                return NULL;
            return alias(name, str, c->c_arena);
        }
        case dotted_as_name:
            if (NCH(n) == 1) {
                n = CHILD(n, 0);
                goto loop;
            }
            else {
                node *asname_node = CHILD(n, 2);
                alias_ty a = alias_for_import_name(c, CHILD(n, 0), 0);
                if (!a)
                    return NULL;
                assert(!a->asname);
                if (!forbidden_check(c, asname_node, STR(asname_node)))
                    return NULL;
                a->asname = NEW_IDENTIFIER(asname_node);
                if (!a->asname)
                    return NULL;
                return a;
            }
            break;
        case dotted_name:
            if (NCH(n) == 1) {
                node *name_node = CHILD(n, 0);
                if (store && !forbidden_check(c, name_node, STR(name_node)))
                    return NULL;
                name = NEW_IDENTIFIER(name_node);
                if (!name)
                    return NULL;
                return alias(name, NULL, c->c_arena);
            }
            else {
                /* Create a string of the form "a.b.c" */
                int i;
                size_t len;
                char *s;
                PyObject *uni;

                len = 0;
                for (i = 0; i < NCH(n); i += 2)
                    /* length of string plus one for the dot */
                    len += strlen(STR(CHILD(n, i))) + 1;
                len--; /* the last name doesn't have a dot */
                str = PyBytes_FromStringAndSize(NULL, len);
                if (!str)
                    return NULL;
                s = PyBytes_AS_STRING(str);
                if (!s)
                    return NULL;
                for (i = 0; i < NCH(n); i += 2) {
                    char *sch = STR(CHILD(n, i));
                    strcpy(s, STR(CHILD(n, i)));
                    s += strlen(sch);
                    *s++ = '.';
                }
                --s;
                *s = '\0';
                uni = PyUnicode_DecodeUTF8(PyBytes_AS_STRING(str),
                                           PyBytes_GET_SIZE(str),
                                           NULL);
                Py_DECREF(str);
                if (!uni)
                    return NULL;
                str = uni;
                PyUnicode_InternInPlace(&str);
                if (PyArena_AddPyObject(c->c_arena, str) < 0) {
                    Py_DECREF(str);
                    return NULL;
                }
                return alias(str, NULL, c->c_arena);
            }
            break;
        case STAR:
            str = PyUnicode_InternFromString("*");
            if (PyArena_AddPyObject(c->c_arena, str) < 0) {
                Py_DECREF(str);
                return NULL;
            }
            return alias(str, NULL, c->c_arena);
        default:
            PyErr_Format(PyExc_SystemError,
                         "unexpected import name: %d", TYPE(n));
            return NULL;
    }

    PyErr_SetString(PyExc_SystemError, "unhandled import name condition");
    return NULL;
}

Project: typed_ast
File: Python-ast.c

keyword(identifier arg, expr_ty value, PyArena *arena)
{
    keyword_ty p;
    if (!value) {
        PyErr_SetString(PyExc_ValueError,
                        "field value is required for keyword");
        return NULL;
    }
    p = (keyword_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->arg = arg;
    p->value = value;
    return p;
}

Project: typed_ast

ast_for_comp_op(struct compiling *c, const node *n)
{
    /* comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'
               |'is' 'not'
    */
    REQ(n, comp_op);
    if (NCH(n) == 1) {
        n = CHILD(n, 0);
        switch (TYPE(n)) {
            case LESS:
                return Lt;
            case GREATER:
                return Gt;
            case EQEQUAL:                       /* == */
                return Eq;
            case LESSEQUAL:
                return LtE;
            case GREATEREQUAL:
                return GtE;
            case NOTEQUAL:
                return NotEq;
            case NAME:
                if (strcmp(STR(n), "in") == 0)
                    return In;
                if (strcmp(STR(n), "is") == 0)
                    return Is;
            default:
                PyErr_Format(PyExc_SystemError, "invalid comp_op: %s",
                             STR(n));
                return (cmpop_ty)0;
        }
    }
    else if (NCH(n) == 2) {
        /* handle "not in" and "is not" */
        switch (TYPE(CHILD(n, 0))) {
            case NAME:
                if (strcmp(STR(CHILD(n, 1)), "in") == 0)
                    return NotIn;
                if (strcmp(STR(CHILD(n, 0)), "is") == 0)
                    return IsNot;
            default:
                PyErr_Format(PyExc_SystemError, "invalid comp_op: %s %s",
                             STR(CHILD(n, 0)), STR(CHILD(n, 1)));
                return (cmpop_ty)0;
        }
    }
    PyErr_Format(PyExc_SystemError, "invalid comp_op: has %d children",
                 NCH(n));
    return (cmpop_ty)0;
}

Project: typed_ast
File: Python-ast.c

TypeIgnore(int lineno, PyArena *arena)
{
    type_ignore_ty p;
    p = (type_ignore_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = TypeIgnore_kind;
    p->v.TypeIgnore.lineno = lineno;
    return p;
}

Project: typed_ast

ast_for_try_stmt(struct compiling *c, const node *n)
{
    const int nch = NCH(n);
    int n_except = (nch - 3)/3;
    asdl_seq *body, *orelse = NULL, *finally = NULL;

    REQ(n, try_stmt);

    body = ast_for_suite(c, CHILD(n, 2));
    if (body == NULL)
        return NULL;

    if (TYPE(CHILD(n, nch - 3)) == NAME) {
        if (strcmp(STR(CHILD(n, nch - 3)), "finally") == 0) {
            if (nch >= 9 && TYPE(CHILD(n, nch - 6)) == NAME) {
                /* we can assume it's an "else",
                   because nch >= 9 for try-else-finally and
                   it would otherwise have a type of except_clause */
                orelse = ast_for_suite(c, CHILD(n, nch - 4));
                if (orelse == NULL)
                    return NULL;
                n_except--;
            }

            finally = ast_for_suite(c, CHILD(n, nch - 1));
            if (finally == NULL)
                return NULL;
            n_except--;
        }
        else {
            /* we can assume it's an "else",
               otherwise it would have a type of except_clause */
            orelse = ast_for_suite(c, CHILD(n, nch - 1));
            if (orelse == NULL)
                return NULL;
            n_except--;
        }
    }
    else if (TYPE(CHILD(n, nch - 3)) != except_clause) {
        ast_error(n, "malformed 'try' statement");
        return NULL;
    }

    if (n_except > 0) {
        int i;
        stmt_ty except_st;
        /* process except statements to create a try ... except */
        asdl_seq *handlers = asdl_seq_new(n_except, c->c_arena);
        if (handlers == NULL)
            return NULL;

        for (i = 0; i < n_except; i++) {
            excepthandler_ty e = ast_for_except_clause(c, CHILD(n, 3 + i * 3),
                                                       CHILD(n, 5 + i * 3));
            if (!e)
                return NULL;
            asdl_seq_SET(handlers, i, e);
        }

        except_st = TryExcept(body, handlers, orelse, LINENO(n),
                              n->n_col_offset, c->c_arena);
        if (!finally)
            return except_st;

        /* if a 'finally' is present too, we nest the TryExcept within a
           TryFinally to emulate try ... except ... finally */
        body = asdl_seq_new(1, c->c_arena);
        if (body == NULL)
            return NULL;
        asdl_seq_SET(body, 0, except_st);
    }

    /* must be a try ... finally (except clauses are in body, if any exist) */
    assert(finally != NULL);
    return TryFinally(body, finally, LINENO(n), n->n_col_offset, c->c_arena);
}

Project: typed_ast
File: tokenizer.c

static int valid_utf8(const unsigned char* s)
{
    int expected = 0;
    int length;
    if (*s < 0x80)
        /* single-byte code */
        return 1;
    if (*s < 0xc0)
        /* following byte */
        return 0;
    if (*s < 0xE0)
        expected = 1;
    else if (*s < 0xF0)
        expected = 2;
    else if (*s < 0xF8)
        expected = 3;
    else
        return 0;
    length = expected + 1;
    for (; expected; expected--)
        if (s[expected] < 0x80 || s[expected] >= 0xC0)
            return 0;
    return length;
}

Project: typed_ast

Ta3Parser_ParseString(const char *s, grammar *g, int start, perrdetail *err_ret)
{
    return Ta3Parser_ParseStringFlagsFilename(s, NULL, g, start, err_ret, 0);
}

Project: typed_ast

ast_for_expr_stmt(struct compiling *c, const node *n)
{
    int num;

    REQ(n, expr_stmt);
    /* expr_stmt: testlist_star_expr (annassign | augassign (yield_expr|testlist) |
                           ('=' (yield_expr|testlist_star_expr))* [TYPE_COMMENT])
       annassign: ':' test ['=' test]
       testlist_star_expr: (test|star_expr) (',' test|star_expr)* [',']
       augassign: '+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^='
                | '<<=' | '>>=' | '**=' | '//='
       test: ... here starts the operator precedence dance
     */
    num = NCH(n);

    if (num == 1 || (num == 2 && TYPE(CHILD(n, 1)) == TYPE_COMMENT)) {
        expr_ty e = ast_for_testlist(c, CHILD(n, 0));
        if (!e)
            return NULL;

        return Expr(e, LINENO(n), n->n_col_offset, c->c_arena);
    }
    else if (TYPE(CHILD(n, 1)) == augassign) {
        expr_ty expr1, expr2;
        operator_ty newoperator;
        node *ch = CHILD(n, 0);

        expr1 = ast_for_testlist(c, ch);
        if (!expr1)
            return NULL;
        if(!set_context(c, expr1, Store, ch))
            return NULL;
        /* set_context checks that most expressions are not the left side.
          Augmented assignments can only have a name, a subscript, or an
          attribute on the left, though, so we have to explicitly check for
          those. */
        switch (expr1->kind) {
            case Name_kind:
            case Attribute_kind:
            case Subscript_kind:
                break;
            default:
                ast_error(c, ch, "illegal expression for augmented assignment");
                return NULL;
        }

        ch = CHILD(n, 2);
        if (TYPE(ch) == testlist)
            expr2 = ast_for_testlist(c, ch);
        else
            expr2 = ast_for_expr(c, ch);
        if (!expr2)
            return NULL;

        newoperator = ast_for_augassign(c, CHILD(n, 1));
        if (!newoperator)
            return NULL;

        return AugAssign(expr1, newoperator, expr2, LINENO(n), n->n_col_offset, c->c_arena);
    }
    else if (TYPE(CHILD(n, 1)) == annassign) {
        expr_ty expr1, expr2, expr3;
        node *ch = CHILD(n, 0);
        node *deep, *ann = CHILD(n, 1);
        int simple = 1;

        /* AnnAssigns are only allowed in Python 3.6 or greater */
        if (c->c_feature_version < 6) {
            ast_error(c, ch,
                    "Variable annotation syntax is only supported in Python 3.6 and greater");
            return NULL;
        }

        /* we keep track of parens to qualify (x) as expression not name */
        deep = ch;
        while (NCH(deep) == 1) {
            deep = CHILD(deep, 0);
        }
        if (NCH(deep) > 0 && TYPE(CHILD(deep, 0)) == LPAR) {
            simple = 0;
        }
        expr1 = ast_for_testlist(c, ch);
        if (!expr1) {
            return NULL;
        }
        switch (expr1->kind) {
            case Name_kind:
                if (forbidden_name(c, expr1->v.Name.id, n, 0)) {
                    return NULL;
                }
                expr1->v.Name.ctx = Store;
                break;
            case Attribute_kind:
                if (forbidden_name(c, expr1->v.Attribute.attr, n, 1)) {
                    return NULL;
                }
                expr1->v.Attribute.ctx = Store;
                break;
            case Subscript_kind:
                expr1->v.Subscript.ctx = Store;
                break;
            case List_kind:
                ast_error(c, ch,
                          "only single target (not list) can be annotated");
                return NULL;
            case Tuple_kind:
                ast_error(c, ch,
                          "only single target (not tuple) can be annotated");
                return NULL;
            default:
                ast_error(c, ch,
                          "illegal target for annotation");
                return NULL;
        }

        if (expr1->kind != Name_kind) {
            simple = 0;
        }
        ch = CHILD(ann, 1);
        expr2 = ast_for_expr(c, ch);
        if (!expr2) {
            return NULL;
        }
        if (NCH(ann) == 2) {
            return AnnAssign(expr1, expr2, NULL, simple,
                             LINENO(n), n->n_col_offset, c->c_arena);
        }
        else {
            ch = CHILD(ann, 3);
            expr3 = ast_for_expr(c, ch);
            if (!expr3) {
                return NULL;
            }
            return AnnAssign(expr1, expr2, expr3, simple,
                             LINENO(n), n->n_col_offset, c->c_arena);
        }
    }
    else {
        int i, nch_minus_type, has_type_comment;
        asdl_seq *targets;
        node *value;
        expr_ty expression;
        string type_comment;

        /* a normal assignment */
        REQ(CHILD(n, 1), EQUAL);

        has_type_comment = TYPE(CHILD(n, num - 1)) == TYPE_COMMENT;
        nch_minus_type = num - has_type_comment;

        targets = _Ta3_asdl_seq_new(nch_minus_type / 2, c->c_arena);
        if (!targets)
            return NULL;
        for (i = 0; i < nch_minus_type - 2; i += 2) {
            expr_ty e;
            node *ch = CHILD(n, i);
            if (TYPE(ch) == yield_expr) {
                ast_error(c, ch, "assignment to yield expression not possible");
                return NULL;
            }
            e = ast_for_testlist(c, ch);
            if (!e)
              return NULL;

            /* set context to assign */
            if (!set_context(c, e, Store, CHILD(n, i)))
              return NULL;

            asdl_seq_SET(targets, i / 2, e);
        }
        value = CHILD(n, nch_minus_type - 1);
        if (TYPE(value) == testlist_star_expr)
            expression = ast_for_testlist(c, value);
        else
            expression = ast_for_expr(c, value);
        if (!expression)
            return NULL;
        if (has_type_comment)
            type_comment = NEW_TYPE_COMMENT(CHILD(n, nch_minus_type));
        else
            type_comment = NULL;
        return Assign(targets, expression, type_comment, LINENO(n), n->n_col_offset, c->c_arena);
    }
}

Project: typed_ast

BoolOp(boolop_ty op, asdl_seq * values, int lineno, int col_offset, PyArena
       *arena)
{
    expr_ty p;
    if (!op) {
        PyErr_SetString(PyExc_ValueError,
                        "field op is required for BoolOp");
        return NULL;
    }
    p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = BoolOp_kind;
    p->v.BoolOp.op = op;
    p->v.BoolOp.values = values;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast

ast_for_funcdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
{
    /* funcdef: 'def' NAME parameters ':' [TYPE_COMMENT] suite */
    identifier name;
    arguments_ty args;
    asdl_seq *body;
    int name_i = 1;
    node *tc;
    string type_comment = NULL;

    REQ(n, funcdef);

    name = NEW_IDENTIFIER(CHILD(n, name_i));
    if (!name)
        return NULL;
    else if (!forbidden_check(c, CHILD(n, name_i), STR(CHILD(n, name_i))))
        return NULL;
    args = ast_for_arguments(c, CHILD(n, name_i + 1));
    if (!args)
        return NULL;
    if (TYPE(CHILD(n, name_i + 3)) == TYPE_COMMENT) {
        type_comment = NEW_TYPE_COMMENT(CHILD(n, name_i + 3));
        name_i += 1;
    }
    body = ast_for_suite(c, CHILD(n, name_i + 3));
    if (!body)
        return NULL;

    if (!type_comment && NCH(CHILD(n, name_i + 3)) > 1) {
        /* If the function doesn't have a type comment on the same line, check
         * if the suite has a type comment in it. */
        tc = CHILD(CHILD(n, name_i + 3), 1);

        if (TYPE(tc) == TYPE_COMMENT)
            type_comment = NEW_TYPE_COMMENT(tc);
    }

    return FunctionDef(name, args, body, decorator_seq, type_comment, LINENO(n),
                       n->n_col_offset, c->c_arena);
}

Project: typed_ast

ast_for_arguments(struct compiling *c, const node *n)
{
    /* This function handles both typedargslist (function definition)
       and varargslist (lambda definition).

       parameters: '(' [typedargslist] ')'
       typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [',' [
               '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
             | '**' tfpdef [',']]]
         | '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
         | '**' tfpdef [','])
       tfpdef: NAME [':' test]
       varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [
               '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
             | '**' vfpdef [',']]]
         | '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
         | '**' vfpdef [',']
       )
       vfpdef: NAME

    */
    int i, j, k, nposargs = 0, nkwonlyargs = 0;
    int nposdefaults = 0, found_default = 0;
    asdl_seq *posargs, *posdefaults, *kwonlyargs, *kwdefaults;
    arg_ty vararg = NULL, kwarg = NULL;
    arg_ty arg;
    node *ch;

    if (TYPE(n) == parameters) {
        if (NCH(n) == 2) /* () as argument list */
            return arguments(NULL, NULL, NULL, NULL, NULL, NULL, c->c_arena);
        n = CHILD(n, 1);
    }
    assert(TYPE(n) == typedargslist || TYPE(n) == varargslist);

    /* First count the number of positional args & defaults.  The
       variable i is the loop index for this for loop and the next.
       The next loop picks up where the first leaves off.
    */
    for (i = 0; i < NCH(n); i++) {
        ch = CHILD(n, i);
        if (TYPE(ch) == STAR) {
            /* skip star */
            i++;
            if (i < NCH(n) && /* skip argument following star */
                (TYPE(CHILD(n, i)) == tfpdef ||
                 TYPE(CHILD(n, i)) == vfpdef)) {
                i++;
            }
            break;
        }
        if (TYPE(ch) == DOUBLESTAR) break;
        if (TYPE(ch) == vfpdef || TYPE(ch) == tfpdef) nposargs++;
        if (TYPE(ch) == EQUAL) nposdefaults++;
    }
    /* count the number of keyword only args &
       defaults for keyword only args */
    for ( ; i < NCH(n); ++i) {
        ch = CHILD(n, i);
        if (TYPE(ch) == DOUBLESTAR) break;
        if (TYPE(ch) == tfpdef || TYPE(ch) == vfpdef) nkwonlyargs++;
    }
    posargs = (nposargs ? _Ta3_asdl_seq_new(nposargs, c->c_arena) : NULL);
    if (!posargs && nposargs)
        return NULL;
    kwonlyargs = (nkwonlyargs ?
                   _Ta3_asdl_seq_new(nkwonlyargs, c->c_arena) : NULL);
    if (!kwonlyargs && nkwonlyargs)
        return NULL;
    posdefaults = (nposdefaults ?
                    _Ta3_asdl_seq_new(nposdefaults, c->c_arena) : NULL);
    if (!posdefaults && nposdefaults)
        return NULL;
    /* The length of kwonlyargs and kwdefaults are same
       since we set NULL as default for keyword only argument w/o default
       - we have sequence data structure, but no dictionary */
    kwdefaults = (nkwonlyargs ?
                   _Ta3_asdl_seq_new(nkwonlyargs, c->c_arena) : NULL);
    if (!kwdefaults && nkwonlyargs)
        return NULL;

    /* tfpdef: NAME [':' test]
       vfpdef: NAME
    */
    i = 0;
    j = 0;  /* index for defaults */
    k = 0;  /* index for args */
    while (i < NCH(n)) {
        ch = CHILD(n, i);
        switch (TYPE(ch)) {
            case tfpdef:
            case vfpdef:
                /* XXX Need to worry about checking if TYPE(CHILD(n, i+1)) is
                   anything other than EQUAL or a comma? */
                /* XXX Should NCH(n) check be made a separate check? */
                if (i + 1 < NCH(n) && TYPE(CHILD(n, i + 1)) == EQUAL) {
                    expr_ty expression = ast_for_expr(c, CHILD(n, i + 2));
                    if (!expression)
                        return NULL;
                    assert(posdefaults != NULL);
                    asdl_seq_SET(posdefaults, j++, expression);
                    i += 2;
                    found_default = 1;
                }
                else if (found_default) {
                    ast_error(c, n,
                             "non-default argument follows default argument");
                    return NULL;
                }
                arg = ast_for_arg(c, ch);
                if (!arg)
                    return NULL;
                asdl_seq_SET(posargs, k++, arg);
                i += 1; /* the name */
                if (TYPE(CHILD(n, i)) == COMMA)
                    i += 1; /* the comma, if present */
                break;
            case STAR:
                if (i+1 >= NCH(n) ||
                    (i+2 == NCH(n) && (TYPE(CHILD(n, i+1)) == COMMA
                                       || TYPE(CHILD(n, i+1)) == TYPE_COMMENT))) {
                    ast_error(c, CHILD(n, i),
                        "named arguments must follow bare *");
                    return NULL;
                }
                ch = CHILD(n, i+1);  /* tfpdef or COMMA */
                if (TYPE(ch) == COMMA) {
                    int res = 0;
                    i += 2; /* now follows keyword only arguments */

                    if (TYPE(CHILD(n, i)) == TYPE_COMMENT) {
                        ast_error(c, CHILD(n, i),
                                "bare * has associated type comment");
                        return NULL;
                    }

                    res = handle_keywordonly_args(c, n, i,
                                                  kwonlyargs, kwdefaults);
                    if (res == -1) return NULL;
                    i = res; /* res has new position to process */
                }
                else {
                    vararg = ast_for_arg(c, ch);
                    if (!vararg)
                        return NULL;

                i += 2; /* the star and the name */
                if (TYPE(CHILD(n, i)) == COMMA)
                    i += 1; /* the comma, if present */

                    if (TYPE(CHILD(n, i)) == TYPE_COMMENT) {
                        vararg->type_comment = NEW_TYPE_COMMENT(CHILD(n, i));
                        i += 1;
                    }

                    if (i < NCH(n) && (TYPE(CHILD(n, i)) == tfpdef
                                    || TYPE(CHILD(n, i)) == vfpdef)) {
                        int res = 0;
                        res = handle_keywordonly_args(c, n, i,
                                                      kwonlyargs, kwdefaults);
                        if (res == -1) return NULL;
                        i = res; /* res has new position to process */
                    }
                }
                break;
            case DOUBLESTAR:
                ch = CHILD(n, i+1);  /* tfpdef */
                assert(TYPE(ch) == tfpdef || TYPE(ch) == vfpdef);
                kwarg = ast_for_arg(c, ch);
                if (!kwarg)
                    return NULL;
                i += 2; /* the double star and the name */
                if (TYPE(CHILD(n, i)) == COMMA)
                    i += 1; /* the comma, if present */
                break;
            case TYPE_COMMENT:
                assert(i);

                if (kwarg)
                    arg = kwarg;

                /* arg will be equal to the last argument processed */
                arg->type_comment = NEW_TYPE_COMMENT(ch);
                i += 1;
                break;
            default:
                PyErr_Format(PyExc_SystemError,
                             "unexpected node in varargslist: %d @ %d",
                             TYPE(ch), i);
                return NULL;
        }
    }
    return arguments(posargs, vararg, kwonlyargs, kwdefaults, kwarg, posdefaults, c->c_arena);
}

Project: typed_ast

If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int
   col_offset, PyArena *arena)
{
    stmt_ty p;
    if (!test) {
        PyErr_SetString(PyExc_ValueError,
                        "field test is required for If");
        return NULL;
    }
    p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = If_kind;
    p->v.If.test = test;
    p->v.If.body = body;
    p->v.If.orelse = orelse;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast

IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, int col_offset,
      PyArena *arena)
{
    expr_ty p;
    if (!test) {
        PyErr_SetString(PyExc_ValueError,
                        "field test is required for IfExp");
        return NULL;
    }
    if (!body) {
        PyErr_SetString(PyExc_ValueError,
                        "field body is required for IfExp");
        return NULL;
    }
    if (!orelse) {
        PyErr_SetString(PyExc_ValueError,
                        "field orelse is required for IfExp");
        return NULL;
    }
    p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = IfExp_kind;
    p->v.IfExp.test = test;
    p->v.IfExp.body = body;
    p->v.IfExp.orelse = orelse;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast
File: Python-ast.c

FunctionType(asdl_seq * argtypes, expr_ty returns, PyArena *arena)
{
    mod_ty p;
    if (!returns) {
        PyErr_SetString(PyExc_ValueError,
                        "field returns is required for FunctionType");
        return NULL;
    }
    p = (mod_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = FunctionType_kind;
    p->v.FunctionType.argtypes = argtypes;
    p->v.FunctionType.returns = returns;
    return p;
}

Project: typed_ast
File: Python-ast.c

ast_dealloc(AST_object *self)
{
    /* bpo-31095: UnTrack is needed before calling any callbacks */
    PyObject_GC_UnTrack(self);
    Py_CLEAR(self->dict);
    Py_TYPE(self)->tp_free(self);
}

Project: typed_ast
File: ast.c

ast_for_async_stmt(struct compiling *c, const node *n)
{
    /* async_stmt: 'async' (funcdef | with_stmt | for_stmt) */
    REQ(n, async_stmt);
    REQ(CHILD(n, 0), NAME);
    assert(strcmp(STR(CHILD(n, 0)), "async") == 0);

    switch (TYPE(CHILD(n, 1))) {
        case funcdef:
            return ast_for_funcdef_impl(c, n, NULL,
                                        true /* is_async */);
        case with_stmt:
            return ast_for_with_stmt(c, n,
                                     true /* is_async */);

        case for_stmt:
            return ast_for_for_stmt(c, n,
                                    true /* is_async */);

        default:
            PyErr_Format(PyExc_SystemError,
                         "invalid async stament: %s",
                         STR(CHILD(n, 1)));
            return NULL;
    }
}

Project: typed_ast
File: Python-ast.c

Slice(expr_ty lower, expr_ty upper, expr_ty step, PyArena *arena)
{
    slice_ty p;
    p = (slice_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = Slice_kind;
    p->v.Slice.lower = lower;
    p->v.Slice.upper = upper;
    p->v.Slice.step = step;
    return p;
}

Project: typed_ast

FunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq *
            decorator_list, expr_ty returns, string type_comment, int lineno,
            int col_offset, PyArena *arena)
{
    stmt_ty p;
    if (!name) {
        PyErr_SetString(PyExc_ValueError,
                        "field name is required for FunctionDef");
        return NULL;
    }
    if (!args) {
        PyErr_SetString(PyExc_ValueError,
                        "field args is required for FunctionDef");
        return NULL;
    }
    p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = FunctionDef_kind;
    p->v.FunctionDef.name = name;
    p->v.FunctionDef.args = args;
    p->v.FunctionDef.body = body;
    p->v.FunctionDef.decorator_list = decorator_list;
    p->v.FunctionDef.returns = returns;
    p->v.FunctionDef.type_comment = type_comment;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast

ast_for_power(struct compiling *c, const node *n)
{
    /* power: atom trailer* ('**' factor)*
     */
    int i;
    expr_ty e, tmp;
    REQ(n, power);
    e = ast_for_atom(c, CHILD(n, 0));
    if (!e)
        return NULL;
    if (NCH(n) == 1)
        return e;
    for (i = 1; i < NCH(n); i++) {
        node *ch = CHILD(n, i);
        if (TYPE(ch) != trailer)
            break;
        tmp = ast_for_trailer(c, ch, e);
        if (!tmp)
            return NULL;
        tmp->lineno = e->lineno;
        tmp->col_offset = e->col_offset;
        e = tmp;
    }
    if (TYPE(CHILD(n, NCH(n) - 1)) == factor) {
        expr_ty f = ast_for_expr(c, CHILD(n, NCH(n) - 1));
        if (!f)
            return NULL;
        tmp = BinOp(e, Pow, f, LINENO(n), n->n_col_offset, c->c_arena);
        if (!tmp)
            return NULL;
        e = tmp;
    }
    return e;
}

Project: typed_ast
File: Python-ast.c

comprehension(expr_ty target, expr_ty iter, asdl_seq * ifs, int is_async,
              PyArena *arena)
{
    comprehension_ty p;
    if (!target) {
        PyErr_SetString(PyExc_ValueError,
                        "field target is required for comprehension");
        return NULL;
    }
    if (!iter) {
        PyErr_SetString(PyExc_ValueError,
                        "field iter is required for comprehension");
        return NULL;
    }
    p = (comprehension_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->target = target;
    p->iter = iter;
    p->ifs = ifs;
    p->is_async = is_async;
    return p;
}

Project: typed_ast
File: Python-ast.c

ast2obj_alias(void* _o)
{
    alias_ty o = (alias_ty)_o;
    PyObject *result = NULL, *value = NULL;
    if (!o) {
        Py_RETURN_NONE;
    }

    result = PyType_GenericNew(alias_type, NULL, NULL);
    if (!result) return NULL;
    value = ast2obj_identifier(o->name);
    if (!value) goto failed;
    if (_PyObject_SetAttrId(result, &PyId_name, value) == -1)
        goto failed;
    Py_DECREF(value);
    value = ast2obj_identifier(o->asname);
    if (!value) goto failed;
    if (_PyObject_SetAttrId(result, &PyId_asname, value) == -1)
        goto failed;
    Py_DECREF(value);
    return result;
failed:
    Py_XDECREF(value);
    Py_XDECREF(result);
    return NULL;
}

Project: typed_ast

ast_for_factor(struct compiling *c, const node *n)
{
    node *pfactor, *ppower, *patom, *pnum;
    expr_ty expression;

    /* If the unary - operator is applied to a constant, don't generate
       a UNARY_NEGATIVE opcode.  Just store the approriate value as a
       constant.  The peephole optimizer already does something like
       this but it doesn't handle the case where the constant is
       (sys.maxint - 1).  In that case, we want a PyIntObject, not a
       PyLongObject.
    */
    if (TYPE(CHILD(n, 0)) == MINUS &&
        NCH(n) == 2 &&
        TYPE((pfactor = CHILD(n, 1))) == factor &&
        NCH(pfactor) == 1 &&
        TYPE((ppower = CHILD(pfactor, 0))) == power &&
        NCH(ppower) == 1 &&
        TYPE((patom = CHILD(ppower, 0))) == atom &&
        TYPE((pnum = CHILD(patom, 0))) == NUMBER) {
        PyObject *pynum;
        char *s = PyObject_MALLOC(strlen(STR(pnum)) + 2);
        if (s == NULL)
            return NULL;
        s[0] = '-';
        strcpy(s + 1, STR(pnum));
        pynum = parsenumber(c, s);
        PyObject_FREE(s);
        if (!pynum)
            return NULL;

        PyArena_AddPyObject(c->c_arena, pynum);
        return Num(pynum, LINENO(n), n->n_col_offset, c->c_arena);
    }

    expression = ast_for_expr(c, CHILD(n, 1));
    if (!expression)
        return NULL;

    switch (TYPE(CHILD(n, 0))) {
        case PLUS:
            return UnaryOp(UAdd, expression, LINENO(n), n->n_col_offset,
                           c->c_arena);
        case MINUS:
            return UnaryOp(USub, expression, LINENO(n), n->n_col_offset,
                           c->c_arena);
        case TILDE:
            return UnaryOp(Invert, expression, LINENO(n),
                           n->n_col_offset, c->c_arena);
    }
    PyErr_Format(PyExc_SystemError, "unhandled factor: %d",
                 TYPE(CHILD(n, 0)));
    return NULL;
}

Project: typed_ast

ast_for_binop(struct compiling *c, const node *n)
{
        /* Must account for a sequence of expressions.
           How should A op B op C by represented?
           BinOp(BinOp(A, op, B), op, C).
        */

        int i, nops;
        expr_ty expr1, expr2, result;
        operator_ty newoperator;

        expr1 = ast_for_expr(c, CHILD(n, 0));
        if (!expr1)
            return NULL;

        expr2 = ast_for_expr(c, CHILD(n, 2));
        if (!expr2)
            return NULL;

        newoperator = get_operator(CHILD(n, 1));
        if (!newoperator)
            return NULL;

        result = BinOp(expr1, newoperator, expr2, LINENO(n), n->n_col_offset,
                       c->c_arena);
        if (!result)
            return NULL;

        nops = (NCH(n) - 1) / 2;
        for (i = 1; i < nops; i++) {
                expr_ty tmp_result, tmp;
                const node* next_oper = CHILD(n, i * 2 + 1);

                newoperator = get_operator(next_oper);
                if (!newoperator)
                    return NULL;

                tmp = ast_for_expr(c, CHILD(n, i * 2 + 2));
                if (!tmp)
                    return NULL;

                tmp_result = BinOp(result, newoperator, tmp,
                                   LINENO(next_oper), next_oper->n_col_offset,
                                   c->c_arena);
                if (!tmp_result)
                        return NULL;
                result = tmp_result;
        }
        return result;
}

Project: typed_ast

ast_for_exprlist(struct compiling *c, const node *n, expr_context_ty context)
{
    asdl_seq *seq;
    int i;
    expr_ty e;

    REQ(n, exprlist);

    seq = asdl_seq_new((NCH(n) + 1) / 2, c->c_arena);
    if (!seq)
        return NULL;
    for (i = 0; i < NCH(n); i += 2) {
        e = ast_for_expr(c, CHILD(n, i));
        if (!e)
            return NULL;
        asdl_seq_SET(seq, i / 2, e);
        if (context && !set_context(c, e, context, CHILD(n, i)))
            return NULL;
    }
    return seq;
}

Project: typed_ast

NameConstant(singleton value, int lineno, int col_offset, PyArena *arena)
{
    expr_ty p;
    if (!value) {
        PyErr_SetString(PyExc_ValueError,
                        "field value is required for NameConstant");
        return NULL;
    }
    p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = NameConstant_kind;
    p->v.NameConstant.value = value;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast

Yield(expr_ty value, int lineno, int col_offset, PyArena *arena)
{
    expr_ty p;
    p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = Yield_kind;
    p->v.Yield.value = value;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast

Lambda(arguments_ty args, expr_ty body, int lineno, int col_offset, PyArena
       *arena)
{
    expr_ty p;
    if (!args) {
        PyErr_SetString(PyExc_ValueError,
                        "field args is required for Lambda");
        return NULL;
    }
    if (!body) {
        PyErr_SetString(PyExc_ValueError,
                        "field body is required for Lambda");
        return NULL;
    }
    p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = Lambda_kind;
    p->v.Lambda.args = args;
    p->v.Lambda.body = body;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast

ast2obj_arg(void* _o)
{
    arg_ty o = (arg_ty)_o;
    PyObject *result = NULL, *value = NULL;
    if (!o) {
        Py_RETURN_NONE;
    }

    result = PyType_GenericNew(arg_type, NULL, NULL);
    if (!result) return NULL;
    value = ast2obj_identifier(o->arg);
    if (!value) goto failed;
    if (_PyObject_SetAttrId(result, &PyId_arg, value) == -1)
        goto failed;
    Py_DECREF(value);
    value = ast2obj_expr(o->annotation);
    if (!value) goto failed;
    if (_PyObject_SetAttrId(result, &PyId_annotation, value) == -1)
        goto failed;
    Py_DECREF(value);
    value = ast2obj_string(o->type_comment);
    if (!value) goto failed;
    if (_PyObject_SetAttrId(result, &PyId_type_comment, value) == -1)
        goto failed;
    Py_DECREF(value);
    value = ast2obj_int(o->lineno);
    if (!value) goto failed;
    if (_PyObject_SetAttrId(result, &PyId_lineno, value) < 0)
        goto failed;
    Py_DECREF(value);
    value = ast2obj_int(o->col_offset);
    if (!value) goto failed;
    if (_PyObject_SetAttrId(result, &PyId_col_offset, value) < 0)
        goto failed;
    Py_DECREF(value);
    return result;
failed:
    Py_XDECREF(value);
    Py_XDECREF(result);
    return NULL;
}

Project: typed_ast

new_type_comment(const char *s, struct compiling *c)
{
  return PyUnicode_DecodeUTF8(s, strlen(s), NULL);
}

Project: typed_ast

compiler_complex_args(struct compiling *c, const node *n)
{
    int i, len = (NCH(n) + 1) / 2;
    expr_ty result;
    asdl_seq *args = asdl_seq_new(len, c->c_arena);
    if (!args)
        return NULL;

    /* fpdef: NAME | '(' fplist ')'
       fplist: fpdef (',' fpdef)* [',']
    */
    REQ(n, fplist);
    for (i = 0; i < len; i++) {
        PyObject *arg_id;
        const node *fpdef_node = CHILD(n, 2*i);
        const node *child;
        expr_ty arg;
set_name:
        /* fpdef_node is either a NAME or an fplist */
        child = CHILD(fpdef_node, 0);
        if (TYPE(child) == NAME) {
            if (!forbidden_check(c, n, STR(child)))
                return NULL;
            arg_id = NEW_IDENTIFIER(child);
            if (!arg_id)
                return NULL;
            arg = Name(arg_id, Store, LINENO(child), child->n_col_offset,
                       c->c_arena);
        }
        else {
            assert(TYPE(fpdef_node) == fpdef);
            /* fpdef_node[0] is not a name, so it must be '(', get CHILD[1] */
            child = CHILD(fpdef_node, 1);
            assert(TYPE(child) == fplist);
            /* NCH == 1 means we have (x), we need to elide the extra parens */
            if (NCH(child) == 1) {
                fpdef_node = CHILD(child, 0);
                assert(TYPE(fpdef_node) == fpdef);
                goto set_name;
            }
            arg = compiler_complex_args(c, child);
        }
        asdl_seq_SET(args, i, arg);
    }

    result = Tuple(args, Store, LINENO(n), n->n_col_offset, c->c_arena);
    if (!set_context(c, result, Store, n))
        return NULL;
    return result;
}

Project: typed_ast

Raise(expr_ty exc, expr_ty cause, int lineno, int col_offset, PyArena *arena)
{
    stmt_ty p;
    p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = Raise_kind;
    p->v.Raise.exc = exc;
    p->v.Raise.cause = cause;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast
File: Python-ast.c

static int obj2ast_string(PyObject* obj, PyObject** out, PyArena* arena)
{
    if (!PyUnicode_CheckExact(obj) && !PyBytes_CheckExact(obj)) {
        PyErr_SetString(PyExc_TypeError, "AST string must be of type str");
        return 1;
    }
    return obj2ast_object(obj, out, arena);
}

Project: typed_ast
File: tokenizer.c

translate_into_utf8(const char* str, const char* enc) {
    PyObject *utf8;
    PyObject* buf = PyUnicode_Decode(str, strlen(str), enc, NULL);
    if (buf == NULL)
        return NULL;
    utf8 = PyUnicode_AsUTF8String(buf);
    Py_DECREF(buf);
    return utf8;
}

Project: typed_ast

count_list_fors(struct compiling *c, const node *n)
{
    int n_fors = 0;
    node *ch = CHILD(n, 1);

 count_list_for:
    n_fors++;
    REQ(ch, list_for);
    if (NCH(ch) == 5)
        ch = CHILD(ch, 4);
    else
        return n_fors;
 count_list_iter:
    REQ(ch, list_iter);
    ch = CHILD(ch, 0);
    if (TYPE(ch) == list_for)
        goto count_list_for;
    else if (TYPE(ch) == list_if) {
        if (NCH(ch) == 3) {
            ch = CHILD(ch, 2);
            goto count_list_iter;
        }
        else
            return n_fors;
    }

    /* Should never be reached */
    PyErr_SetString(PyExc_SystemError, "logic error in count_list_fors");
    return -1;
}

Project: typed_ast
File: tokenizer.c

decoding_fgets(char *s, int size, struct tok_state *tok)
{
    return fgets(s, size, tok->fp);
}

Project: typed_ast
File: Python-ast.c

PyObject* ast2obj_unaryop(unaryop_ty o)
{
    switch(o) {
        case Invert:
            Py_INCREF(Invert_singleton);
            return Invert_singleton;
        case Not:
            Py_INCREF(Not_singleton);
            return Not_singleton;
        case UAdd:
            Py_INCREF(UAdd_singleton);
            return UAdd_singleton;
        case USub:
            Py_INCREF(USub_singleton);
            return USub_singleton;
        default:
            /* should never happen, but just in case ... */
            PyErr_Format(PyExc_SystemError, "unknown unaryop found");
            return NULL;
    }
}

Project: typed_ast

Ta3Tokenizer_Free(struct tok_state *tok)
{
    if (tok->encoding != NULL)
        PyMem_FREE(tok->encoding);
#ifndef PGEN
    Py_XDECREF(tok->decoding_readline);
    Py_XDECREF(tok->decoding_buffer);
    Py_XDECREF(tok->filename);
#endif
    if (tok->fp != NULL && tok->buf != NULL)
        PyMem_FREE(tok->buf);
    if (tok->input)
        PyMem_FREE((char *)tok->input);
    PyMem_FREE(tok);
}

Project: typed_ast

verify_identifier(struct tok_state *tok)
{
    PyObject *s;
    int result;
    if (tok->decoding_erred)
        return 0;
    s = PyUnicode_DecodeUTF8(tok->start, tok->cur - tok->start, NULL);
    if (s == NULL || PyUnicode_READY(s) == -1) {
        if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
            PyErr_Clear();
            tok->done = E_IDENTIFIER;
        } else {
            tok->done = E_ERROR;
        }
        return 0;
    }
    result = PyUnicode_IsIdentifier(s);
    Py_DECREF(s);
    if (result == 0)
        tok->done = E_IDENTIFIER;
    return result;
}

Project: typed_ast

Compare(expr_ty left, asdl_int_seq * ops, asdl_seq * comparators, int lineno,
        int col_offset, PyArena *arena)
{
    expr_ty p;
    if (!left) {
        PyErr_SetString(PyExc_ValueError,
                        "field left is required for Compare");
        return NULL;
    }
    p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = Compare_kind;
    p->v.Compare.left = left;
    p->v.Compare.ops = ops;
    p->v.Compare.comparators = comparators;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast

Assign(asdl_seq * targets, expr_ty value, string type_comment, int lineno, int
       col_offset, PyArena *arena)
{
    stmt_ty p;
    if (!value) {
        PyErr_SetString(PyExc_ValueError,
                        "field value is required for Assign");
        return NULL;
    }
    p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = Assign_kind;
    p->v.Assign.targets = targets;
    p->v.Assign.value = value;
    p->v.Assign.type_comment = type_comment;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast

make_str_node_and_del(PyObject **str, struct compiling *c, const node* n)
{
    PyObject *kind, *s = *str;
    const char *raw = STR(CHILD(n, 0));
    /* currently Python allows up to 2 string modifiers */
    char *ch, s_kind[3] = {0, 0, 0};
    ch = s_kind;
    while (*raw && *raw != '\'' && *raw != '"') {
        *ch++ = *raw++;
    }
    kind = PyUnicode_FromString(s_kind);
    if (!kind) {
        return NULL;
    }
    *str = NULL;
    assert(PyUnicode_CheckExact(s));
    if (PyArena_AddPyObject(c->c_arena, s) < 0) {
        Py_DECREF(s);
        return NULL;
    }
    return Str(s, kind, LINENO(n), n->n_col_offset, c->c_arena);
}

Project: typed_ast

ast_for_if_stmt(struct compiling *c, const node *n)
{
    /* if_stmt: 'if' test ':' suite ('elif' test ':' suite)*
       ['else' ':' suite]
    */
    char *s;

    REQ(n, if_stmt);

    if (NCH(n) == 4) {
        expr_ty expression;
        asdl_seq *suite_seq;

        expression = ast_for_expr(c, CHILD(n, 1));
        if (!expression)
            return NULL;
        suite_seq = ast_for_suite(c, CHILD(n, 3));
        if (!suite_seq)
            return NULL;

        return If(expression, suite_seq, NULL, LINENO(n), n->n_col_offset,
                  c->c_arena);
    }

    s = STR(CHILD(n, 4));
    /* s[2], the third character in the string, will be
       's' for el_s_e, or
       'i' for el_i_f
    */
    if (s[2] == 's') {
        expr_ty expression;
        asdl_seq *seq1, *seq2;

        expression = ast_for_expr(c, CHILD(n, 1));
        if (!expression)
            return NULL;
        seq1 = ast_for_suite(c, CHILD(n, 3));
        if (!seq1)
            return NULL;
        seq2 = ast_for_suite(c, CHILD(n, 6));
        if (!seq2)
            return NULL;

        return If(expression, seq1, seq2, LINENO(n), n->n_col_offset,
                  c->c_arena);
    }
    else if (s[2] == 'i') {
        int i, n_elif, has_else = 0;
        expr_ty expression;
        asdl_seq *suite_seq;
        asdl_seq *orelse = NULL;
        n_elif = NCH(n) - 4;
        /* must reference the child n_elif+1 since 'else' token is third,
           not fourth, child from the end. */
        if (TYPE(CHILD(n, (n_elif + 1))) == NAME
            && STR(CHILD(n, (n_elif + 1)))[2] == 's') {
            has_else = 1;
            n_elif -= 3;
        }
        n_elif /= 4;

        if (has_else) {
            asdl_seq *suite_seq2;

            orelse = asdl_seq_new(1, c->c_arena);
            if (!orelse)
                return NULL;
            expression = ast_for_expr(c, CHILD(n, NCH(n) - 6));
            if (!expression)
                return NULL;
            suite_seq = ast_for_suite(c, CHILD(n, NCH(n) - 4));
            if (!suite_seq)
                return NULL;
            suite_seq2 = ast_for_suite(c, CHILD(n, NCH(n) - 1));
            if (!suite_seq2)
                return NULL;

            asdl_seq_SET(orelse, 0,
                         If(expression, suite_seq, suite_seq2,
                            LINENO(CHILD(n, NCH(n) - 6)),
                            CHILD(n, NCH(n) - 6)->n_col_offset,
                            c->c_arena));
            /* the just-created orelse handled the last elif */
            n_elif--;
        }

        for (i = 0; i < n_elif; i++) {
            int off = 5 + (n_elif - i - 1) * 4;
            asdl_seq *newobj = asdl_seq_new(1, c->c_arena);
            if (!newobj)
                return NULL;
            expression = ast_for_expr(c, CHILD(n, off));
            if (!expression)
                return NULL;
            suite_seq = ast_for_suite(c, CHILD(n, off + 2));
            if (!suite_seq)
                return NULL;

            asdl_seq_SET(newobj, 0,
                         If(expression, suite_seq, orelse,
                            LINENO(CHILD(n, off)),
                            CHILD(n, off)->n_col_offset, c->c_arena));
            orelse = newobj;
        }
        expression = ast_for_expr(c, CHILD(n, 1));
        if (!expression)
            return NULL;
        suite_seq = ast_for_suite(c, CHILD(n, 3));
        if (!suite_seq)
            return NULL;
        return If(expression, suite_seq, orelse,
                  LINENO(n), n->n_col_offset, c->c_arena);
    }

    PyErr_Format(PyExc_SystemError,
                 "unexpected token in 'if' statement: %s", s);
    return NULL;
}

Project: typed_ast

List(asdl_seq * elts, expr_context_ty ctx, int lineno, int col_offset, PyArena
     *arena)
{
    expr_ty p;
    if (!ctx) {
        PyErr_SetString(PyExc_ValueError,
                        "field ctx is required for List");
        return NULL;
    }
    p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = List_kind;
    p->v.List.elts = elts;
    p->v.List.ctx = ctx;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast

Str(string s, string kind, int lineno, int col_offset, PyArena *arena)
{
    expr_ty p;
    if (!s) {
        PyErr_SetString(PyExc_ValueError,
                        "field s is required for Str");
        return NULL;
    }
    if (!kind) {
        PyErr_SetString(PyExc_ValueError,
                        "field kind is required for Str");
        return NULL;
    }
    p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = Str_kind;
    p->v.Str.s = s;
    p->v.Str.kind = kind;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast

ast_for_comprehension(struct compiling *c, const node *n)
{
    int i, n_fors;
    asdl_seq *comps;

    n_fors = count_comp_fors(c, n);
    if (n_fors == -1)
        return NULL;

    comps = _Ta3_asdl_seq_new(n_fors, c->c_arena);
    if (!comps)
        return NULL;

    for (i = 0; i < n_fors; i++) {
        comprehension_ty comp;
        asdl_seq *t;
        expr_ty expression, first;
        node *for_ch;
        node *sync_n;
        int is_async = 0;

        REQ(n, comp_for);

        if (NCH(n) == 2) {
            is_async = 1;
            REQ(CHILD(n, 0), NAME);
            assert(strcmp(STR(CHILD(n, 0)), "async") == 0);
            sync_n = CHILD(n, 1);
        }
        else {
            sync_n = CHILD(n, 0);
        }
        REQ(sync_n, sync_comp_for);

        /* Async comprehensions only allowed in Python 3.6 and greater */
        if (is_async && c->c_feature_version < 6) {
            ast_error(c, n,
                    "Async comprehensions are only supported in Python 3.6 and greater");
            return NULL;
        }

        for_ch = CHILD(sync_n, 1);
        t = ast_for_exprlist(c, for_ch, Store);
        if (!t)
            return NULL;
        expression = ast_for_expr(c, CHILD(sync_n, 3));
        if (!expression)
            return NULL;

        /* Check the # of children rather than the length of t, since
           (x for x, in ...) has 1 element in t, but still requires a Tuple. */
        first = (expr_ty)asdl_seq_GET(t, 0);
        if (NCH(for_ch) == 1)
            comp = comprehension(first, expression, NULL,
                                 is_async, c->c_arena);
        else
            comp = comprehension(Tuple(t, Store, first->lineno,
                                       first->col_offset, c->c_arena),
                                 expression, NULL, is_async, c->c_arena);
        if (!comp)
            return NULL;

        if (NCH(sync_n) == 5) {
            int j, n_ifs;
            asdl_seq *ifs;

            n = CHILD(sync_n, 4);
            n_ifs = count_comp_ifs(c, n);
            if (n_ifs == -1)
                return NULL;

            ifs = _Ta3_asdl_seq_new(n_ifs, c->c_arena);
            if (!ifs)
                return NULL;

            for (j = 0; j < n_ifs; j++) {
                REQ(n, comp_iter);
                n = CHILD(n, 0);
                REQ(n, comp_if);

                expression = ast_for_expr(c, CHILD(n, 1));
                if (!expression)
                    return NULL;
                asdl_seq_SET(ifs, j, expression);
                if (NCH(n) == 3)
                    n = CHILD(n, 2);
            }
            /* on exit, must guarantee that n is a comp_for */
            if (TYPE(n) == comp_iter)
                n = CHILD(n, 0);
            comp->ifs = ifs;
        }
        asdl_seq_SET(comps, i, comp);
    }
    return comps;
}

Project: typed_ast

ast_for_testlist(struct compiling *c, const node* n)
{
    /* testlist_comp: test (',' test)* [','] */
    /* testlist: test (',' test)* [','] */
    /* testlist_safe: test (',' test)+ [','] */
    /* testlist1: test (',' test)* */
    assert(NCH(n) > 0);
    if (TYPE(n) == testlist_comp) {
        if (NCH(n) > 1)
            assert(TYPE(CHILD(n, 1)) != comp_for);
    }
    else {
        assert(TYPE(n) == testlist ||
               TYPE(n) == testlist_safe ||
               TYPE(n) == testlist1);
    }
    if (NCH(n) == 1)
        return ast_for_expr(c, CHILD(n, 0));
    else {
        asdl_seq *tmp = seq_for_testlist(c, n);
        if (!tmp)
            return NULL;
        return Tuple(tmp, Load, LINENO(n), n->n_col_offset, c->c_arena);
    }
}

Project: typed_ast

Nonlocal(asdl_seq * names, int lineno, int col_offset, PyArena *arena)
{
    stmt_ty p;
    p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = Nonlocal_kind;
    p->v.Nonlocal.names = names;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast

Constant(constant value, int lineno, int col_offset, PyArena *arena)
{
    expr_ty p;
    if (!value) {
        PyErr_SetString(PyExc_ValueError,
                        "field value is required for Constant");
        return NULL;
    }
    p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = Constant_kind;
    p->v.Constant.value = value;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast

ast_for_flow_stmt(struct compiling *c, const node *n)
{
    /*
      flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt
                 | yield_stmt
      break_stmt: 'break'
      continue_stmt: 'continue'
      return_stmt: 'return' [testlist]
      yield_stmt: yield_expr
      yield_expr: 'yield' testlist
      raise_stmt: 'raise' [test [',' test [',' test]]]
    */
    node *ch;

    REQ(n, flow_stmt);
    ch = CHILD(n, 0);
    switch (TYPE(ch)) {
        case break_stmt:
            return Break(LINENO(n), n->n_col_offset, c->c_arena);
        case continue_stmt:
            return Continue(LINENO(n), n->n_col_offset, c->c_arena);
        case yield_stmt: { /* will reduce to yield_expr */
            expr_ty exp = ast_for_expr(c, CHILD(ch, 0));
            if (!exp)
                return NULL;
            return Expr(exp, LINENO(n), n->n_col_offset, c->c_arena);
        }
        case return_stmt:
            if (NCH(ch) == 1)
                return Return(NULL, LINENO(n), n->n_col_offset, c->c_arena);
            else {
                expr_ty expression = ast_for_testlist(c, CHILD(ch, 1));
                if (!expression)
                    return NULL;
                return Return(expression, LINENO(n), n->n_col_offset,
                              c->c_arena);
            }
        case raise_stmt:
            if (NCH(ch) == 1)
                return Raise(NULL, NULL, NULL, LINENO(n), n->n_col_offset,
                             c->c_arena);
            else if (NCH(ch) == 2) {
                expr_ty expression = ast_for_expr(c, CHILD(ch, 1));
                if (!expression)
                    return NULL;
                return Raise(expression, NULL, NULL, LINENO(n),
                             n->n_col_offset, c->c_arena);
            }
            else if (NCH(ch) == 4) {
                expr_ty expr1, expr2;

                expr1 = ast_for_expr(c, CHILD(ch, 1));
                if (!expr1)
                    return NULL;
                expr2 = ast_for_expr(c, CHILD(ch, 3));
                if (!expr2)
                    return NULL;

                return Raise(expr1, expr2, NULL, LINENO(n), n->n_col_offset,
                             c->c_arena);
            }
            else if (NCH(ch) == 6) {
                expr_ty expr1, expr2, expr3;

                expr1 = ast_for_expr(c, CHILD(ch, 1));
                if (!expr1)
                    return NULL;
                expr2 = ast_for_expr(c, CHILD(ch, 3));
                if (!expr2)
                    return NULL;
                expr3 = ast_for_expr(c, CHILD(ch, 5));
                if (!expr3)
                    return NULL;

                return Raise(expr1, expr2, expr3, LINENO(n), n->n_col_offset,
                             c->c_arena);
            }
        default:
            PyErr_Format(PyExc_SystemError,
                         "unexpected flow_stmt: %d", TYPE(ch));
            return NULL;
    }
}

Project: typed_ast

mod_ty Ta3AST_obj2mod(PyObject* ast, PyArena* arena, int mode)
{
    mod_ty res;
    PyObject *req_type[3];
    char *req_name[] = {"Module", "Expression", "Interactive"};
    int isinstance;

    req_type[0] = (PyObject*)Module_type;
    req_type[1] = (PyObject*)Expression_type;
    req_type[2] = (PyObject*)Interactive_type;

    assert(0 <= mode && mode <= 2);

    if (!init_types())
        return NULL;

    isinstance = PyObject_IsInstance(ast, req_type[mode]);
    if (isinstance == -1)
        return NULL;
    if (!isinstance) {
        PyErr_Format(PyExc_TypeError, "expected %s node, got %.400s",
                     req_name[mode], Py_TYPE(ast)->tp_name);
        return NULL;
    }
    if (obj2ast_mod(ast, &res, arena) != 0)
        return NULL;
    else
        return res;
}

Project: typed_ast

AsyncFunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq
                 * decorator_list, expr_ty returns, string type_comment, int
                 lineno, int col_offset, PyArena *arena)
{
    stmt_ty p;
    if (!name) {
        PyErr_SetString(PyExc_ValueError,
                        "field name is required for AsyncFunctionDef");
        return NULL;
    }
    if (!args) {
        PyErr_SetString(PyExc_ValueError,
                        "field args is required for AsyncFunctionDef");
        return NULL;
    }
    p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = AsyncFunctionDef_kind;
    p->v.AsyncFunctionDef.name = name;
    p->v.AsyncFunctionDef.args = args;
    p->v.AsyncFunctionDef.body = body;
    p->v.AsyncFunctionDef.decorator_list = decorator_list;
    p->v.AsyncFunctionDef.returns = returns;
    p->v.AsyncFunctionDef.type_comment = type_comment;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast

ast_for_with_item(struct compiling *c, const node *n, asdl_seq *content, string type_comment)
{
    expr_ty context_expr, optional_vars = NULL;

    REQ(n, with_item);
    context_expr = ast_for_expr(c, CHILD(n, 0));
    if (!context_expr)
        return NULL;
    if (NCH(n) == 3) {
        optional_vars = ast_for_expr(c, CHILD(n, 2));

        if (!optional_vars) {
            return NULL;
        }
        if (!set_context(c, optional_vars, Store, n)) {
            return NULL;
        }
    }

    return With(context_expr, optional_vars, content, type_comment, LINENO(n),
                n->n_col_offset, c->c_arena);
}

Project: typed_ast
File: Python-ast.c

alias(identifier name, identifier asname, PyArena *arena)
{
    alias_ty p;
    if (!name) {
        PyErr_SetString(PyExc_ValueError,
                        "field name is required for alias");
        return NULL;
    }
    p = (alias_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->name = name;
    p->asname = asname;
    return p;
}

Project: typed_ast

With(asdl_seq * items, asdl_seq * body, string type_comment, int lineno, int
     col_offset, PyArena *arena)
{
    stmt_ty p;
    p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = With_kind;
    p->v.With.items = items;
    p->v.With.body = body;
    p->v.With.type_comment = type_comment;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast

static int obj2ast_bytes(PyObject* obj, PyObject** out, PyArena* arena)
{
    if (!PyBytes_CheckExact(obj)) {
        PyErr_SetString(PyExc_TypeError, "AST bytes must be of type bytes");
        return 1;
    }
    return obj2ast_object(obj, out, arena);
}

Project: typed_ast

ast_for_for_stmt(struct compiling *c, const node *n)
{
    asdl_seq *_target, *seq = NULL, *suite_seq;
    expr_ty expression;
    expr_ty target, first;
    const node *node_target;
    int has_type_comment;
    string type_comment;
    /* for_stmt: 'for' exprlist 'in' testlist ':' [TYPE_COMMENT] suite ['else' ':' suite] */
    REQ(n, for_stmt);

    has_type_comment = TYPE(CHILD(n, 5)) == TYPE_COMMENT;

    if (NCH(n) == 9 + has_type_comment) {
        seq = ast_for_suite(c, CHILD(n, 8 + has_type_comment));
        if (!seq)
            return NULL;
    }

    node_target = CHILD(n, 1);
    _target = ast_for_exprlist(c, node_target, Store);
    if (!_target)
        return NULL;
    /* Check the # of children rather than the length of _target, since
       for x, in ... has 1 element in _target, but still requires a Tuple. */
    first = (expr_ty)asdl_seq_GET(_target, 0);
    if (NCH(node_target) == 1)
        target = first;
    else
        target = Tuple(_target, Store, first->lineno, first->col_offset, c->c_arena);

    expression = ast_for_testlist(c, CHILD(n, 3));
    if (!expression)
        return NULL;
    suite_seq = ast_for_suite(c, CHILD(n, 5 + has_type_comment));
    if (!suite_seq)
        return NULL;

    if (has_type_comment)
        type_comment = NEW_TYPE_COMMENT(CHILD(n, 5));
    else
        type_comment = NULL;

    return For(target, expression, suite_seq, seq, type_comment, LINENO(n), n->n_col_offset,
               c->c_arena);
}

Project: typed_ast

ast_for_listcomp(struct compiling *c, const node *n)
{
    /* listmaker: test ( list_for | (',' test)* [','] )
       list_for: 'for' exprlist 'in' testlist_safe [list_iter]
       list_iter: list_for | list_if
       list_if: 'if' test [list_iter]
       testlist_safe: test [(',' test)+ [',']]
    */
    expr_ty elt, first;
    asdl_seq *listcomps;
    int i, n_fors;
    node *ch;

    REQ(n, listmaker);
    assert(NCH(n) > 1);

    elt = ast_for_expr(c, CHILD(n, 0));
    if (!elt)
        return NULL;

    n_fors = count_list_fors(c, n);
    if (n_fors == -1)
        return NULL;

    listcomps = asdl_seq_new(n_fors, c->c_arena);
    if (!listcomps)
        return NULL;

    ch = CHILD(n, 1);
    for (i = 0; i < n_fors; i++) {
        comprehension_ty lc;
        asdl_seq *t;
        expr_ty expression;
        node *for_ch;

        REQ(ch, list_for);

        for_ch = CHILD(ch, 1);
        t = ast_for_exprlist(c, for_ch, Store);
        if (!t)
            return NULL;
        expression = ast_for_testlist(c, CHILD(ch, 3));
        if (!expression)
            return NULL;

        /* Check the # of children rather than the length of t, since
           [x for x, in ... ] has 1 element in t, but still requires a Tuple.
        */
        first = (expr_ty)asdl_seq_GET(t, 0);
        if (NCH(for_ch) == 1)
            lc = comprehension(first, expression, NULL, c->c_arena);
        else
            lc = comprehension(Tuple(t, Store, first->lineno, first->col_offset,
                                     c->c_arena),
                               expression, NULL, c->c_arena);
        if (!lc)
            return NULL;

        if (NCH(ch) == 5) {
            int j, n_ifs;
            asdl_seq *ifs;
            expr_ty list_for_expr;

            ch = CHILD(ch, 4);
            n_ifs = count_list_ifs(c, ch);
            if (n_ifs == -1)
                return NULL;

            ifs = asdl_seq_new(n_ifs, c->c_arena);
            if (!ifs)
                return NULL;

            for (j = 0; j < n_ifs; j++) {
                REQ(ch, list_iter);
                ch = CHILD(ch, 0);
                REQ(ch, list_if);

                list_for_expr = ast_for_expr(c, CHILD(ch, 1));
                if (!list_for_expr)
                    return NULL;

                asdl_seq_SET(ifs, j, list_for_expr);
                if (NCH(ch) == 3)
                    ch = CHILD(ch, 2);
            }
            /* on exit, must guarantee that ch is a list_for */
            if (TYPE(ch) == list_iter)
                ch = CHILD(ch, 0);
            lc->ifs = ifs;
        }
        asdl_seq_SET(listcomps, i, lc);
    }

    return ListComp(elt, listcomps, LINENO(n), n->n_col_offset, c->c_arena);
}

Project: typed_ast

ast_for_print_stmt(struct compiling *c, const node *n)
{
    /* print_stmt: 'print' ( [ test (',' test)* [','] ]
                             | '>>' test [ (',' test)+ [','] ] )
     */
    expr_ty dest = NULL, expression;
    asdl_seq *seq = NULL;
    bool nl;
    int i, j, values_count, start = 1;

    REQ(n, print_stmt);
    if (NCH(n) >= 2 && TYPE(CHILD(n, 1)) == RIGHTSHIFT) {
        dest = ast_for_expr(c, CHILD(n, 2));
        if (!dest)
            return NULL;
            start = 4;
    }
    values_count = (NCH(n) + 1 - start) / 2;
    if (values_count) {
        seq = asdl_seq_new(values_count, c->c_arena);
        if (!seq)
            return NULL;
        for (i = start, j = 0; i < NCH(n); i += 2, ++j) {
            expression = ast_for_expr(c, CHILD(n, i));
            if (!expression)
                return NULL;
            asdl_seq_SET(seq, j, expression);
        }
    }
    nl = (TYPE(CHILD(n, NCH(n) - 1)) == COMMA) ? false : true;
    return Print(dest, seq, nl, LINENO(n), n->n_col_offset, c->c_arena);
}

Project: typed_ast

parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
         int *flags)
{
    parser_state *ps;
    node *n;
    int started = 0;

    growable_int_array type_ignores;
    if (!growable_int_array_init(&type_ignores, 10)) {
        err_ret->error = E_NOMEM;
        Ta3Tokenizer_Free(tok);
        return NULL;
    }

    if ((ps = Ta3Parser_New(g, start)) == NULL) {
        err_ret->error = E_NOMEM;
        Ta3Tokenizer_Free(tok);
        return NULL;
    }
#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
    if (*flags & PyPARSE_BARRY_AS_BDFL)
        ps->p_flags |= CO_FUTURE_BARRY_AS_BDFL;
#endif

    for (;;) {
        char *a, *b;
        int type;
        size_t len;
        char *str;
        int col_offset;

        type = Ta3Tokenizer_Get(tok, &a, &b);
        if (type == ERRORTOKEN) {
            err_ret->error = tok->done;
            break;
        }
        if (type == ENDMARKER && started) {
            type = NEWLINE; /* Add an extra newline */
            started = 0;
            /* Add the right number of dedent tokens,
               except if a certain flag is given --
               codeop.py uses this. */
            if (tok->indent &&
                !(*flags & PyPARSE_DONT_IMPLY_DEDENT))
            {
                tok->pendin = -tok->indent;
                tok->indent = 0;
            }
        }
        else
            started = 1;
        len = (a != NULL && b != NULL) ? b - a : 0;
        str = (char *) PyObject_MALLOC(len + 1);
        if (str == NULL) {
            err_ret->error = E_NOMEM;
            break;
        }
        if (len > 0)
            strncpy(str, a, len);
        str[len] = '\0';

#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
        if (type == NOTEQUAL) {
            if (!(ps->p_flags & CO_FUTURE_BARRY_AS_BDFL) &&
                            strcmp(str, "!=")) {
                PyObject_FREE(str);
                err_ret->error = E_SYNTAX;
                break;
            }
            else if ((ps->p_flags & CO_FUTURE_BARRY_AS_BDFL) &&
                            strcmp(str, "<>")) {
                PyObject_FREE(str);
                err_ret->expected = NOTEQUAL;
                err_ret->error = E_SYNTAX;
                break;
            }
        }
#endif
        if (a != NULL && a >= tok->line_start) {
            col_offset = Py_SAFE_DOWNCAST(a - tok->line_start,
                                          intptr_t, int);
        }
        else {
            col_offset = -1;
        }

        if (type == TYPE_IGNORE) {
            if (!growable_int_array_add(&type_ignores, tok->lineno)) {
                err_ret->error = E_NOMEM;
                break;
            }
            continue;
        }

        if ((err_ret->error =
             Ta3Parser_AddToken(ps, (int)type, str,
                               tok->lineno, col_offset,
                               &(err_ret->expected))) != E_OK) {
            if (err_ret->error != E_DONE) {
                PyObject_FREE(str);
                err_ret->token = type;
            }
            break;
        }
    }

    if (err_ret->error == E_DONE) {
        n = ps->p_tree;
        ps->p_tree = NULL;

        if (n->n_type == file_input) {
            /* Put type_ignore nodes in the ENDMARKER of file_input. */
            int num;
            node *ch;
            size_t i;

            num = NCH(n);
            ch = CHILD(n, num - 1);
            REQ(ch, ENDMARKER);

            for (i = 0; i < type_ignores.num_items; i++) {
                Ta3Node_AddChild(ch, TYPE_IGNORE, NULL, type_ignores.items[i], 0);
            }
        }
        growable_int_array_deallocate(&type_ignores);

#ifndef PGEN
        /* Check that the source for a single input statement really
           is a single statement by looking at what is left in the
           buffer after parsing.  Trailing whitespace and comments
           are OK.  */
        if (start == single_input) {
            char *cur = tok->cur;
            char c = *tok->cur;

            for (;;) {
                while (c == ' ' || c == '\t' || c == '\n' || c == '\014')
                    c = *++cur;

                if (!c)
                    break;

                if (c != '#') {
                    err_ret->error = E_BADSINGLE;
                    Ta3Node_Free(n);
                    n = NULL;
                    break;
                }

                /* Suck up comment. */
                while (c && c != '\n')
                    c = *++cur;
            }
        }
#endif
    }
    else
        n = NULL;

#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
    *flags = ps->p_flags;
#endif
    Ta3Parser_Delete(ps);

    if (n == NULL) {
        if (tok->done == E_EOF)
            err_ret->error = E_EOF;
        err_ret->lineno = tok->lineno;
        if (tok->buf != NULL) {
            size_t len;
            assert(tok->cur - tok->buf < INT_MAX);
            err_ret->offset = (int)(tok->cur - tok->buf);
            len = tok->inp - tok->buf;
            err_ret->text = (char *) PyObject_MALLOC(len + 1);
            if (err_ret->text != NULL) {
                if (len > 0)
                    strncpy(err_ret->text, tok->buf, len);
                err_ret->text[len] = '\0';
            }
        }
    } else if (tok->encoding != NULL) {
        /* 'nodes->n_str' uses PyObject_*, while 'tok->encoding' was
         * allocated using PyMem_
         */
        node* r = Ta3Node_New(encoding_decl);
        if (r)
            r->n_str = PyObject_MALLOC(strlen(tok->encoding)+1);
        if (!r || !r->n_str) {
            err_ret->error = E_NOMEM;
            if (r)
                PyObject_FREE(r);
            n = NULL;
            goto done;
        }
        strcpy(r->n_str, tok->encoding);
        PyMem_FREE(tok->encoding);
        tok->encoding = NULL;
        r->n_nchildren = 1;
        r->n_child = n;
        n = r;
    }

done:
    Ta3Tokenizer_Free(tok);

    return n;
}

Project: typed_ast

Import(asdl_seq * names, int lineno, int col_offset, PyArena *arena)
{
    stmt_ty p;
    p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = Import_kind;
    p->v.Import.names = names;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast

ast_for_arguments(struct compiling *c, const node *n)
{
    /* parameters: '(' [varargslist] ')'
       varargslist: ((fpdef ['=' test] ',' [TYPE_COMMENT])*
                     ('*' NAME [',' [TYPE_COMMENT]  '**' NAME] [TYPE_COMMENT] | '**' NAME [TYPE_COMMENT]) |
                     fpdef ['=' test] (',' [TYPE_COMMENT] fpdef ['=' test])* [','] [TYPE_COMMENT])
    */
    int i, j, k, l, n_args = 0, n_all_args = 0, n_defaults = 0, found_default = 0;
    asdl_seq *args, *defaults, *type_comments = NULL;
    identifier vararg = NULL, kwarg = NULL;
    node *ch;

    if (TYPE(n) == parameters) {
        if (NCH(n) == 2) /* () as argument list */
            return arguments(NULL, NULL, NULL, NULL, NULL, c->c_arena);
        n = CHILD(n, 1);
    }
    REQ(n, varargslist);

    /* first count the number of normal args & defaults */
    for (i = 0; i < NCH(n); i++) {
        ch = CHILD(n, i);
        if (TYPE(ch) == fpdef)
            n_args++;
        if (TYPE(ch) == EQUAL)
            n_defaults++;
        if (TYPE(ch) == STAR || TYPE(ch) == DOUBLESTAR)
            n_all_args++;
    }
    n_all_args += n_args;
    args = (n_args ? asdl_seq_new(n_args, c->c_arena) : NULL);
    if (!args && n_args)
        return NULL;
    defaults = (n_defaults ? asdl_seq_new(n_defaults, c->c_arena) : NULL);
    if (!defaults && n_defaults)
        return NULL;
    /* type_comments will be lazily initialized if needed.  If there are no
       per-argument type comments, it will remain NULL.  Otherwise, it will be
       an asdl_seq with length equal to the number of args (including varargs
       and kwargs, if present) and with members set to the string of each arg's
       type comment, if present, or NULL otherwise.
     */

    /* fpdef: NAME | '(' fplist ')'
       fplist: fpdef (',' fpdef)* [',']
    */
    i = 0;
    j = 0;  /* index for defaults */
    k = 0;  /* index for args */
    l = 0;  /* index for type comments */
    while (i < NCH(n)) {
        ch = CHILD(n, i);
        switch (TYPE(ch)) {
            case fpdef: {
                int complex_args = 0, parenthesized = 0;
            handle_fpdef:
                /* XXX Need to worry about checking if TYPE(CHILD(n, i+1)) is
                   anything other than EQUAL or a comma? */
                /* XXX Should NCH(n) check be made a separate check? */
                if (i + 1 < NCH(n) && TYPE(CHILD(n, i + 1)) == EQUAL) {
                    expr_ty expression = ast_for_expr(c, CHILD(n, i + 2));
                    if (!expression)
                        return NULL;
                    assert(defaults != NULL);
                    asdl_seq_SET(defaults, j++, expression);
                    i += 2;
                    found_default = 1;
                }
                else if (found_default) {
                    /* def f((x)=4): pass should raise an error.
                       def f((x, (y))): pass will just incur the tuple unpacking warning. */
                    if (parenthesized && !complex_args) {
                        ast_error(n, "parenthesized arg with default");
                        return NULL;
                    }
                    ast_error(n,
                             "non-default argument follows default argument");
                    return NULL;
                }
                if (NCH(ch) == 3) {
                    ch = CHILD(ch, 1);
                    /* def foo((x)): is not complex, special case. */
                    if (NCH(ch) != 1) {
                        /* We have complex arguments, setup for unpacking. */
                        if (Py_Py3kWarningFlag && !ast_warn(c, ch,
                            "tuple parameter unpacking has been removed in 3.x"))
                            return NULL;
                        complex_args = 1;
                        asdl_seq_SET(args, k++, compiler_complex_args(c, ch));
                        if (!asdl_seq_GET(args, k-1))
                                return NULL;
                    } else {
                        /* def foo((x)): setup for checking NAME below. */
                        /* Loop because there can be many parens and tuple
                           unpacking mixed in. */
                        parenthesized = 1;
                        ch = CHILD(ch, 0);
                        assert(TYPE(ch) == fpdef);
                        goto handle_fpdef;
                    }
                }
                if (TYPE(CHILD(ch, 0)) == NAME) {
                    PyObject *id;
                    expr_ty name;
                    if (!forbidden_check(c, n, STR(CHILD(ch, 0))))
                        return NULL;
                    id = NEW_IDENTIFIER(CHILD(ch, 0));
                    if (!id)
                        return NULL;
                    name = Name(id, Param, LINENO(ch), ch->n_col_offset,
                                c->c_arena);
                    if (!name)
                        return NULL;
                    asdl_seq_SET(args, k++, name);

                }
                i += 1; /* the name */
                if (i < NCH(n) && TYPE(CHILD(n, i)) == COMMA)
                    i += 1; /* the comma, if present */
                if (parenthesized && Py_Py3kWarningFlag &&
                    !ast_warn(c, ch, "parenthesized argument names "
                              "are invalid in 3.x"))
                    return NULL;

                break;
            }
            case STAR:
                if (!forbidden_check(c, CHILD(n, i+1), STR(CHILD(n, i+1))))
                    return NULL;
                vararg = NEW_IDENTIFIER(CHILD(n, i+1));
                if (!vararg)
                    return NULL;
                i += 2; /* the star and the name */
                if (i < NCH(n) && TYPE(CHILD(n, i)) == COMMA)
                    i += 1; /* the comma, if present */
                break;
            case DOUBLESTAR:
                if (!forbidden_check(c, CHILD(n, i+1), STR(CHILD(n, i+1))))
                    return NULL;
                kwarg = NEW_IDENTIFIER(CHILD(n, i+1));
                if (!kwarg)
                    return NULL;
                i += 2; /* the double star and the name */
                if (i < NCH(n) && TYPE(CHILD(n, i)) == COMMA)
                    i += 1; /* the comma, if present */
                break;
            case TYPE_COMMENT:
                assert(l < k + !!vararg + !!kwarg);

                if (!type_comments) {
                    /* lazily allocate the type_comments seq for perf reasons */
                    type_comments = asdl_seq_new(n_all_args, c->c_arena);
                    if (!type_comments)
                        return NULL;
                }

                while (l < k + !!vararg + !!kwarg - 1) {
                    asdl_seq_SET(type_comments, l++, NULL);
                }

                asdl_seq_SET(type_comments, l++, NEW_TYPE_COMMENT(ch));
                i += 1;
                break;
            default:
                PyErr_Format(PyExc_SystemError,
                             "unexpected node in varargslist: %d @ %d",
                             TYPE(ch), i);
                return NULL;
        }
    }

    if (type_comments) {
        while (l < n_all_args) {
            asdl_seq_SET(type_comments, l++, NULL);
        }
    }

    return arguments(args, vararg, kwarg, defaults, type_comments, c->c_arena);
}

Project: typed_ast

Ta3Grammar_LabelRepr(label *lb)
{
    static char buf[100];

    if (lb->lb_type == ENDMARKER)
        return "EMPTY";
    else if (ISNONTERMINAL(lb->lb_type)) {
        if (lb->lb_str == NULL) {
            PyOS_snprintf(buf, sizeof(buf), "NT%d", lb->lb_type);
            return buf;
        }
        else
            return lb->lb_str;
    }
    else if (lb->lb_type < N_TOKENS) {
        if (lb->lb_str == NULL)
            return _Ta3Parser_TokenNames[lb->lb_type];
        else {
            PyOS_snprintf(buf, sizeof(buf), "%.32s(%.32s)",
                _Ta3Parser_TokenNames[lb->lb_type], lb->lb_str);
            return buf;
        }
    }
    else {
        Py_FatalError("invalid label");
        return NULL;
    }
}

Project: typed_ast

ast_for_dictcomp(struct compiling *c, const node *n)
{
    expr_ty key, value;
    asdl_seq *comps;

    assert(NCH(n) > 3);
    REQ(CHILD(n, 1), COLON);

    key = ast_for_expr(c, CHILD(n, 0));
    if (!key)
        return NULL;

    value = ast_for_expr(c, CHILD(n, 2));
    if (!value)
        return NULL;

    comps = ast_for_comprehension(c, CHILD(n, 3));
    if (!comps)
        return NULL;

    return DictComp(key, value, comps, LINENO(n), n->n_col_offset, c->c_arena);
}

Project: typed_ast

Ta3Token_OneChar(int c)
{
    switch (c) {
    case '(':           return LPAR;
    case ')':           return RPAR;
    case '[':           return LSQB;
    case ']':           return RSQB;
    case ':':           return COLON;
    case ',':           return COMMA;
    case ';':           return SEMI;
    case '+':           return PLUS;
    case '-':           return MINUS;
    case '*':           return STAR;
    case '/':           return SLASH;
    case '|':           return VBAR;
    case '&':           return AMPER;
    case '<':           return LESS;
    case '>':           return GREATER;
    case '=':           return EQUAL;
    case '.':           return DOT;
    case '%':           return PERCENT;
    case '{':           return LBRACE;
    case '}':           return RBRACE;
    case '^':           return CIRCUMFLEX;
    case '~':           return TILDE;
    case '@':           return AT;
    default:            return OP;
    }
}

Project: typed_ast
File: ast.c

count_comp_fors(struct compiling *c, const node *n)
{
    int n_fors = 0;

  count_comp_for:
    n_fors++;
    REQ(n, comp_for);
    if (NCH(n) == 2) {
        REQ(CHILD(n, 0), NAME);
        assert(strcmp(STR(CHILD(n, 0)), "async") == 0);
        n = CHILD(n, 1);
    }
    else if (NCH(n) == 1) {
        n = CHILD(n, 0);
    }
    else {
        goto error;
    }
    if (NCH(n) == (5)) {
        n = CHILD(n, 4);
    }
    else {
        return n_fors;
    }
  count_comp_iter:
    REQ(n, comp_iter);
    n = CHILD(n, 0);
    if (TYPE(n) == comp_for)
        goto count_comp_for;
    else if (TYPE(n) == comp_if) {
        if (NCH(n) == 3) {
            n = CHILD(n, 2);
            goto count_comp_iter;
        }
        else
            return n_fors;
    }

  error:
    /* Should never be reached */
    PyErr_SetString(PyExc_SystemError,
                    "logic error in count_comp_fors");
    return -1;
}

Project: typed_ast

ExceptHandler(expr_ty type, identifier name, asdl_seq * body, int lineno, int
              col_offset, PyArena *arena)
{
    excepthandler_ty p;
    p = (excepthandler_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = ExceptHandler_kind;
    p->v.ExceptHandler.type = type;
    p->v.ExceptHandler.name = name;
    p->v.ExceptHandler.body = body;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast

ast_for_atom(struct compiling *c, const node *n)
{
    /* atom: '(' [yield_expr|testlist_comp] ')' | '[' [listmaker] ']'
       | '{' [dictmaker] '}' | '`' testlist '`' | NAME | NUMBER | STRING+
    */
    node *ch = CHILD(n, 0);

    switch (TYPE(ch)) {
    case NAME: {
        /* All names start in Load context, but may later be
           changed. */
        PyObject *name = NEW_IDENTIFIER(ch);
        if (!name)
            return NULL;
        return Name(name, Load, LINENO(n), n->n_col_offset, c->c_arena);
    }
    case STRING: {
        PyObject *kind, *str = parsestrplus(c, n);
        const char *raw, *s = STR(CHILD(n, 0));
        /* currently Python allows up to 2 string modifiers */
        char *ch, s_kind[3] = {0, 0, 0};
        ch = s_kind;
        raw = s;
        while (*raw && *raw != '\'' && *raw != '"') {
            *ch++ = *raw++;
        }
        kind = PyUnicode_FromString(s_kind);
        if (!kind) {
            return NULL;
        }
        if (!str) {
#ifdef Py_USING_UNICODE
            if (PyErr_ExceptionMatches(PyExc_UnicodeError)){
                PyObject *type, *value, *tback, *errstr;
                PyErr_Fetch(&type, &value, &tback);
                errstr = PyObject_Str(value);
                if (errstr) {
                    const char *s = "";
                    char buf[128];
                    s = _PyUnicode_AsString(errstr);
                    PyOS_snprintf(buf, sizeof(buf), "(unicode error) %s", s);
                    ast_error(n, buf);
                    Py_DECREF(errstr);
                } else {
                    ast_error(n, "(unicode error) unknown error");
                }
                Py_DECREF(type);
                Py_DECREF(value);
                Py_XDECREF(tback);
            }
#endif
            return NULL;
        }
        PyArena_AddPyObject(c->c_arena, str);
        return Str(str, kind, LINENO(n), n->n_col_offset, c->c_arena);
    }
    case NUMBER: {
        PyObject *pynum = parsenumber(c, STR(ch));
        if (!pynum)
            return NULL;

        PyArena_AddPyObject(c->c_arena, pynum);
        return Num(pynum, LINENO(n), n->n_col_offset, c->c_arena);
    }
    case LPAR: /* some parenthesized expressions */
        ch = CHILD(n, 1);

        if (TYPE(ch) == RPAR)
            return Tuple(NULL, Load, LINENO(n), n->n_col_offset, c->c_arena);

        if (TYPE(ch) == yield_expr)
            return ast_for_expr(c, ch);

        return ast_for_testlist_comp(c, ch);
    case LSQB: /* list (or list comprehension) */
        ch = CHILD(n, 1);

        if (TYPE(ch) == RSQB)
            return List(NULL, Load, LINENO(n), n->n_col_offset, c->c_arena);

        REQ(ch, listmaker);
        if (NCH(ch) == 1 || TYPE(CHILD(ch, 1)) == COMMA) {
            asdl_seq *elts = seq_for_testlist(c, ch);
            if (!elts)
                return NULL;

            return List(elts, Load, LINENO(n), n->n_col_offset, c->c_arena);
        }
        else
            return ast_for_listcomp(c, ch);
    case LBRACE: {
        /* dictorsetmaker:
         *    (test ':' test (comp_for | (',' test ':' test)* [','])) |
         *    (test (comp_for | (',' test)* [',']))
         */
        int i, size;
        asdl_seq *keys, *values;

        ch = CHILD(n, 1);
        if (TYPE(ch) == RBRACE) {
            /* it's an empty dict */
            return Dict(NULL, NULL, LINENO(n), n->n_col_offset, c->c_arena);
        } else if (NCH(ch) == 1 || TYPE(CHILD(ch, 1)) == COMMA) {
            /* it's a simple set */
            asdl_seq *elts;
            size = (NCH(ch) + 1) / 2; /* +1 in case no trailing comma */
            elts = asdl_seq_new(size, c->c_arena);
            if (!elts)
                return NULL;
            for (i = 0; i < NCH(ch); i += 2) {
                expr_ty expression;
                expression = ast_for_expr(c, CHILD(ch, i));
                if (!expression)
                    return NULL;
                asdl_seq_SET(elts, i / 2, expression);
            }
            return Set(elts, LINENO(n), n->n_col_offset, c->c_arena);
        } else if (TYPE(CHILD(ch, 1)) == comp_for) {
            /* it's a set comprehension */
            return ast_for_setcomp(c, ch);
        } else if (NCH(ch) > 3 && TYPE(CHILD(ch, 3)) == comp_for) {
            return ast_for_dictcomp(c, ch);
        } else {
            /* it's a dict */
            size = (NCH(ch) + 1) / 4; /* +1 in case no trailing comma */
            keys = asdl_seq_new(size, c->c_arena);
            if (!keys)
                return NULL;

            values = asdl_seq_new(size, c->c_arena);
            if (!values)
                return NULL;

            for (i = 0; i < NCH(ch); i += 4) {
                expr_ty expression;

                expression = ast_for_expr(c, CHILD(ch, i));
                if (!expression)
                    return NULL;

                asdl_seq_SET(keys, i / 4, expression);

                expression = ast_for_expr(c, CHILD(ch, i + 2));
                if (!expression)
                    return NULL;

                asdl_seq_SET(values, i / 4, expression);
            }
            return Dict(keys, values, LINENO(n), n->n_col_offset, c->c_arena);
        }
    }
    case BACKQUOTE: { /* repr */
        expr_ty expression;
        if (Py_Py3kWarningFlag &&
            !ast_warn(c, n, "backquote not supported in 3.x; use repr()"))
            return NULL;
        expression = ast_for_testlist(c, CHILD(n, 1));
        if (!expression)
            return NULL;

        return Repr(expression, LINENO(n), n->n_col_offset, c->c_arena);
    }
    default:
        PyErr_Format(PyExc_SystemError, "unhandled atom %d", TYPE(ch));
        return NULL;
    }
}

Project: typed_ast

Expr(expr_ty value, int lineno, int col_offset, PyArena *arena)
{
    stmt_ty p;
    if (!value) {
        PyErr_SetString(PyExc_ValueError,
                        "field value is required for Expr");
        return NULL;
    }
    p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = Expr_kind;
    p->v.Expr.value = value;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast

static int obj2ast_singleton(PyObject *obj, PyObject** out, PyArena* arena)
{
    if (obj != Py_None && obj != Py_True && obj != Py_False) {
        PyErr_SetString(PyExc_ValueError,
                        "AST singleton must be True, False, or None");
        return 1;
    }
    *out = obj;
    return 0;
}

Project: typed_ast

forbidden_check(struct compiling *c, const node *n, const char *x)
{
    if (!strcmp(x, "None"))
        return ast_error(n, "cannot assign to None");
    if (!strcmp(x, "__debug__"))
        return ast_error(n, "cannot assign to __debug__");
    if (Py_Py3kWarningFlag) {
        if (!(strcmp(x, "True") && strcmp(x, "False")) &&
            !ast_warn(c, n, "assignment to True or False is forbidden in 3.x"))
            return 0;
        if (!strcmp(x, "nonlocal") &&
            !ast_warn(c, n, "nonlocal is a keyword in 3.x"))
            return 0;
    }
    return 1;
}

Project: typed_ast

ast_for_comprehension(struct compiling *c, const node *n)
{
    int i, n_fors;
    asdl_seq *comps;

    n_fors = count_comp_fors(c, n);
    if (n_fors == -1)
        return NULL;

    comps = asdl_seq_new(n_fors, c->c_arena);
    if (!comps)
        return NULL;

    for (i = 0; i < n_fors; i++) {
        comprehension_ty comp;
        asdl_seq *t;
        expr_ty expression, first;
        node *for_ch;

        REQ(n, comp_for);

        for_ch = CHILD(n, 1);
        t = ast_for_exprlist(c, for_ch, Store);
        if (!t)
            return NULL;
        expression = ast_for_expr(c, CHILD(n, 3));
        if (!expression)
            return NULL;

        /* Check the # of children rather than the length of t, since
           (x for x, in ...) has 1 element in t, but still requires a Tuple. */
        first = (expr_ty)asdl_seq_GET(t, 0);
        if (NCH(for_ch) == 1)
            comp = comprehension(first, expression, NULL, c->c_arena);
        else
            comp = comprehension(Tuple(t, Store, first->lineno, first->col_offset,
                                     c->c_arena),
                               expression, NULL, c->c_arena);
        if (!comp)
            return NULL;

        if (NCH(n) == 5) {
            int j, n_ifs;
            asdl_seq *ifs;

            n = CHILD(n, 4);
            n_ifs = count_comp_ifs(c, n);
            if (n_ifs == -1)
                return NULL;

            ifs = asdl_seq_new(n_ifs, c->c_arena);
            if (!ifs)
                return NULL;

            for (j = 0; j < n_ifs; j++) {
                REQ(n, comp_iter);
                n = CHILD(n, 0);
                REQ(n, comp_if);

                expression = ast_for_expr(c, CHILD(n, 1));
                if (!expression)
                    return NULL;
                asdl_seq_SET(ifs, j, expression);
                if (NCH(n) == 3)
                    n = CHILD(n, 2);
            }
            /* on exit, must guarantee that n is a comp_for */
            if (TYPE(n) == comp_iter)
                n = CHILD(n, 0);
            comp->ifs = ifs;
        }
        asdl_seq_SET(comps, i, comp);
    }
    return comps;
}

Project: typed_ast
File: Python-ast.c

obj2ast_operator(PyObject* obj, operator_ty* out, PyArena* arena)
{
    int isinstance;

    isinstance = PyObject_IsInstance(obj, (PyObject *)Add_type);
    if (isinstance == -1) {
        return 1;
    }
    if (isinstance) {
        *out = Add;
        return 0;
    }
    isinstance = PyObject_IsInstance(obj, (PyObject *)Sub_type);
    if (isinstance == -1) {
        return 1;
    }
    if (isinstance) {
        *out = Sub;
        return 0;
    }
    isinstance = PyObject_IsInstance(obj, (PyObject *)Mult_type);
    if (isinstance == -1) {
        return 1;
    }
    if (isinstance) {
        *out = Mult;
        return 0;
    }
    isinstance = PyObject_IsInstance(obj, (PyObject *)MatMult_type);
    if (isinstance == -1) {
        return 1;
    }
    if (isinstance) {
        *out = MatMult;
        return 0;
    }
    isinstance = PyObject_IsInstance(obj, (PyObject *)Div_type);
    if (isinstance == -1) {
        return 1;
    }
    if (isinstance) {
        *out = Div;
        return 0;
    }
    isinstance = PyObject_IsInstance(obj, (PyObject *)Mod_type);
    if (isinstance == -1) {
        return 1;
    }
    if (isinstance) {
        *out = Mod;
        return 0;
    }
    isinstance = PyObject_IsInstance(obj, (PyObject *)Pow_type);
    if (isinstance == -1) {
        return 1;
    }
    if (isinstance) {
        *out = Pow;
        return 0;
    }
    isinstance = PyObject_IsInstance(obj, (PyObject *)LShift_type);
    if (isinstance == -1) {
        return 1;
    }
    if (isinstance) {
        *out = LShift;
        return 0;
    }
    isinstance = PyObject_IsInstance(obj, (PyObject *)RShift_type);
    if (isinstance == -1) {
        return 1;
    }
    if (isinstance) {
        *out = RShift;
        return 0;
    }
    isinstance = PyObject_IsInstance(obj, (PyObject *)BitOr_type);
    if (isinstance == -1) {
        return 1;
    }
    if (isinstance) {
        *out = BitOr;
        return 0;
    }
    isinstance = PyObject_IsInstance(obj, (PyObject *)BitXor_type);
    if (isinstance == -1) {
        return 1;
    }
    if (isinstance) {
        *out = BitXor;
        return 0;
    }
    isinstance = PyObject_IsInstance(obj, (PyObject *)BitAnd_type);
    if (isinstance == -1) {
        return 1;
    }
    if (isinstance) {
        *out = BitAnd;
        return 0;
    }
    isinstance = PyObject_IsInstance(obj, (PyObject *)FloorDiv_type);
    if (isinstance == -1) {
        return 1;
    }
    if (isinstance) {
        *out = FloorDiv;
        return 0;
    }

    PyErr_Format(PyExc_TypeError, "expected some sort of operator, but got %R", obj);
    return 1;
}

Project: typed_ast

arg(identifier arg, expr_ty annotation, string type_comment, int lineno, int
    col_offset, PyArena *arena)
{
    arg_ty p;
    if (!arg) {
        PyErr_SetString(PyExc_ValueError,
                        "field arg is required for arg");
        return NULL;
    }
    p = (arg_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->arg = arg;
    p->annotation = annotation;
    p->type_comment = type_comment;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast
File: tokenizer.c

decode_str(const char *input, int single, struct tok_state *tok)
{
    PyObject* utf8 = NULL;
    const char *str;
    const char *s;
    const char *newl[2] = {NULL, NULL};
    int lineno = 0;
    tok->input = str = translate_newlines(input, single, tok);
    if (str == NULL)
        return NULL;
    tok->enc = NULL;
    tok->str = str;
    if (!check_bom(buf_getc, buf_ungetc, buf_setreadl, tok))
        return error_ret(tok);
    str = tok->str;             /* string after BOM if any */
    assert(str);
    if (tok->enc != NULL) {
        utf8 = translate_into_utf8(str, tok->enc);
        if (utf8 == NULL)
            return error_ret(tok);
        str = PyBytes_AsString(utf8);
    }
    for (s = str;; s++) {
        if (*s == '\0') break;
        else if (*s == '\n') {
            assert(lineno < 2);
            newl[lineno] = s;
            lineno++;
            if (lineno == 2) break;
        }
    }
    tok->enc = NULL;
    /* need to check line 1 and 2 separately since check_coding_spec
       assumes a single line as input */
    if (newl[0]) {
        if (!check_coding_spec(str, newl[0] - str, tok, buf_setreadl))
            return error_ret(tok);
        if (tok->enc == NULL && !tok->read_coding_spec && newl[1]) {
            if (!check_coding_spec(newl[0]+1, newl[1] - newl[0],
                                   tok, buf_setreadl))
                return error_ret(tok);
        }
    }
    if (tok->enc != NULL) {
        assert(utf8 == NULL);
        utf8 = translate_into_utf8(str, tok->enc);
        if (utf8 == NULL)
            return error_ret(tok);
        str = PyBytes_AS_STRING(utf8);
    }
    assert(tok->decoding_buffer == NULL);
    tok->decoding_buffer = utf8; /* CAUTION */
    return str;
}

Project: typed_ast

Ta3Parser_ParseFileFlagsEx(FILE *fp, const char *filename,
                          const char *enc, grammar *g, int start,
                          const char *ps1, const char *ps2,
                          perrdetail *err_ret, int *flags)
{
    node *n;
    PyObject *fileobj = NULL;
#ifndef PGEN
    if (filename != NULL) {
        fileobj = PyUnicode_DecodeFSDefault(filename);
        if (fileobj == NULL) {
            err_ret->error = E_ERROR;
            return NULL;
        }
    }
#endif
    n = Ta3Parser_ParseFileObject(fp, fileobj, enc, g,
                                 start, ps1, ps2, err_ret, flags);
#ifndef PGEN
    Py_XDECREF(fileobj);
#endif
    return n;
}

Project: typed_ast
File: Python-ast.c

Expression(expr_ty body, PyArena *arena)
{
    mod_ty p;
    if (!body) {
        PyErr_SetString(PyExc_ValueError,
                        "field body is required for Expression");
        return NULL;
    }
    p = (mod_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = Expression_kind;
    p->v.Expression.body = body;
    return p;
}

Project: typed_ast
File: tokenizer.c

get_coding_spec(const char *s, char **spec, Py_ssize_t size, struct tok_state *tok)
{
    Py_ssize_t i;
    *spec = NULL;
    /* Coding spec must be in a comment, and that comment must be
     * the only statement on the source code line. */
    for (i = 0; i < size - 6; i++) {
        if (s[i] == '#')
            break;
        if (s[i] != ' ' && s[i] != '\t' && s[i] != '\014')
            return 1;
    }
    for (; i < size - 6; i++) { /* XXX inefficient search */
        const char* t = s + i;
        if (strncmp(t, "coding", 6) == 0) {
            const char* begin = NULL;
            t += 6;
            if (t[0] != ':' && t[0] != '=')
                continue;
            do {
                t++;
            } while (t[0] == '\x20' || t[0] == '\t');

            begin = t;
            while (Py_ISALNUM(t[0]) ||
                   t[0] == '-' || t[0] == '_' || t[0] == '.')
                t++;

            if (begin < t) {
                char* r = new_string(begin, t - begin, tok);
                const char* q;
                if (!r)
                    return 0;
                q = get_normal_name(r);
                if (r != q) {
                    PyMem_FREE(r);
                    r = new_string(q, strlen(q), tok);
                    if (!r)
                        return 0;
                }
                *spec = r;
                break;
            }
        }
    }
    return 1;
}

Project: typed_ast
File: tokenizer.c

static void fp_ungetc(int c, struct tok_state *tok) {
    ungetc(c, tok->fp);
}

Project: typed_ast
File: Python-ast.c

obj2ast_boolop(PyObject* obj, boolop_ty* out, PyArena* arena)
{
    int isinstance;

    isinstance = PyObject_IsInstance(obj, (PyObject *)And_type);
    if (isinstance == -1) {
        return 1;
    }
    if (isinstance) {
        *out = And;
        return 0;
    }
    isinstance = PyObject_IsInstance(obj, (PyObject *)Or_type);
    if (isinstance == -1) {
        return 1;
    }
    if (isinstance) {
        *out = Or;
        return 0;
    }

    PyErr_Format(PyExc_TypeError, "expected some sort of boolop, but got %R", obj);
    return 1;
}

Project: typed_ast

Ta3Parser_ParseStringFlagsFilenameEx(const char *s, const char *filename_str,
                          grammar *g, int start,
                          perrdetail *err_ret, int *flags)
{
    node *n;
    PyObject *filename = NULL;
#ifndef PGEN
    if (filename_str != NULL) {
        filename = PyUnicode_DecodeFSDefault(filename_str);
        if (filename == NULL) {
            err_ret->error = E_ERROR;
            return NULL;
        }
    }
#endif
    n = Ta3Parser_ParseStringObject(s, filename, g, start, err_ret, flags);
#ifndef PGEN
    Py_XDECREF(filename);
#endif
    return n;
}

Project: typed_ast

ast_for_slice(struct compiling *c, const node *n)
{
    node *ch;
    expr_ty lower = NULL, upper = NULL, step = NULL;

    REQ(n, subscript);

    /*
       subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop]
       sliceop: ':' [test]
    */
    ch = CHILD(n, 0);
    if (TYPE(ch) == DOT)
        return Ellipsis(c->c_arena);

    if (NCH(n) == 1 && TYPE(ch) == test) {
        /* 'step' variable hold no significance in terms of being used over
           other vars */
        step = ast_for_expr(c, ch);
        if (!step)
            return NULL;

        return Index(step, c->c_arena);
    }

    if (TYPE(ch) == test) {
        lower = ast_for_expr(c, ch);
        if (!lower)
            return NULL;
    }

    /* If there's an upper bound it's in the second or third position. */
    if (TYPE(ch) == COLON) {
        if (NCH(n) > 1) {
            node *n2 = CHILD(n, 1);

            if (TYPE(n2) == test) {
                upper = ast_for_expr(c, n2);
                if (!upper)
                    return NULL;
            }
        }
    } else if (NCH(n) > 2) {
        node *n2 = CHILD(n, 2);

        if (TYPE(n2) == test) {
            upper = ast_for_expr(c, n2);
            if (!upper)
                return NULL;
        }
    }

    ch = CHILD(n, NCH(n) - 1);
    if (TYPE(ch) == sliceop) {
        if (NCH(ch) == 1) {
            /*
              This is an extended slice (ie "x[::]") with no expression in the
              step field. We set this literally to "None" in order to
              disambiguate it from x[:]. (The interpreter might have to call
              __getslice__ for x[:], but it must call __getitem__ for x[::].)
            */
            identifier none = new_identifier("None", c->c_arena);
            if (!none)
                return NULL;
            ch = CHILD(ch, 0);
            step = Name(none, Load, LINENO(ch), ch->n_col_offset, c->c_arena);
            if (!step)
                return NULL;
        } else {
            ch = CHILD(ch, 1);
            if (TYPE(ch) == test) {
                step = ast_for_expr(c, ch);
                if (!step)
                    return NULL;
            }
        }
    }

    return Slice(lower, upper, step, c->c_arena);
}

Project: typed_ast
File: Python-ast.c

arguments(asdl_seq * args, arg_ty vararg, asdl_seq * kwonlyargs, asdl_seq *
          kw_defaults, arg_ty kwarg, asdl_seq * defaults, PyArena *arena)
{
    arguments_ty p;
    p = (arguments_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->args = args;
    p->vararg = vararg;
    p->kwonlyargs = kwonlyargs;
    p->kw_defaults = kw_defaults;
    p->kwarg = kwarg;
    p->defaults = defaults;
    return p;
}

Project: typed_ast

ast_type_reduce(PyObject *self, PyObject *unused)
{
    _Py_IDENTIFIER(__dict__);
    PyObject *dict;
    if (lookup_attr_id(self, &PyId___dict__, &dict) < 0) {
        return NULL;
    }
    if (dict) {
        return Py_BuildValue("O()N", Py_TYPE(self), dict);
    }
    return Py_BuildValue("O()", Py_TYPE(self));
}

Project: typed_ast
File: parsetok.c

warn(const char *msg, const char *filename, int lineno)
{
    if (filename == NULL)
        filename = "<string>";
    PySys_WriteStderr(msg, filename, lineno);
}

Project: typed_ast

Set(asdl_seq * elts, int lineno, int col_offset, PyArena *arena)
{
    expr_ty p;
    p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
    if (!p)
        return NULL;
    p->kind = Set_kind;
    p->v.Set.elts = elts;
    p->lineno = lineno;
    p->col_offset = col_offset;
    return p;
}

Project: typed_ast

parsestr(struct compiling *c, const node *n, const char *s)
{
        size_t len, i;
        int quote = Py_CHARMASK(*s);
        int rawmode = 0;
        int need_encoding;
        int unicode = c->c_future_unicode;
        int bytes = 0;

        if (isalpha(quote) || quote == '_') {
                if (quote == 'u' || quote == 'U') {
                        quote = *++s;
                        unicode = 1;
                }
                if (quote == 'b' || quote == 'B') {
                        quote = *++s;
                        unicode = 0;
                        bytes = 1;
                }
                if (quote == 'r' || quote == 'R') {
                        quote = *++s;
                        rawmode = 1;
                }
        }
        if (quote != '\'' && quote != '\"') {
                PyErr_BadInternalCall();
                return NULL;
        }
        s++;
        len = strlen(s);
        if (len > INT_MAX) {
                PyErr_SetString(PyExc_OverflowError,
                                "string to parse is too long");
                return NULL;
        }
        if (s[--len] != quote) {
                PyErr_BadInternalCall();
                return NULL;
        }
        if (len >= 4 && s[0] == quote && s[1] == quote) {
                s += 2;
                len -= 2;
                if (s[--len] != quote || s[--len] != quote) {
                        PyErr_BadInternalCall();
                        return NULL;
                }
        }
        if (Py_Py3kWarningFlag && bytes) {
            for (i = 0; i < len; i++) {
                if ((unsigned char)s[i] > 127) {
                    if (!ast_warn(c, n,
                        "non-ascii bytes literals not supported in 3.x"))
                        return NULL;
                    break;
                }
            }
        }
#ifdef Py_USING_UNICODE
        if (unicode || Py_UnicodeFlag) {
                return decode_unicode(c, s, len, rawmode, c->c_encoding);
        }
#endif
        need_encoding = (c->c_encoding != NULL &&
                         strcmp(c->c_encoding, "utf-8") != 0 &&
                         strcmp(c->c_encoding, "iso-8859-1") != 0);
        if (rawmode || strchr(s, '\\') == NULL) {
                if (need_encoding) {
#ifndef Py_USING_UNICODE
                        /* This should not happen - we never see any other
                           encoding. */
                        Py_FatalError(
                            "cannot deal with encodings in this build.");
#else
                        PyObject *v, *u = PyUnicode_DecodeUTF8(s, len, NULL);
                        if (u == NULL)
                                return NULL;
                        v = PyUnicode_AsEncodedString(u, c->c_encoding, NULL);
                        Py_DECREF(u);
                        return v;
#endif
                } else {
                  return PyBytes_FromStringAndSize(s, len);
                }
        }

        return PyBytes_DecodeEscape(s, len, NULL, 1,
                                    need_encoding ? c->c_encoding : NULL);
}